From 74db647dbbaa224ffa5667a3203d53f48532d072 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Wed, 14 Mar 2018 15:21:24 -0300 Subject: [PATCH] CLOUDSTACK-10321: CPU Cap for KVM (#2482) --- .../cloud/agent/api/to/VirtualMachineTO.java | 10 ++ .../resource/LibvirtComputingResource.java | 28 +++ .../hypervisor/kvm/resource/LibvirtVMDef.java | 27 +++ .../LibvirtComputingResourceTest.java | 39 ++++ server/src/com/cloud/hypervisor/KVMGuru.java | 50 ++++++ .../com/cloud/hypervisor/KVMGuruTest.java | 99 ++++++++++ .../smoke/test_service_offerings.py | 169 +++++++++++++++++- 7 files changed, 419 insertions(+), 3 deletions(-) create mode 100644 server/test/com/cloud/hypervisor/KVMGuruTest.java diff --git a/api/src/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/com/cloud/agent/api/to/VirtualMachineTO.java index f982e4b5ca9..84a6bf5fc16 100644 --- a/api/src/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/com/cloud/agent/api/to/VirtualMachineTO.java @@ -70,6 +70,8 @@ public class VirtualMachineTO { String configDriveIsoRootFolder = null; String configDriveIsoFile = null; + Double cpuQuotaPercentage = null; + Map guestOsDetails = new HashMap(); public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader, @@ -340,4 +342,12 @@ public class VirtualMachineTO { public void setGuestOsDetails(Map guestOsDetails) { this.guestOsDetails = guestOsDetails; } + + public Double getCpuQuotaPercentage() { + return cpuQuotaPercentage; + } + + public void setCpuQuotaPercentage(Double cpuQuotaPercentage) { + this.cpuQuotaPercentage = cpuQuotaPercentage; + } } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 9b7fb2ea013..dd039e54263 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1984,6 +1984,31 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return uuid; } + /** + * Set quota and period tags on 'ctd' when CPU limit use is set + */ + protected void setQuotaAndPeriod(VirtualMachineTO vmTO, CpuTuneDef ctd) { + if (vmTO.getLimitCpuUse() && vmTO.getCpuQuotaPercentage() != null) { + Double cpuQuotaPercentage = vmTO.getCpuQuotaPercentage(); + int period = CpuTuneDef.DEFAULT_PERIOD; + int quota = (int) (period * cpuQuotaPercentage); + if (quota < CpuTuneDef.MIN_QUOTA) { + s_logger.info("Calculated quota (" + quota + ") below the minimum (" + CpuTuneDef.MIN_QUOTA + ") for VM domain " + vmTO.getUuid() + ", setting it to minimum " + + "and calculating period instead of using the default"); + quota = CpuTuneDef.MIN_QUOTA; + period = (int) ((double) quota / cpuQuotaPercentage); + if (period > CpuTuneDef.MAX_PERIOD) { + s_logger.info("Calculated period (" + period + ") exceeds the maximum (" + CpuTuneDef.MAX_PERIOD + + "), setting it to the maximum"); + period = CpuTuneDef.MAX_PERIOD; + } + } + ctd.setQuota(quota); + ctd.setPeriod(period); + s_logger.info("Setting quota=" + quota + ", period=" + period + " to VM domain " + vmTO.getUuid()); + } + } + public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) { final LibvirtVMDef vm = new LibvirtVMDef(); vm.setDomainName(vmTO.getName()); @@ -2059,6 +2084,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } else { ctd.setShares(vmTO.getCpus() * vmTO.getSpeed()); } + + setQuotaAndPeriod(vmTO, ctd); + vm.addComp(ctd); } diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 90674eb99a8..7c12c0713c5 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -1171,6 +1171,11 @@ public class LibvirtVMDef { public static class CpuTuneDef { private int _shares = 0; + private int quota = 0; + private int period = 0; + static final int DEFAULT_PERIOD = 10000; + static final int MIN_QUOTA = 1000; + static final int MAX_PERIOD = 1000000; public void setShares(int shares) { _shares = shares; @@ -1180,6 +1185,22 @@ public class LibvirtVMDef { return _shares; } + public int getQuota() { + return quota; + } + + public void setQuota(int quota) { + this.quota = quota; + } + + public int getPeriod() { + return period; + } + + public void setPeriod(int period) { + this.period = period; + } + @Override public String toString() { StringBuilder cpuTuneBuilder = new StringBuilder(); @@ -1187,6 +1208,12 @@ public class LibvirtVMDef { if (_shares > 0) { cpuTuneBuilder.append("" + _shares + "\n"); } + if (quota > 0) { + cpuTuneBuilder.append("" + quota + "\n"); + } + if (period > 0) { + cpuTuneBuilder.append("" + period + "\n"); + } cpuTuneBuilder.append("\n"); return cpuTuneBuilder.toString(); } diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 2fd7692df58..795b96175ab 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -38,6 +38,7 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.CpuTuneDef; import org.apache.commons.lang.SystemUtils; import org.joda.time.Duration; import org.junit.Assert; @@ -184,6 +185,8 @@ public class LibvirtComputingResourceTest { @Mock private LibvirtComputingResource libvirtComputingResource; + @Mock + VirtualMachineTO vmTO; String hyperVisorType = "kvm"; Random random = new Random(); @@ -5152,4 +5155,40 @@ public class LibvirtComputingResourceTest { when(domainMock.memoryStats(2)).thenReturn(mem); return domainMock; } + + @Test + public void testSetQuotaAndPeriod() { + double pct = 0.33d; + Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true); + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct); + CpuTuneDef cpuTuneDef = new CpuTuneDef(); + final LibvirtComputingResource lcr = new LibvirtComputingResource(); + lcr.setQuotaAndPeriod(vmTO, cpuTuneDef); + Assert.assertEquals((int) (CpuTuneDef.DEFAULT_PERIOD * pct), cpuTuneDef.getQuota()); + Assert.assertEquals(CpuTuneDef.DEFAULT_PERIOD, cpuTuneDef.getPeriod()); + } + + @Test + public void testSetQuotaAndPeriodNoCpuLimitUse() { + double pct = 0.33d; + Mockito.when(vmTO.getLimitCpuUse()).thenReturn(false); + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct); + CpuTuneDef cpuTuneDef = new CpuTuneDef(); + final LibvirtComputingResource lcr = new LibvirtComputingResource(); + lcr.setQuotaAndPeriod(vmTO, cpuTuneDef); + Assert.assertEquals(0, cpuTuneDef.getQuota()); + Assert.assertEquals(0, cpuTuneDef.getPeriod()); + } + + @Test + public void testSetQuotaAndPeriodMinQuota() { + double pct = 0.01d; + Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true); + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct); + CpuTuneDef cpuTuneDef = new CpuTuneDef(); + final LibvirtComputingResource lcr = new LibvirtComputingResource(); + lcr.setQuotaAndPeriod(vmTO, cpuTuneDef); + Assert.assertEquals(CpuTuneDef.MIN_QUOTA, cpuTuneDef.getQuota()); + Assert.assertEquals((int) (CpuTuneDef.MIN_QUOTA / pct), cpuTuneDef.getPeriod()); + } } diff --git a/server/src/com/cloud/hypervisor/KVMGuru.java b/server/src/com/cloud/hypervisor/KVMGuru.java index 1a476a2948d..df6038d715a 100644 --- a/server/src/com/cloud/hypervisor/KVMGuru.java +++ b/server/src/com/cloud/hypervisor/KVMGuru.java @@ -28,11 +28,16 @@ import com.cloud.storage.GuestOSVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.GuestOSHypervisorDao; import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.log4j.Logger; import javax.inject.Inject; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.Map; public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { @@ -43,6 +48,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { @Inject HostDao _hostDao; + public static final Logger s_logger = Logger.getLogger(KVMGuru.class); + @Override public HypervisorType getHypervisorType() { return HypervisorType.KVM; @@ -52,10 +59,53 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { super(); } + /** + * Retrieve host max CPU speed + */ + protected double getHostCPUSpeed(HostVO host) { + return host.getSpeed(); + } + + protected double getVmSpeed(VirtualMachineTO to) { + return to.getMaxSpeed() != null ? to.getMaxSpeed() : to.getSpeed(); + } + + /** + * Set VM CPU quota percentage with respect to host CPU on 'to' if CPU limit option is set + * @param to vm to + * @param vmProfile vm profile + */ + protected void setVmQuotaPercentage(VirtualMachineTO to, VirtualMachineProfile vmProfile) { + if (to.getLimitCpuUse()) { + VirtualMachine vm = vmProfile.getVirtualMachine(); + HostVO host = _hostDao.findById(vm.getHostId()); + if (host == null) { + throw new CloudRuntimeException("Host with id: " + vm.getHostId() + " not found"); + } + s_logger.debug("Limiting CPU usage for VM: " + vm.getUuid() + " on host: " + host.getUuid()); + double hostMaxSpeed = getHostCPUSpeed(host); + double maxSpeed = getVmSpeed(to); + try { + BigDecimal percent = new BigDecimal(maxSpeed / hostMaxSpeed); + percent = percent.setScale(2, RoundingMode.HALF_DOWN); + if (percent.compareTo(new BigDecimal(1)) == 1) { + s_logger.debug("VM " + vm.getUuid() + " CPU MHz exceeded host " + host.getUuid() + " CPU MHz, limiting VM CPU to the host maximum"); + percent = new BigDecimal(1); + } + to.setCpuQuotaPercentage(percent.doubleValue()); + s_logger.debug("Host: " + host.getUuid() + " max CPU speed = " + hostMaxSpeed + "MHz, VM: " + vm.getUuid() + + "max CPU speed = " + maxSpeed + "MHz. Setting CPU quota percentage as: " + percent.doubleValue()); + } catch (NumberFormatException e) { + s_logger.error("Error calculating VM: " + vm.getUuid() + " quota percentage, it wll not be set. Error: " + e.getMessage(), e); + } + } + } + @Override public VirtualMachineTO implement(VirtualMachineProfile vm) { VirtualMachineTO to = toVirtualMachineTO(vm); + setVmQuotaPercentage(to, vm); // Determine the VM's OS description GuestOSVO guestOS = _guestOsDao.findByIdIncludingRemoved(vm.getVirtualMachine().getGuestOSId()); diff --git a/server/test/com/cloud/hypervisor/KVMGuruTest.java b/server/test/com/cloud/hypervisor/KVMGuruTest.java new file mode 100644 index 00000000000..597e20fe0af --- /dev/null +++ b/server/test/com/cloud/hypervisor/KVMGuruTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor; + +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class KVMGuruTest { + + @Mock + HostDao hostDao; + + @Spy + @InjectMocks + private KVMGuru guru = new KVMGuru(); + + @Mock + VirtualMachineTO vmTO; + @Mock + VirtualMachineProfile vmProfile; + @Mock + VirtualMachine vm; + @Mock + HostVO host; + + private static final long hostId = 1l; + + @Before + public void setup() { + Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true); + Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm); + Mockito.when(vm.getHostId()).thenReturn(hostId); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + Mockito.when(host.getCpus()).thenReturn(3); + Mockito.when(host.getSpeed()).thenReturn(1995l); + Mockito.when(vmTO.getMaxSpeed()).thenReturn(500); + } + + @Test + public void testSetVmQuotaPercentage() { + guru.setVmQuotaPercentage(vmTO, vmProfile); + Mockito.verify(vmTO).setCpuQuotaPercentage(Mockito.anyDouble()); + } + + @Test(expected = CloudRuntimeException.class) + public void testSetVmQuotaPercentageNullHost() { + Mockito.when(hostDao.findById(hostId)).thenReturn(null); + guru.setVmQuotaPercentage(vmTO, vmProfile); + } + + @Test + public void testSetVmQuotaPercentageZeroDivision() { + Mockito.when(host.getSpeed()).thenReturn(0l); + guru.setVmQuotaPercentage(vmTO, vmProfile); + Mockito.verify(vmTO, Mockito.never()).setCpuQuotaPercentage(Mockito.anyDouble()); + } + + @Test + public void testSetVmQuotaPercentageNotCPULimit() { + Mockito.when(vmTO.getLimitCpuUse()).thenReturn(false); + guru.setVmQuotaPercentage(vmTO, vmProfile); + Mockito.verify(vmProfile, Mockito.never()).getVirtualMachine(); + Mockito.verify(vmTO, Mockito.never()).setCpuQuotaPercentage(Mockito.anyDouble()); + } + + @Test + public void testSetVmQuotaPercentageOverProvision() { + Mockito.when(vmTO.getMaxSpeed()).thenReturn(3000); + guru.setVmQuotaPercentage(vmTO, vmProfile); + Mockito.verify(vmTO).setCpuQuotaPercentage(1d); + } +} \ No newline at end of file diff --git a/test/integration/smoke/test_service_offerings.py b/test/integration/smoke/test_service_offerings.py index 50c69d7147b..2788d0e8355 100644 --- a/test/integration/smoke/test_service_offerings.py +++ b/test/integration/smoke/test_service_offerings.py @@ -31,9 +31,13 @@ from marvin.lib.common import (list_service_offering, list_virtual_machines, get_domain, get_zone, - get_test_template) + get_template, + list_hosts) from nose.plugins.attrib import attr +import time +from marvin.sshClient import SshClient +from marvin.lib.decoratorGenerators import skipTestIf _multiprocess_shared_ = True @@ -163,13 +167,13 @@ class TestServiceOfferings(cloudstackTestCase): cls.apiclient, cls.services["service_offerings"]["tiny"] ) - template = get_test_template( + template = get_template( cls.apiclient, cls.zone.id, cls.hypervisor ) if template == FAILED: - assert False, "get_test_template() failed to return template" + assert False, "get_template() failed to return template" # Set Zones and disk offerings cls.services["small"]["zoneid"] = cls.zone.id @@ -400,3 +404,162 @@ class TestServiceOfferings(cloudstackTestCase): "Check Memory(kb) for small offering" ) return + +class TestCpuCapServiceOfferings(cloudstackTestCase): + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + def tearDown(self): + try: + # Clean up, terminate the created templates + cleanup_resources(self.apiclient, self.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + return + + def get_ssh_client(self, id, public_ip, username, password, retries): + """ Setup ssh client connection and return connection + vm requires attributes public_ip, public_port, username, password """ + + try: + ssh_client = SshClient( + public_ip, + 22, + username, + password, + retries) + + except Exception as e: + self.fail("Unable to create ssh connection: " % e) + + self.assertIsNotNone( + ssh_client, "Failed to setup ssh connection to host=%s on public_ip=%s" % (id, public_ip)) + + return ssh_client + + @classmethod + def setUpClass(cls): + testClient = super(TestCpuCapServiceOfferings, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + + cls.hypervisorNotSupported = False + if cls.hypervisor.lower() not in ["kvm"]: + cls.hypervisorNotSupported = True + return + + domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + template = get_template(cls.apiclient, cls.zone.id, cls.hypervisor) + if template == FAILED: + assert False, "get_template() failed to return template" + + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["small"]["template"] = template.id + cls.services["small"]["hypervisor"] = cls.hypervisor + cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__ + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) + + offering_data = { + 'displaytext': 'TestOffering', + 'cpuspeed': 512, + 'cpunumber': 2, + 'name': 'TestOffering', + 'memory': 1024 + } + + cls.offering = ServiceOffering.create( + cls.apiclient, + offering_data, + limitcpuuse=True + ) + + def getHost(self, hostId=None): + response = list_hosts( + self.apiclient, + type='Routing', + hypervisor='kvm', + id=hostId + ) + # Check if more than one kvm hosts are available in order to successfully configure host-ha + if response and len(response) > 0: + self.host = response[0] + return self.host + raise self.skipTest("Not enough KVM hosts found, skipping host-ha test") + + cls.host = getHost(cls) + + cls.vm = VirtualMachine.create( + cls.apiclient, + cls.services["small"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.offering.id, + mode=cls.services["mode"], + hostid=cls.host.id + + ) + cls._cleanup = [ + cls.offering, + cls.account + ] + return + + @classmethod + def tearDownClass(cls): + try: + cls.apiclient = super( + TestCpuCapServiceOfferings, + cls).getClsTestClient().getApiClient() + # Clean up, terminate the created templates + cleanup_resources(cls.apiclient, cls._cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @skipTestIf("hypervisorNotSupported") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") + def test_01_service_offering_cpu_limit_use(self): + """ + Test CPU Cap on KVM + """ + + ssh_host = self.get_ssh_client(self.host.id, self.host.ipaddress, self.hostConfig["username"], self.hostConfig["password"], 10) + + #Get host CPU usage from top command before and after VM consuming 100% CPU + find_pid_cmd = "ps -ax | grep '%s' | head -1 | awk '{print $1}'" % self.vm.id + pid = ssh_host.execute(find_pid_cmd)[0] + cpu_usage_cmd = "top -b n 1 p %s | tail -1 | awk '{print $9}'" % pid + host_cpu_usage_before_str = ssh_host.execute(cpu_usage_cmd)[0] + + host_cpu_usage_before = round(float(host_cpu_usage_before_str)) + self.debug("Host CPU usage before the infinite loop on the VM: " + str(host_cpu_usage_before)) + + #Execute loop command in background on the VM + ssh_vm = self.vm.get_ssh_client(reconnect=True) + ssh_vm.execute("echo 'while true; do x=$(($x+1)); done' > cputest.sh") + ssh_vm.execute("sh cputest.sh > /dev/null 2>&1 &") + + time.sleep(5) + host_cpu_usage_after_str = ssh_host.execute(cpu_usage_cmd)[0] + host_cpu_usage_after = round(float(host_cpu_usage_after_str)) + self.debug("Host CPU usage after the infinite loop on the VM: " + str(host_cpu_usage_after)) + + limit = 95 + self.assertTrue(host_cpu_usage_after < limit, "Host CPU usage after VM usage increased is high") + + return