CLOUDSTACK-10321: CPU Cap for KVM (#2482)

api/src/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -70,6 +70,8 @@ public class VirtualMachineTO {
     String configDriveIsoRootFolder = null;
     String configDriveIsoFile = null;
 
+    Double cpuQuotaPercentage = null;
+
     Map<String, String> guestOsDetails = new HashMap<String, String>();
 
     public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader,
@@ -340,4 +342,12 @@ public class VirtualMachineTO {
     public void setGuestOsDetails(Map<String, String> guestOsDetails) {
         this.guestOsDetails = guestOsDetails;
     }
+
+    public Double getCpuQuotaPercentage() {
+        return cpuQuotaPercentage;
+    }
+
+    public void setCpuQuotaPercentage(Double cpuQuotaPercentage) {
+        this.cpuQuotaPercentage = cpuQuotaPercentage;
+    }
 }

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -1984,6 +1984,31 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         return uuid;
     }
 
+    /**
+     * Set quota and period tags on 'ctd' when CPU limit use is set
+     */
+    protected void setQuotaAndPeriod(VirtualMachineTO vmTO, CpuTuneDef ctd) {
+        if (vmTO.getLimitCpuUse() && vmTO.getCpuQuotaPercentage() != null) {
+            Double cpuQuotaPercentage = vmTO.getCpuQuotaPercentage();
+            int period = CpuTuneDef.DEFAULT_PERIOD;
+            int quota = (int) (period * cpuQuotaPercentage);
+            if (quota < CpuTuneDef.MIN_QUOTA) {
+                s_logger.info("Calculated quota (" + quota + ") below the minimum (" + CpuTuneDef.MIN_QUOTA + ") for VM domain " + vmTO.getUuid() + ", setting it to minimum " +
+                        "and calculating period instead of using the default");
+                quota = CpuTuneDef.MIN_QUOTA;
+                period = (int) ((double) quota / cpuQuotaPercentage);
+                if (period > CpuTuneDef.MAX_PERIOD) {
+                    s_logger.info("Calculated period (" + period + ") exceeds the maximum (" + CpuTuneDef.MAX_PERIOD +
+                            "), setting it to the maximum");
+                    period = CpuTuneDef.MAX_PERIOD;
+                }
+            }
+            ctd.setQuota(quota);
+            ctd.setPeriod(period);
+            s_logger.info("Setting quota=" + quota + ", period=" + period + " to VM domain " + vmTO.getUuid());
+        }
+    }
+
     public LibvirtVMDef createVMFromSpec(final VirtualMachineTO vmTO) {
         final LibvirtVMDef vm = new LibvirtVMDef();
         vm.setDomainName(vmTO.getName());
@@ -2059,6 +2084,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
             } else {
                 ctd.setShares(vmTO.getCpus() * vmTO.getSpeed());
             }
+
+            setQuotaAndPeriod(vmTO, ctd);
+
             vm.addComp(ctd);
         }
 

plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -1171,6 +1171,11 @@ public class LibvirtVMDef {
 
     public static class CpuTuneDef {
         private int _shares = 0;
+        private int quota = 0;
+        private int period = 0;
+        static final int DEFAULT_PERIOD = 10000;
+        static final int MIN_QUOTA = 1000;
+        static final int MAX_PERIOD = 1000000;
 
         public void setShares(int shares) {
             _shares = shares;
@@ -1180,6 +1185,22 @@ public class LibvirtVMDef {
             return _shares;
         }
 
+        public int getQuota() {
+            return quota;
+        }
+
+        public void setQuota(int quota) {
+            this.quota = quota;
+        }
+
+        public int getPeriod() {
+            return period;
+        }
+
+        public void setPeriod(int period) {
+            this.period = period;
+        }
+
         @Override
         public String toString() {
             StringBuilder cpuTuneBuilder = new StringBuilder();
@@ -1187,6 +1208,12 @@ public class LibvirtVMDef {
             if (_shares > 0) {
                 cpuTuneBuilder.append("<shares>" + _shares + "</shares>\n");
             }
+            if (quota > 0) {
+                cpuTuneBuilder.append("<quota>" + quota + "</quota>\n");
+            }
+            if (period > 0) {
+                cpuTuneBuilder.append("<period>" + period + "</period>\n");
+            }
             cpuTuneBuilder.append("</cputune>\n");
             return cpuTuneBuilder.toString();
         }

plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java

@@ -38,6 +38,7 @@ import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathExpressionException;
 import javax.xml.xpath.XPathFactory;
 
+import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.CpuTuneDef;
 import org.apache.commons.lang.SystemUtils;
 import org.joda.time.Duration;
 import org.junit.Assert;
@@ -184,6 +185,8 @@ public class LibvirtComputingResourceTest {
 
     @Mock
     private LibvirtComputingResource libvirtComputingResource;
+    @Mock
+    VirtualMachineTO vmTO;
 
     String hyperVisorType = "kvm";
     Random random = new Random();
@@ -5152,4 +5155,40 @@ public class LibvirtComputingResourceTest {
         when(domainMock.memoryStats(2)).thenReturn(mem);
         return domainMock;
     }
+
+    @Test
+    public void testSetQuotaAndPeriod() {
+        double pct = 0.33d;
+        Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true);
+        Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct);
+        CpuTuneDef cpuTuneDef = new CpuTuneDef();
+        final LibvirtComputingResource lcr = new LibvirtComputingResource();
+        lcr.setQuotaAndPeriod(vmTO, cpuTuneDef);
+        Assert.assertEquals((int) (CpuTuneDef.DEFAULT_PERIOD * pct), cpuTuneDef.getQuota());
+        Assert.assertEquals(CpuTuneDef.DEFAULT_PERIOD, cpuTuneDef.getPeriod());
+    }
+
+    @Test
+    public void testSetQuotaAndPeriodNoCpuLimitUse() {
+        double pct = 0.33d;
+        Mockito.when(vmTO.getLimitCpuUse()).thenReturn(false);
+        Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct);
+        CpuTuneDef cpuTuneDef = new CpuTuneDef();
+        final LibvirtComputingResource lcr = new LibvirtComputingResource();
+        lcr.setQuotaAndPeriod(vmTO, cpuTuneDef);
+        Assert.assertEquals(0, cpuTuneDef.getQuota());
+        Assert.assertEquals(0, cpuTuneDef.getPeriod());
+    }
+
+    @Test
+    public void testSetQuotaAndPeriodMinQuota() {
+        double pct = 0.01d;
+        Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true);
+        Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(pct);
+        CpuTuneDef cpuTuneDef = new CpuTuneDef();
+        final LibvirtComputingResource lcr = new LibvirtComputingResource();
+        lcr.setQuotaAndPeriod(vmTO, cpuTuneDef);
+        Assert.assertEquals(CpuTuneDef.MIN_QUOTA, cpuTuneDef.getQuota());
+        Assert.assertEquals((int) (CpuTuneDef.MIN_QUOTA / pct), cpuTuneDef.getPeriod());
+    }
 }

server/src/com/cloud/hypervisor/KVMGuru.java

@@ -28,11 +28,16 @@ import com.cloud.storage.GuestOSVO;
 import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.storage.dao.GuestOSHypervisorDao;
 import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
 import org.apache.cloudstack.storage.command.CopyCommand;
 import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
+import org.apache.log4j.Logger;
 
 import javax.inject.Inject;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.util.Map;
 
 public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
@@ -43,6 +48,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
     @Inject
     HostDao _hostDao;
 
+    public static final Logger s_logger = Logger.getLogger(KVMGuru.class);
+
     @Override
     public HypervisorType getHypervisorType() {
         return HypervisorType.KVM;
@@ -52,10 +59,53 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
         super();
     }
 
+    /**
+     * Retrieve host max CPU speed
+     */
+    protected double getHostCPUSpeed(HostVO host) {
+        return host.getSpeed();
+    }
+
+    protected double getVmSpeed(VirtualMachineTO to) {
+        return to.getMaxSpeed() != null ? to.getMaxSpeed() : to.getSpeed();
+    }
+
+    /**
+    * Set VM CPU quota percentage with respect to host CPU on 'to' if CPU limit option is set
+    * @param to vm to
+    * @param vmProfile vm profile
+    */
+    protected void setVmQuotaPercentage(VirtualMachineTO to, VirtualMachineProfile vmProfile) {
+        if (to.getLimitCpuUse()) {
+            VirtualMachine vm = vmProfile.getVirtualMachine();
+            HostVO host = _hostDao.findById(vm.getHostId());
+            if (host == null) {
+                throw new CloudRuntimeException("Host with id: " + vm.getHostId() + " not found");
+            }
+            s_logger.debug("Limiting CPU usage for VM: " + vm.getUuid() + " on host: " + host.getUuid());
+            double hostMaxSpeed = getHostCPUSpeed(host);
+            double maxSpeed = getVmSpeed(to);
+            try {
+                BigDecimal percent = new BigDecimal(maxSpeed / hostMaxSpeed);
+                percent = percent.setScale(2, RoundingMode.HALF_DOWN);
+                if (percent.compareTo(new BigDecimal(1)) == 1) {
+                    s_logger.debug("VM " + vm.getUuid() + " CPU MHz exceeded host " + host.getUuid() + " CPU MHz, limiting VM CPU to the host maximum");
+                    percent = new BigDecimal(1);
+                }
+                to.setCpuQuotaPercentage(percent.doubleValue());
+                s_logger.debug("Host: " + host.getUuid() + " max CPU speed = " + hostMaxSpeed + "MHz, VM: " + vm.getUuid() +
+                        "max CPU speed = " + maxSpeed + "MHz. Setting CPU quota percentage as: " + percent.doubleValue());
+            } catch (NumberFormatException e) {
+                s_logger.error("Error calculating VM: " + vm.getUuid() + " quota percentage, it wll not be set. Error: " + e.getMessage(), e);
+            }
+        }
+    }
+
     @Override
 
     public VirtualMachineTO implement(VirtualMachineProfile vm) {
         VirtualMachineTO to = toVirtualMachineTO(vm);
+        setVmQuotaPercentage(to, vm);
 
         // Determine the VM's OS description
         GuestOSVO guestOS = _guestOsDao.findByIdIncludingRemoved(vm.getVirtualMachine().getGuestOSId());

server/test/com/cloud/hypervisor/KVMGuruTest.java

@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.hypervisor;
+
+import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class KVMGuruTest {
+
+    @Mock
+    HostDao hostDao;
+
+    @Spy
+    @InjectMocks
+    private KVMGuru guru = new KVMGuru();
+
+    @Mock
+    VirtualMachineTO vmTO;
+    @Mock
+    VirtualMachineProfile vmProfile;
+    @Mock
+    VirtualMachine vm;
+    @Mock
+    HostVO host;
+
+    private static final long hostId = 1l;
+
+    @Before
+    public void setup() {
+        Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true);
+        Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm);
+        Mockito.when(vm.getHostId()).thenReturn(hostId);
+        Mockito.when(hostDao.findById(hostId)).thenReturn(host);
+        Mockito.when(host.getCpus()).thenReturn(3);
+        Mockito.when(host.getSpeed()).thenReturn(1995l);
+        Mockito.when(vmTO.getMaxSpeed()).thenReturn(500);
+    }
+
+    @Test
+    public void testSetVmQuotaPercentage() {
+        guru.setVmQuotaPercentage(vmTO, vmProfile);
+        Mockito.verify(vmTO).setCpuQuotaPercentage(Mockito.anyDouble());
+    }
+
+    @Test(expected = CloudRuntimeException.class)
+    public void testSetVmQuotaPercentageNullHost() {
+        Mockito.when(hostDao.findById(hostId)).thenReturn(null);
+        guru.setVmQuotaPercentage(vmTO, vmProfile);
+    }
+
+    @Test
+    public void testSetVmQuotaPercentageZeroDivision() {
+        Mockito.when(host.getSpeed()).thenReturn(0l);
+        guru.setVmQuotaPercentage(vmTO, vmProfile);
+        Mockito.verify(vmTO, Mockito.never()).setCpuQuotaPercentage(Mockito.anyDouble());
+    }
+
+    @Test
+    public void testSetVmQuotaPercentageNotCPULimit() {
+        Mockito.when(vmTO.getLimitCpuUse()).thenReturn(false);
+        guru.setVmQuotaPercentage(vmTO, vmProfile);
+        Mockito.verify(vmProfile, Mockito.never()).getVirtualMachine();
+        Mockito.verify(vmTO, Mockito.never()).setCpuQuotaPercentage(Mockito.anyDouble());
+    }
+
+    @Test
+    public void testSetVmQuotaPercentageOverProvision() {
+        Mockito.when(vmTO.getMaxSpeed()).thenReturn(3000);
+        guru.setVmQuotaPercentage(vmTO, vmProfile);
+        Mockito.verify(vmTO).setCpuQuotaPercentage(1d);
+    }
+}

test/integration/smoke/test_service_offerings.py

@@ -31,9 +31,13 @@ from marvin.lib.common import (list_service_offering,
                                list_virtual_machines,
                                get_domain,
                                get_zone,
-                               get_test_template)
+                               get_template,
+                               list_hosts)
 from nose.plugins.attrib import attr
 
+import time
+from marvin.sshClient import SshClient
+from marvin.lib.decoratorGenerators import skipTestIf
 
 _multiprocess_shared_ = True
 
@@ -163,13 +167,13 @@ class TestServiceOfferings(cloudstackTestCase):
             cls.apiclient,
             cls.services["service_offerings"]["tiny"]
         )
-        template = get_test_template(
+        template = get_template(
             cls.apiclient,
             cls.zone.id,
             cls.hypervisor
         )
         if template == FAILED:
-            assert False, "get_test_template() failed to return template"
+            assert False, "get_template() failed to return template"
 
         # Set Zones and disk offerings
         cls.services["small"]["zoneid"] = cls.zone.id
@@ -400,3 +404,162 @@ class TestServiceOfferings(cloudstackTestCase):
             "Check Memory(kb) for small offering"
         )
         return
+
+class TestCpuCapServiceOfferings(cloudstackTestCase):
+
+    def setUp(self):
+        self.apiclient = self.testClient.getApiClient()
+        self.dbclient = self.testClient.getDbConnection()
+        self.cleanup = []
+
+    def tearDown(self):
+        try:
+            # Clean up, terminate the created templates
+            cleanup_resources(self.apiclient, self.cleanup)
+
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+
+        return
+
+    def get_ssh_client(self, id, public_ip, username, password, retries):
+        """ Setup ssh client connection and return connection
+        vm requires attributes public_ip, public_port, username, password """
+
+        try:
+            ssh_client = SshClient(
+                public_ip,
+                22,
+                username,
+                password,
+                retries)
+
+        except Exception as e:
+            self.fail("Unable to create ssh connection: " % e)
+
+        self.assertIsNotNone(
+            ssh_client, "Failed to setup ssh connection to host=%s on public_ip=%s" % (id, public_ip))
+
+        return ssh_client
+
+    @classmethod
+    def setUpClass(cls):
+        testClient = super(TestCpuCapServiceOfferings, cls).getClsTestClient()
+        cls.apiclient = testClient.getApiClient()
+        cls.services = testClient.getParsedTestDataConfig()
+        cls.hypervisor = testClient.getHypervisorInfo()
+
+        cls.hypervisorNotSupported = False
+        if cls.hypervisor.lower() not in ["kvm"]:
+            cls.hypervisorNotSupported = True
+            return
+
+        domain = get_domain(cls.apiclient)
+        cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
+        cls.services['mode'] = cls.zone.networktype
+
+        template = get_template(cls.apiclient, cls.zone.id, cls.hypervisor)
+        if template == FAILED:
+            assert False, "get_template() failed to return template"
+
+        cls.services["small"]["zoneid"] = cls.zone.id
+        cls.services["small"]["template"] = template.id
+        cls.services["small"]["hypervisor"] = cls.hypervisor
+        cls.hostConfig = cls.config.__dict__["zones"][0].__dict__["pods"][0].__dict__["clusters"][0].__dict__["hosts"][0].__dict__
+
+        cls.account = Account.create(
+            cls.apiclient,
+            cls.services["account"],
+            domainid=domain.id
+        )
+
+        offering_data = {
+            'displaytext': 'TestOffering',
+            'cpuspeed': 512,
+            'cpunumber': 2,
+            'name': 'TestOffering',
+            'memory': 1024
+        }
+
+        cls.offering = ServiceOffering.create(
+            cls.apiclient,
+            offering_data,
+            limitcpuuse=True
+        )
+
+        def getHost(self, hostId=None):
+            response = list_hosts(
+                self.apiclient,
+                type='Routing',
+                hypervisor='kvm',
+                id=hostId
+            )
+            # Check if more than one kvm hosts are available in order to successfully configure host-ha
+            if response and len(response) > 0:
+                self.host = response[0]
+                return self.host
+            raise self.skipTest("Not enough KVM hosts found, skipping host-ha test")
+
+        cls.host = getHost(cls)
+
+        cls.vm = VirtualMachine.create(
+            cls.apiclient,
+            cls.services["small"],
+            accountid=cls.account.name,
+            domainid=cls.account.domainid,
+            serviceofferingid=cls.offering.id,
+            mode=cls.services["mode"],
+            hostid=cls.host.id
+
+        )
+        cls._cleanup = [
+            cls.offering,
+            cls.account
+        ]
+        return
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            cls.apiclient = super(
+                TestCpuCapServiceOfferings,
+                cls).getClsTestClient().getApiClient()
+            # Clean up, terminate the created templates
+            cleanup_resources(cls.apiclient, cls._cleanup)
+
+        except Exception as e:
+            raise Exception("Warning: Exception during cleanup : %s" % e)
+        return
+
+    @skipTestIf("hypervisorNotSupported")
+    @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true")
+    def test_01_service_offering_cpu_limit_use(self):
+        """
+        Test CPU Cap on KVM
+        """
+
+        ssh_host = self.get_ssh_client(self.host.id, self.host.ipaddress, self.hostConfig["username"], self.hostConfig["password"], 10)
+
+        #Get host CPU usage from top command before and after VM consuming 100% CPU
+        find_pid_cmd = "ps -ax | grep '%s' | head -1 | awk '{print $1}'" % self.vm.id
+        pid = ssh_host.execute(find_pid_cmd)[0]
+        cpu_usage_cmd = "top -b n 1 p %s | tail -1 | awk '{print $9}'" % pid
+        host_cpu_usage_before_str = ssh_host.execute(cpu_usage_cmd)[0]
+
+        host_cpu_usage_before = round(float(host_cpu_usage_before_str))
+        self.debug("Host CPU usage before the infinite loop on the VM: " + str(host_cpu_usage_before))
+
+        #Execute loop command in background on the VM
+        ssh_vm = self.vm.get_ssh_client(reconnect=True)
+        ssh_vm.execute("echo 'while true; do x=$(($x+1)); done' > cputest.sh")
+        ssh_vm.execute("sh cputest.sh > /dev/null 2>&1 &")
+
+        time.sleep(5)
+        host_cpu_usage_after_str = ssh_host.execute(cpu_usage_cmd)[0]
+        host_cpu_usage_after = round(float(host_cpu_usage_after_str))
+        self.debug("Host CPU usage after the infinite loop on the VM: " + str(host_cpu_usage_after))
+
+        limit = 95
+        self.assertTrue(host_cpu_usage_after < limit, "Host CPU usage after VM usage increased is high")
+
+        return