KVM Agent config to reserve dom0 CPUs (#326)

Cherry-picked from 1a722edfce6234319c5eaecf3cd0076fa9690267

Co-authored-by: Marcus Sorensen <mls@apple.com>

agent/conf/agent.properties

@@ -279,6 +279,11 @@ hypervisor.type=kvm
 # If this parameter is used, property host.overcommit.mem.mb must be set to 0.
 #host.reserved.mem.mb=1024
 
+# Number of CPU cores to subtract from advertised available cores.
+# These are reserved for system activity, or otherwise share host CPU resources with
+# CloudStack VM allocation.
+#host.reserved.cpu.count = 0
+
 # The model of Watchdog timer to present to the Guest.
 # For all models refer to the libvirt documentation.
 #vm.watchdog.model=i6300esb

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -452,6 +452,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     private long _dom0OvercommitMem;
 
+    private int _dom0MinCpuCores;
+
     protected int _cmdsTimeout;
     protected int _stopTimeout;
     protected CPUStat _cpuStat = new CPUStat();
@@ -1049,6 +1051,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         // Reserve 1GB unless admin overrides
         _dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB));
 
+        value = (String)params.get("host.reserved.cpu.count");
+        _dom0MinCpuCores = NumbersUtil.parseInt(value, 0);
+
         // Support overcommit memory for host if host uses ZSWAP, KSM and other memory
         // compressing technologies
         _dom0OvercommitMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_OVERCOMMIT_MEM_MB));
@@ -3522,7 +3527,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
     @Override
     public StartupCommand[] initialize() {
 
-        final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem, _manualCpuSpeed);
+        final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem, _manualCpuSpeed, _dom0MinCpuCores);
 
         String capabilities = String.join(",", info.getCapabilities());
         if (dpdkSupport) {
@@ -3530,7 +3535,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         }
 
         final StartupRoutingCommand cmd =
-                new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType,
+                new StartupRoutingCommand(info.getAllocatableCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType,
                         RouterPrivateIpStrategy.HostLocal);
         cmd.setCpuSockets(info.getCpuSockets());
         fillNetworkInformation(cmd);

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java

@@ -48,7 +48,8 @@ public class KVMHostInfo {
 
     private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class);
 
-    private int cpus;
+    private int totalCpus;
+    private int allocatableCpus;
     private int cpusockets;
     private long cpuSpeed;
     private long totalMemory;
@@ -58,16 +59,25 @@ public class KVMHostInfo {
 
     private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency";
 
-    public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed) {
+    public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) {
         this.cpuSpeed = manualSpeed;
         this.reservedMemory = reservedMemory;
         this.overCommitMemory = overCommitMemory;
         this.getHostInfoFromLibvirt();
         this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal();
+        this.allocatableCpus = totalCpus - reservedCpus;
+        if (allocatableCpus < 1) {
+            LOGGER.warn(String.format("Aggressive reserved CPU config leaves no usable CPUs for VMs! Total system CPUs: %d, Reserved: %d, Allocatable: %d", totalCpus, reservedCpus, allocatableCpus));
+            allocatableCpus = 0;
+        }
     }
 
-    public int getCpus() {
-        return this.cpus;
+    public int getTotalCpus() {
+        return this.totalCpus;
+    }
+
+    public int getAllocatableCpus() {
+        return this.allocatableCpus;
     }
 
     public int getCpuSockets() {
@@ -189,7 +199,7 @@ public class KVMHostInfo {
             if (hosts.nodes > 0) {
                 this.cpusockets = hosts.sockets * hosts.nodes;
             }
-            this.cpus = hosts.cpus;
+            this.totalCpus = hosts.cpus;
 
             final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
             parser.parseCapabilitiesXML(capabilities);

plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java

@@ -78,7 +78,30 @@ public class KVMHostInfoTest {
         PowerMockito.when(conn.close()).thenReturn(0);
         int manualSpeed = 500;
 
-        KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed);
+        KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed, 0);
         Assert.assertEquals(kvmHostInfo.getCpuSpeed(), manualSpeed);
     }
+
+    @Test
+    public void reservedCpuCoresTest() throws Exception {
+        PowerMockito.mockStatic(LibvirtConnection.class);
+        Connect conn = Mockito.mock(Connect.class);
+        NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
+        nodeInfo.cpus = 10;
+        String capabilitiesXml = "<capabilities></capabilities>";
+
+        PowerMockito.doReturn(conn).when(LibvirtConnection.class, "getConnection");
+        PowerMockito.when(conn.nodeInfo()).thenReturn(nodeInfo);
+        PowerMockito.when(conn.getCapabilities()).thenReturn(capabilitiesXml);
+        PowerMockito.when(conn.close()).thenReturn(0);
+
+        KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, 100, 2);
+        Assert.assertEquals("reserve two CPU cores", 8, kvmHostInfo.getAllocatableCpus());
+
+        kvmHostInfo = new KVMHostInfo(10, 10, 100, 0);
+        Assert.assertEquals("no reserve CPU core setting", 10, kvmHostInfo.getAllocatableCpus());
+
+        kvmHostInfo = new KVMHostInfo(10, 10, 100, 12);
+        Assert.assertEquals("Misconfigured/too large CPU reserve", 0, kvmHostInfo.getAllocatableCpus());
+    }
 }