From 82b981854b88e5bc8d164a4bf7fe92971cd679f0 Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Thu, 5 Oct 2023 23:20:18 -0600 Subject: [PATCH] KVM Agent config to reserve dom0 CPUs (#7987) This PR allows an admin to reserve some hypervisor host CPUs for system use. Another way to think of it is limiting the number of CPUs allocatable to VMs. This can be useful if the admin wants to do other things with the hypervisor's CPU, for example reserve some cores for running hyperconverged storage processes. Co-authored-by: Marcus Sorensen --- agent/conf/agent.properties | 5 ++++ .../agent/properties/AgentProperties.java | 9 ++++++ .../resource/LibvirtComputingResource.java | 8 +++-- .../cloudstack/utils/linux/KVMHostInfo.java | 20 +++++++++---- .../utils/linux/KVMHostInfoTest.java | 30 ++++++++++++++++++- 5 files changed, 64 insertions(+), 8 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 9174da7fd7b..b88da3621cd 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -279,6 +279,11 @@ hypervisor.type=kvm # If this parameter is used, property host.overcommit.mem.mb must be set to 0. #host.reserved.mem.mb=1024 +# Number of CPU cores to subtract from advertised available cores. +# These are reserved for system activity, or otherwise share host CPU resources with +# CloudStack VM allocation. +# host.reserved.cpu.count = 0 + # The model of Watchdog timer to present to the Guest. # For all models refer to the libvirt documentation. #vm.watchdog.model=i6300esb diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 75248fb01bf..610c5be759f 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -502,6 +502,15 @@ public class AgentProperties{ */ public static final Property HOST_RESERVED_MEM_MB = new Property<>("host.reserved.mem.mb", 1024); + /** + * How many host CPUs to reserve for non-allocation.
+ * This can be used to set aside CPU cores on the host for other tasks, such as running hyperconverged storage
+ * processes, etc. + * Data type: Integer.
+ * Default value: 0 + */ + public static final Property HOST_RESERVED_CPU_CORE_COUNT = new Property<>("host.reserved.cpu.count", 0); + /** * The model of Watchdog timer to present to the Guest.
* For all models refer to the libvirt documentation.
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 16cccb0150b..6bbafcacef2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -460,6 +460,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private long dom0OvercommitMem; + private int dom0MinCpuCores; + protected int cmdsTimeout; protected int stopTimeout; protected CPUStat cpuStat = new CPUStat(); @@ -1063,6 +1065,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv // Reserve 1GB unless admin overrides dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB)); + dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT); + // Support overcommit memory for host if host uses ZSWAP, KSM and other memory // compressing technologies dom0OvercommitMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_OVERCOMMIT_MEM_MB)); @@ -3540,7 +3544,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv @Override public StartupCommand[] initialize() { - final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed); + final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed, dom0MinCpuCores); String capabilities = String.join(",", info.getCapabilities()); if (dpdkSupport) { @@ -3548,7 +3552,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } final StartupRoutingCommand cmd = - new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType, + new StartupRoutingCommand(info.getAllocatableCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType, RouterPrivateIpStrategy.HostLocal); cmd.setCpuSockets(info.getCpuSockets()); fillNetworkInformation(cmd); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java index 807b2541fd3..d160cbfac3b 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java @@ -48,7 +48,8 @@ public class KVMHostInfo { private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class); - private int cpus; + private int totalCpus; + private int allocatableCpus; private int cpusockets; private long cpuSpeed; private long totalMemory; @@ -58,16 +59,25 @@ public class KVMHostInfo { private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency"; - public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed) { + public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) { this.cpuSpeed = manualSpeed; this.reservedMemory = reservedMemory; this.overCommitMemory = overCommitMemory; this.getHostInfoFromLibvirt(); this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal(); + this.allocatableCpus = totalCpus - reservedCpus; + if (allocatableCpus < 1) { + LOGGER.warn(String.format("Aggressive reserved CPU config leaves no usable CPUs for VMs! Total system CPUs: %d, Reserved: %d, Allocatable: %d", totalCpus, reservedCpus, allocatableCpus)); + allocatableCpus = 0; + } } - public int getCpus() { - return this.cpus; + public int getTotalCpus() { + return this.totalCpus; + } + + public int getAllocatableCpus() { + return this.allocatableCpus; } public int getCpuSockets() { @@ -189,7 +199,7 @@ public class KVMHostInfo { if (hosts.nodes > 0) { this.cpusockets = hosts.sockets * hosts.nodes; } - this.cpus = hosts.cpus; + this.totalCpus = hosts.cpus; final LibvirtCapXMLParser parser = new LibvirtCapXMLParser(); parser.parseCapabilitiesXML(capabilities); diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java index 92cece6ae06..3b5422f564c 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java @@ -73,8 +73,36 @@ public class KVMHostInfoTest { Mockito.when(conn.close()).thenReturn(0); int manualSpeed = 500; - KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed); + KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed, 0); Assert.assertEquals(kvmHostInfo.getCpuSpeed(), manualSpeed); } } + + @Test + public void reservedCpuCoresTest() throws Exception { + if (!System.getProperty("os.name").equals("Linux")) { + return; + } + try (MockedStatic ignored = Mockito.mockStatic(LibvirtConnection.class)) { + Connect conn = Mockito.mock(Connect.class); + NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); + nodeInfo.cpus = 10; + String capabilitiesXml = ""; + + Mockito.when(LibvirtConnection.getConnection()).thenReturn(conn); + Mockito.when(conn.nodeInfo()).thenReturn(nodeInfo); + Mockito.when(conn.getCapabilities()).thenReturn(capabilitiesXml); + Mockito.when(conn.close()).thenReturn(0); + int manualSpeed = 500; + + KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, 100, 2); + Assert.assertEquals("reserve two CPU cores", 8, kvmHostInfo.getAllocatableCpus()); + + kvmHostInfo = new KVMHostInfo(10, 10, 100, 0); + Assert.assertEquals("no reserve CPU core setting", 10, kvmHostInfo.getAllocatableCpus()); + + kvmHostInfo = new KVMHostInfo(10, 10, 100, 12); + Assert.assertEquals("Misconfigured/too large CPU reserve", 0, kvmHostInfo.getAllocatableCpus()); + } + } }