From 33a5c159a35f99469a57509f08ec7acd78b814eb Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Wed, 27 Sep 2023 17:15:40 +0530 Subject: [PATCH] KVM Agent config to reserve dom0 CPUs (#326) Cherry-picked from 1a722edfce6234319c5eaecf3cd0076fa9690267 Co-authored-by: Marcus Sorensen --- agent/conf/agent.properties | 5 ++++ .../resource/LibvirtComputingResource.java | 9 +++++-- .../cloudstack/utils/linux/KVMHostInfo.java | 20 +++++++++++---- .../utils/linux/KVMHostInfoTest.java | 25 ++++++++++++++++++- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 9174da7fd7b..648a34f10b0 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -279,6 +279,11 @@ hypervisor.type=kvm # If this parameter is used, property host.overcommit.mem.mb must be set to 0. #host.reserved.mem.mb=1024 +# Number of CPU cores to subtract from advertised available cores. +# These are reserved for system activity, or otherwise share host CPU resources with +# CloudStack VM allocation. +#host.reserved.cpu.count = 0 + # The model of Watchdog timer to present to the Guest. # For all models refer to the libvirt documentation. #vm.watchdog.model=i6300esb diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 002dca7f0a8..481573f1a09 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -452,6 +452,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private long _dom0OvercommitMem; + private int _dom0MinCpuCores; + protected int _cmdsTimeout; protected int _stopTimeout; protected CPUStat _cpuStat = new CPUStat(); @@ -1049,6 +1051,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv // Reserve 1GB unless admin overrides _dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB)); + value = (String)params.get("host.reserved.cpu.count"); + _dom0MinCpuCores = NumbersUtil.parseInt(value, 0); + // Support overcommit memory for host if host uses ZSWAP, KSM and other memory // compressing technologies _dom0OvercommitMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_OVERCOMMIT_MEM_MB)); @@ -3522,7 +3527,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv @Override public StartupCommand[] initialize() { - final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem, _manualCpuSpeed); + final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem, _manualCpuSpeed, _dom0MinCpuCores); String capabilities = String.join(",", info.getCapabilities()); if (dpdkSupport) { @@ -3530,7 +3535,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } final StartupRoutingCommand cmd = - new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType, + new StartupRoutingCommand(info.getAllocatableCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, _hypervisorType, RouterPrivateIpStrategy.HostLocal); cmd.setCpuSockets(info.getCpuSockets()); fillNetworkInformation(cmd); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java index 807b2541fd3..d160cbfac3b 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java @@ -48,7 +48,8 @@ public class KVMHostInfo { private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class); - private int cpus; + private int totalCpus; + private int allocatableCpus; private int cpusockets; private long cpuSpeed; private long totalMemory; @@ -58,16 +59,25 @@ public class KVMHostInfo { private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency"; - public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed) { + public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) { this.cpuSpeed = manualSpeed; this.reservedMemory = reservedMemory; this.overCommitMemory = overCommitMemory; this.getHostInfoFromLibvirt(); this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal(); + this.allocatableCpus = totalCpus - reservedCpus; + if (allocatableCpus < 1) { + LOGGER.warn(String.format("Aggressive reserved CPU config leaves no usable CPUs for VMs! Total system CPUs: %d, Reserved: %d, Allocatable: %d", totalCpus, reservedCpus, allocatableCpus)); + allocatableCpus = 0; + } } - public int getCpus() { - return this.cpus; + public int getTotalCpus() { + return this.totalCpus; + } + + public int getAllocatableCpus() { + return this.allocatableCpus; } public int getCpuSockets() { @@ -189,7 +199,7 @@ public class KVMHostInfo { if (hosts.nodes > 0) { this.cpusockets = hosts.sockets * hosts.nodes; } - this.cpus = hosts.cpus; + this.totalCpus = hosts.cpus; final LibvirtCapXMLParser parser = new LibvirtCapXMLParser(); parser.parseCapabilitiesXML(capabilities); diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java index c8d5721f245..23837de00c2 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java @@ -78,7 +78,30 @@ public class KVMHostInfoTest { PowerMockito.when(conn.close()).thenReturn(0); int manualSpeed = 500; - KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed); + KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed, 0); Assert.assertEquals(kvmHostInfo.getCpuSpeed(), manualSpeed); } + + @Test + public void reservedCpuCoresTest() throws Exception { + PowerMockito.mockStatic(LibvirtConnection.class); + Connect conn = Mockito.mock(Connect.class); + NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); + nodeInfo.cpus = 10; + String capabilitiesXml = ""; + + PowerMockito.doReturn(conn).when(LibvirtConnection.class, "getConnection"); + PowerMockito.when(conn.nodeInfo()).thenReturn(nodeInfo); + PowerMockito.when(conn.getCapabilities()).thenReturn(capabilitiesXml); + PowerMockito.when(conn.close()).thenReturn(0); + + KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, 100, 2); + Assert.assertEquals("reserve two CPU cores", 8, kvmHostInfo.getAllocatableCpus()); + + kvmHostInfo = new KVMHostInfo(10, 10, 100, 0); + Assert.assertEquals("no reserve CPU core setting", 10, kvmHostInfo.getAllocatableCpus()); + + kvmHostInfo = new KVMHostInfo(10, 10, 100, 12); + Assert.assertEquals("Misconfigured/too large CPU reserve", 0, kvmHostInfo.getAllocatableCpus()); + } }