diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index 62519412fdb..6770e4f3ba6 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -190,6 +190,10 @@ public interface VirtualMachineProfile { Map getParameters(); + void setCpuOvercommitRatio(Float cpuOvercommitRatio); + + void setMemoryOvercommitRatio(Float memoryOvercommitRatio); + Float getCpuOvercommitRatio(); Float getMemoryOvercommitRatio(); diff --git a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java index 27251f4bb78..3acdb9c351b 100644 --- a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java +++ b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java @@ -40,6 +40,9 @@ public class MigrateCommand extends Command { private boolean executeInSequence = false; private List migrateDiskInfoList = new ArrayList<>(); private Map dpdkInterfaceMapping = new HashMap<>(); + + private int newVmCpuShares; + Map vlanToPersistenceMap = new HashMap<>(); public Map getDpdkInterfaceMapping() { @@ -138,6 +141,14 @@ public class MigrateCommand extends Command { this.migrateDiskInfoList = migrateDiskInfoList; } + public int getNewVmCpuShares() { + return newVmCpuShares; + } + + public void setNewVmCpuShares(int newVmCpuShares) { + this.newVmCpuShares = newVmCpuShares; + } + public static class MigrateDiskInfo { public enum DiskType { FILE, BLOCK; diff --git a/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java b/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java index d0a544ba081..190e844ddc5 100644 --- a/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java @@ -28,6 +28,8 @@ public class PrepareForMigrationAnswer extends Answer { private Map dpdkInterfaceMapping = new HashMap<>(); + private Integer newVmCpuShares = null; + protected PrepareForMigrationAnswer() { } @@ -50,4 +52,12 @@ public class PrepareForMigrationAnswer extends Answer { public Map getDpdkInterfaceMapping() { return this.dpdkInterfaceMapping; } + + public Integer getNewVmCpuShares() { + return newVmCpuShares; + } + + public void setNewVmCpuShares(Integer newVmCpuShares) { + this.newVmCpuShares = newVmCpuShares; + } } diff --git a/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java b/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java index bc689fef873..8297a73d77f 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java +++ b/engine/components-api/src/main/java/com/cloud/vm/VirtualMachineProfileImpl.java @@ -264,11 +264,13 @@ public class VirtualMachineProfileImpl implements VirtualMachineProfile { _offering = offering; } + @Override public void setCpuOvercommitRatio(Float cpuOvercommitRatio) { this.cpuOvercommitRatio = cpuOvercommitRatio; } + @Override public void setMemoryOvercommitRatio(Float memoryOvercommitRatio) { this.memoryOvercommitRatio = memoryOvercommitRatio; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index e51a1d7fa97..c0edf69b944 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -51,6 +51,7 @@ import javax.persistence.EntityExistsException; import com.cloud.configuration.Resource; import com.cloud.event.ActionEventUtils; import com.cloud.exception.ResourceAllocationException; +import com.google.gson.Gson; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -1252,21 +1253,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac long destHostId = dest.getHost().getId(); vm.setPodIdToDeployIn(dest.getPod().getId()); - final Long cluster_id = dest.getCluster().getId(); - final ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO); - final ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + final Long clusterId = dest.getCluster().getId(); + updateOverCommitRatioForVmProfile(vmProfile, clusterId); - if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) == null && - (Float.parseFloat(cluster_detail_cpu.getValue()) > 1f || Float.parseFloat(cluster_detail_ram.getValue()) > 1f)) { - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); - } else if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO) != null) { - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cluster_detail_cpu.getValue(), true); - userVmDetailsDao.addDetail(vm.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, cluster_detail_ram.getValue(), true); - } - - vmProfile.setCpuOvercommitRatio(Float.parseFloat(cluster_detail_cpu.getValue())); - vmProfile.setMemoryOvercommitRatio(Float.parseFloat(cluster_detail_ram.getValue())); StartAnswer startAnswer = null; try { @@ -1281,7 +1270,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac resetVmNicsDeviceId(vm.getId()); _networkMgr.prepare(vmProfile, dest, ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { - checkAndAttemptMigrateVmAcrossCluster(vm, cluster_id, dest.getStorageForDisks()); + checkAndAttemptMigrateVmAcrossCluster(vm, clusterId, dest.getStorageForDisks()); volumeMgr.prepare(vmProfile, dest); } @@ -1485,6 +1474,27 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } + private void updateOverCommitRatioForVmProfile(VirtualMachineProfile vmProfile, long clusterId) { + final ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO); + final ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + final float parsedClusterCpuDetailCpu = Float.parseFloat(clusterDetailCpu.getValue()); + final float parsedClusterDetailRam = Float.parseFloat(clusterDetailRam.getValue()); + UserVmDetailVO vmDetailCpu = userVmDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO); + UserVmDetailVO vmDetailRam = userVmDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + + if ((vmDetailCpu == null && parsedClusterCpuDetailCpu > 1f) || + (vmDetailCpu != null && Float.parseFloat(vmDetailCpu.getValue()) != parsedClusterCpuDetailCpu)) { + userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, clusterDetailCpu.getValue(), true); + } + if ((vmDetailRam == null && parsedClusterDetailRam > 1f) || + (vmDetailRam != null && Float.parseFloat(vmDetailRam.getValue()) != parsedClusterDetailRam)) { + userVmDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, clusterDetailRam.getValue(), true); + } + + vmProfile.setCpuOvercommitRatio(Float.parseFloat(clusterDetailCpu.getValue())); + vmProfile.setMemoryOvercommitRatio(Float.parseFloat(clusterDetailRam.getValue())); + } + /** * Setting pod id to null can result in migration of Volumes across pods. This is not desirable for VMs which * have a volume in Ready state (happens when a VM is shutdown and started again). @@ -2766,6 +2776,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _networkMgr.prepareNicForMigration(profile, dest); volumeMgr.prepareForMigration(profile, dest); profile.setConfigDriveLabel(VmConfigDriveLabel.value()); + updateOverCommitRatioForVmProfile(profile, dest.getHost().getClusterId()); final VirtualMachineTO to = toVmTO(profile); final PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(to); @@ -2828,23 +2839,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } boolean migrated = false; - Map dpdkInterfaceMapping = null; + Map dpdkInterfaceMapping = new HashMap<>(); try { - final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); - final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType())); - if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { - mc.setVlanToPersistenceMap(vlanToPersistenceMap); - } - - boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); - mc.setAutoConvergence(kvmAutoConvergence); - mc.setHostGuid(dest.getHost().getGuid()); - - dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping(); - if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { - mc.setDpdkInterfaceMapping(dpdkInterfaceMapping); - } + final MigrateCommand mc = buildMigrateCommand(vm, to, dest, pfma, dpdkInterfaceMapping); try { final Answer ma = _agentMgr.send(vm.getLastHostId(), mc); @@ -2916,6 +2913,43 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } + /** + * Create and set parameters for the {@link MigrateCommand} used in the migration and scaling of VMs. + */ + protected MigrateCommand buildMigrateCommand(VMInstanceVO vmInstance, VirtualMachineTO virtualMachineTO, DeployDestination destination, Answer answer, + Map dpdkInterfaceMapping) { + final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vmInstance.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); + final MigrateCommand migrateCommand = new MigrateCommand(vmInstance.getInstanceName(), destination.getHost().getPrivateIpAddress(), isWindows, virtualMachineTO, + getExecuteInSequence(vmInstance.getHypervisorType())); + + Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vmInstance.getId()); + if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { + s_logger.debug(String.format("Setting VLAN persistence to [%s] as part of migrate command for VM [%s].", new Gson().toJson(vlanToPersistenceMap), virtualMachineTO)); + migrateCommand.setVlanToPersistenceMap(vlanToPersistenceMap); + } + + migrateCommand.setAutoConvergence(StorageManager.KvmAutoConvergence.value()); + migrateCommand.setHostGuid(destination.getHost().getGuid()); + + PrepareForMigrationAnswer prepareForMigrationAnswer = (PrepareForMigrationAnswer) answer; + + Map answerDpdkInterfaceMapping = prepareForMigrationAnswer.getDpdkInterfaceMapping(); + if (MapUtils.isNotEmpty(answerDpdkInterfaceMapping) && dpdkInterfaceMapping != null) { + s_logger.debug(String.format("Setting DPDK interface mapping to [%s] as part of migrate command for VM [%s].", new Gson().toJson(vlanToPersistenceMap), + virtualMachineTO)); + dpdkInterfaceMapping.putAll(answerDpdkInterfaceMapping); + migrateCommand.setDpdkInterfaceMapping(dpdkInterfaceMapping); + } + + Integer newVmCpuShares = prepareForMigrationAnswer.getNewVmCpuShares(); + if (newVmCpuShares != null) { + s_logger.debug(String.format("Setting CPU shares to [%d] as part of migrate command for VM [%s].", newVmCpuShares, virtualMachineTO)); + migrateCommand.setNewVmCpuShares(newVmCpuShares); + } + + return migrateCommand; + } + private void updateVmPod(VMInstanceVO vm, long dstHostId) { // update the VMs pod HostVO host = _hostDao.findById(dstHostId); @@ -4433,16 +4467,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac boolean migrated = false; try { - Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); - final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType())); - if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { - mc.setVlanToPersistenceMap(vlanToPersistenceMap); - } - - boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); - mc.setAutoConvergence(kvmAutoConvergence); - mc.setHostGuid(dest.getHost().getGuid()); + final MigrateCommand mc = buildMigrateCommand(vm, to, dest, pfma, null); try { final Answer ma = _agentMgr.send(vm.getLastHostId(), mc); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 1419ae36d25..a63aa52799d 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import com.cloud.agent.api.PrepareForMigrationAnswer; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -1884,9 +1885,10 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + Answer pfma; try { - Answer pfma = agentManager.send(destHost.getId(), pfmc); + pfma = agentManager.send(destHost.getId(), pfmc); if (pfma == null || !pfma.getResult()) { String details = pfma != null ? pfma.getDetails() : "null answer returned"; @@ -1894,8 +1896,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { throw new AgentUnavailableException(msg, destHost.getId()); } - } - catch (final OperationTimedoutException e) { + } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Operation timed out", destHost.getId()); } @@ -1911,6 +1912,12 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { migrateCommand.setMigrateStorageManaged(managedStorageDestination); migrateCommand.setMigrateNonSharedInc(migrateNonSharedInc); + Integer newVmCpuShares = ((PrepareForMigrationAnswer) pfma).getNewVmCpuShares(); + if (newVmCpuShares != null) { + LOGGER.debug(String.format("Setting CPU shares to [%d] as part of migrate VM with volumes command for VM [%s].", newVmCpuShares, vmTO)); + migrateCommand.setNewVmCpuShares(newVmCpuShares); + } + boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); migrateCommand.setAutoConvergence(kvmAutoConvergence); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index fadc51b8885..b5d7b9c3b4a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -72,6 +72,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.apache.xerces.impl.xpath.regex.Match; @@ -480,6 +481,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv */ private static final String COMMAND_SET_MEM_BALLOON_STATS_PERIOD = "virsh dommemstat %s --period %s --live"; + private static int hostCpuMaxCapacity = 0; + + private static final int CGROUP_V2_UPPER_LIMIT = 10000; + + private static final String COMMAND_GET_CGROUP_HOST_VERSION = "stat -fc %T /sys/fs/cgroup/"; + + public static final String CGROUP_V2 = "cgroup2fs"; + protected long getHypervisorLibvirtVersion() { return _hypervisorLibvirtVersion; } @@ -575,6 +584,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return new ExecutionResult(true, null); } + /** + * @return the host CPU max capacity according to the method {@link LibvirtComputingResource#calculateHostCpuMaxCapacity(int, Long)}; if the host utilizes cgroup v1, this + * value is 0. + */ + public int getHostCpuMaxCapacity() { + return hostCpuMaxCapacity; + } + + public void setHostCpuMaxCapacity(int hostCpuMaxCapacity) { + LibvirtComputingResource.hostCpuMaxCapacity = hostCpuMaxCapacity; + } + public LibvirtKvmAgentHook getTransformer() throws IOException { return new LibvirtKvmAgentHook(_agentHooksBasedir, _agentHooksLibvirtXmlScript, _agentHooksLibvirtXmlMethod); } @@ -2717,12 +2738,41 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv */ protected CpuTuneDef createCpuTuneDef(VirtualMachineTO vmTO) { CpuTuneDef ctd = new CpuTuneDef(); - int shares = vmTO.getCpus() * (vmTO.getMinSpeed() != null ? vmTO.getMinSpeed() : vmTO.getSpeed()); - ctd.setShares(shares); + ctd.setShares(calculateCpuShares(vmTO)); setQuotaAndPeriod(vmTO, ctd); return ctd; } + /** + * Calculates the VM CPU shares considering the cgroup version of the host. + *
    + *
  • + * If the host utilize cgroup v1, then, the CPU shares is calculated as VM CPU shares = CPU cores * CPU frequency. + *
    • + *
  • + * If the host utilize cgroup v2, the CPU shares calculation considers the cgroup v2 upper limit of 10,000, and a linear scale conversion is applied + * considering the maximum host CPU shares (i.e. using the number of CPU cores and CPU nominal frequency of the host). Therefore, the VM CPU shares is calculated as + * VM CPU shares = (VM requested shares * cgroup upper limit) / host max shares. + *
    • + *
+ */ + public int calculateCpuShares(VirtualMachineTO vmTO) { + int vCpus = vmTO.getCpus(); + int cpuSpeed = ObjectUtils.defaultIfNull(vmTO.getMinSpeed(), vmTO.getSpeed()); + int requestedCpuShares = vCpus * cpuSpeed; + int hostCpuMaxCapacity = getHostCpuMaxCapacity(); + + if (hostCpuMaxCapacity > 0) { + int updatedCpuShares = (int) Math.ceil((requestedCpuShares * CGROUP_V2_UPPER_LIMIT) / (double) hostCpuMaxCapacity); + s_logger.debug(String.format("This host utilizes cgroupv2 (as the max shares value is [%s]), thus, the VM requested shares of [%s] will be converted to " + + "consider the host limits; the new CPU shares value is [%s].", hostCpuMaxCapacity, requestedCpuShares, updatedCpuShares)); + return updatedCpuShares; + } + s_logger.debug(String.format("This host does not have a maximum CPU shares set; therefore, this host utilizes cgroupv1 and the VM requested CPU shares [%s] will not be " + + "converted.", requestedCpuShares)); + return requestedCpuShares; + } + private CpuModeDef createCpuModeDef(VirtualMachineTO vmTO, int vcpus) { final CpuModeDef cmd = new CpuModeDef(); cmd.setMode(_guestCpuMode); @@ -3567,6 +3617,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public StartupCommand[] initialize() { final KVMHostInfo info = new KVMHostInfo(_dom0MinMem, _dom0OvercommitMem, _manualCpuSpeed, _dom0MinCpuCores); + calculateHostCpuMaxCapacity(info.getAllocatableCpus(), info.getCpuSpeed()); String capabilities = String.join(",", info.getCapabilities()); if (dpdkSupport) { @@ -3628,6 +3679,33 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return hostTagsList; } + /** + * Calculates and sets the host CPU max capacity according to the cgroup version of the host. + *
    + *
  • + * cgroup v1: the max CPU capacity for the host is set to 0. + *
    • + *
  • + * cgroup v2: the max CPU capacity for the host is the value of cpuCores * cpuSpeed. + *
    • + *
+ */ + protected void calculateHostCpuMaxCapacity(int cpuCores, Long cpuSpeed) { + String output = Script.runSimpleBashScript(COMMAND_GET_CGROUP_HOST_VERSION); + s_logger.info(String.format("Host uses control group [%s].", output)); + + if (!CGROUP_V2.equals(output)) { + s_logger.info(String.format("Setting host CPU max capacity to 0, as it uses cgroup v1.", getHostCpuMaxCapacity())); + setHostCpuMaxCapacity(0); + return; + } + + s_logger.info(String.format("Calculating the max shares of the host.")); + setHostCpuMaxCapacity(cpuCores * cpuSpeed.intValue()); + s_logger.info(String.format("The max shares of the host is [%d].", getHostCpuMaxCapacity())); + } + + private StartupStorageCommand createLocalStoragePool(String localStoragePath, String localStorageUUID, StartupRoutingCommand cmd) { StartupStorageCommand sscmd = null; try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index d0ab77829af..fb526626ef8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Set; @@ -211,6 +212,8 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper + *
  • + * If both hosts utilize cgroup v1; then, the shares value of the VM is equal in both hosts, and there is no need to update the VM CPU shares value for the + * migration.
    • + *
  • + * If, at least, one of the hosts utilize cgroup v2, the VM CPU shares must be recalculated for the migration, accordingly to + * method {@link LibvirtComputingResource#calculateCpuShares(VirtualMachineTO)}. + *
    • + * + */ + protected String updateVmSharesIfNeeded(MigrateCommand migrateCommand, String xmlDesc, LibvirtComputingResource libvirtComputingResource) + throws ParserConfigurationException, IOException, SAXException, TransformerException { + Integer newVmCpuShares = migrateCommand.getNewVmCpuShares(); + int currentCpuShares = libvirtComputingResource.calculateCpuShares(migrateCommand.getVirtualMachine()); + + if (newVmCpuShares == currentCpuShares) { + s_logger.info(String.format("Current CPU shares [%s] is equal in both hosts; therefore, there is no need to update the CPU shares for the new host.", + currentCpuShares)); + return xmlDesc; + } + + InputStream inputStream = IOUtils.toInputStream(xmlDesc, StandardCharsets.UTF_8); + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document document = docBuilder.parse(inputStream); + + Element root = document.getDocumentElement(); + Node sharesNode = root.getElementsByTagName("shares").item(0); + String currentShares = sharesNode.getTextContent(); + + s_logger.info(String.format("VM [%s] will have CPU shares altered from [%s] to [%s] as part of migration because the cgroups version differs between hosts.", + migrateCommand.getVmName(), currentShares, newVmCpuShares)); + sharesNode.setTextContent(String.valueOf(newVmCpuShares)); + return getXml(document); + } + /** * Replace DPDK source path and target before migrations */ diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java index 9109d579c5b..3f281e54bba 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java @@ -122,11 +122,7 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp return new PrepareForMigrationAnswer(command, "failed to connect physical disks to host"); } - PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command); - if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { - answer.setDpdkInterfaceMapping(dpdkInterfaceMapping); - } - return answer; + return createPrepareForMigrationAnswer(command, dpdkInterfaceMapping, libvirtComputingResource, vm); } catch (final LibvirtException | CloudRuntimeException | InternalErrorException | URISyntaxException e) { if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { for (DpdkTO to : dpdkInterfaceMapping.values()) { @@ -143,6 +139,22 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp } } + protected PrepareForMigrationAnswer createPrepareForMigrationAnswer(PrepareForMigrationCommand command, Map dpdkInterfaceMapping, + LibvirtComputingResource libvirtComputingResource, VirtualMachineTO vm) { + PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command); + + if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { + s_logger.debug(String.format("Setting DPDK interface for the migration of VM [%s].", vm)); + answer.setDpdkInterfaceMapping(dpdkInterfaceMapping); + } + + int newCpuShares = libvirtComputingResource.calculateCpuShares(vm); + s_logger.debug(String.format("Setting CPU shares to [%s] for the migration of VM [%s].", newCpuShares, vm)); + answer.setNewVmCpuShares(newCpuShares); + + return answer; + } + private Answer handleRollback(PrepareForMigrationCommand command, LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); VirtualMachineTO vmTO = command.getVirtualMachine(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java index b3e95f8e3aa..1536984f2e8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapper.java @@ -39,8 +39,7 @@ public class LibvirtScaleVmCommandWrapper extends CommandWrapper\n" + ""; + @Mock + MigrateCommand migrateCommandMock; + + @Mock + LibvirtComputingResource libvirtComputingResourceMock; + + @Mock + VirtualMachineTO virtualMachineTOMock; + + @Spy LibvirtMigrateCommandWrapper libvirtMigrateCmdWrapper = new LibvirtMigrateCommandWrapper(); final String memInfo = "MemTotal: 5830236 kB\n" + @@ -871,4 +888,67 @@ public class LibvirtMigrateCommandWrapperTest { Assert.assertTrue(replaced.contains("csdpdk-7")); Assert.assertFalse(replaced.contains("csdpdk-1")); } + + @Test + public void updateVmSharesIfNeededTestNewCpuSharesEqualCurrentSharesShouldNotUpdateVmShares() throws ParserConfigurationException, IOException, TransformerException, + SAXException { + int newVmCpuShares = 1000; + int currentVmCpuShares = 1000; + + Mockito.doReturn(newVmCpuShares).when(migrateCommandMock).getNewVmCpuShares(); + Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine(); + Mockito.doReturn(currentVmCpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock); + + String finalXml = libvirtMigrateCmdWrapper.updateVmSharesIfNeeded(migrateCommandMock, fullfile, libvirtComputingResourceMock); + + Assert.assertEquals(finalXml, fullfile); + } + + @Test + public void updateVmSharesIfNeededTestNewCpuSharesHigherThanCurrentSharesShouldUpdateVmShares() throws ParserConfigurationException, IOException, TransformerException, + SAXException { + int newVmCpuShares = 2000; + int currentVmCpuShares = 1000; + + Mockito.doReturn(newVmCpuShares).when(migrateCommandMock).getNewVmCpuShares(); + Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine(); + Mockito.doReturn(currentVmCpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock); + + String finalXml = libvirtMigrateCmdWrapper.updateVmSharesIfNeeded(migrateCommandMock, fullfile, libvirtComputingResourceMock); + + InputStream inputStream = IOUtils.toInputStream(finalXml, StandardCharsets.UTF_8); + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document document = docBuilder.parse(inputStream); + + Element root = document.getDocumentElement(); + Node sharesNode = root.getElementsByTagName("shares").item(0); + int updateShares = Integer.parseInt(sharesNode.getTextContent()); + + Assert.assertEquals(updateShares, newVmCpuShares); + } + + @Test + public void updateVmSharesIfNeededTestNewCpuSharesLowerThanCurrentSharesShouldUpdateVmShares() throws ParserConfigurationException, IOException, TransformerException, + SAXException { + int newVmCpuShares = 500; + int currentVmCpuShares = 1000; + + Mockito.doReturn(newVmCpuShares).when(migrateCommandMock).getNewVmCpuShares(); + Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine(); + Mockito.doReturn(currentVmCpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock); + + String finalXml = libvirtMigrateCmdWrapper.updateVmSharesIfNeeded(migrateCommandMock, fullfile, libvirtComputingResourceMock); + + InputStream inputStream = IOUtils.toInputStream(finalXml, StandardCharsets.UTF_8); + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document document = docBuilder.parse(inputStream); + + Element root = document.getDocumentElement(); + Node sharesNode = root.getElementsByTagName("shares").item(0); + int updateShares = Integer.parseInt(sharesNode.getTextContent()); + + Assert.assertEquals(updateShares, newVmCpuShares); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapperTest.java new file mode 100644 index 00000000000..5530819c2e4 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapperTest.java @@ -0,0 +1,75 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.PrepareForMigrationAnswer; +import com.cloud.agent.api.PrepareForMigrationCommand; +import com.cloud.agent.api.to.DpdkTO; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.HashMap; +import java.util.Map; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(value = {LibvirtPrepareForMigrationCommandWrapper.class}) +public class LibvirtPrepareForMigrationCommandWrapperTest { + + @Mock + LibvirtComputingResource libvirtComputingResourceMock; + + @Mock + PrepareForMigrationCommand prepareForMigrationCommandMock; + + @Mock + VirtualMachineTO virtualMachineTOMock; + + @Spy + LibvirtPrepareForMigrationCommandWrapper libvirtPrepareForMigrationCommandWrapperSpy = new LibvirtPrepareForMigrationCommandWrapper(); + + @Test + public void createPrepareForMigrationAnswerTestDpdkInterfaceNotEmptyShouldSetParamOnAnswer() { + Map dpdkInterfaceMapping = new HashMap<>(); + dpdkInterfaceMapping.put("Interface", new DpdkTO()); + + PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock, dpdkInterfaceMapping, libvirtComputingResourceMock, + virtualMachineTOMock); + + Assert.assertEquals(prepareForMigrationAnswer.getDpdkInterfaceMapping(), dpdkInterfaceMapping); + } + + @Test + public void createPrepareForMigrationAnswerTestVerifyThatCpuSharesIsSet() { + int cpuShares = 1000; + Mockito.doReturn(cpuShares).when(libvirtComputingResourceMock).calculateCpuShares(virtualMachineTOMock); + PrepareForMigrationAnswer prepareForMigrationAnswer = libvirtPrepareForMigrationCommandWrapperSpy.createPrepareForMigrationAnswer(prepareForMigrationCommandMock,null, + libvirtComputingResourceMock, virtualMachineTOMock); + + Assert.assertEquals(cpuShares, prepareForMigrationAnswer.getNewVmCpuShares().intValue()); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapperTest.java index fb963e87ed4..56f99d41abd 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtScaleVmCommandWrapperTest.java @@ -207,9 +207,11 @@ public class LibvirtScaleVmCommandWrapperTest extends TestCase { @Test public void validateExecuteHandleLibvirtException() throws LibvirtException { String errorMessage = ""; + int shares = vmTo.getCpus() * vmTo.getSpeed(); Mockito.doReturn(vmTo).when(scaleVmCommandMock).getVirtualMachine(); Mockito.doReturn(libvirtUtilitiesHelperMock).when(libvirtComputingResourceMock).getLibvirtUtilitiesHelper(); + Mockito.doReturn(shares).when(libvirtComputingResourceMock).calculateCpuShares(vmTo); Mockito.doThrow(libvirtException).when(libvirtUtilitiesHelperMock).getConnectionByVmName(Mockito.anyString()); Mockito.doReturn(errorMessage).when(libvirtException).getMessage(); @@ -222,9 +224,12 @@ public class LibvirtScaleVmCommandWrapperTest extends TestCase { @Test public void validateExecuteSuccessfully() throws LibvirtException { + int shares = vmTo.getCpus() * vmTo.getSpeed(); + Mockito.doReturn(vmTo).when(scaleVmCommandMock).getVirtualMachine(); Mockito.doReturn(libvirtUtilitiesHelperMock).when(libvirtComputingResourceMock).getLibvirtUtilitiesHelper(); Mockito.doReturn(connectMock).when(libvirtUtilitiesHelperMock).getConnectionByVmName(Mockito.anyString()); + Mockito.doReturn(shares).when(libvirtComputingResourceMock).calculateCpuShares(vmTo); Mockito.doReturn(domainMock).when(connectMock).domainLookupByName(Mockito.anyString()); Mockito.doNothing().when(libvirtScaleVmCommandWrapperSpy).scaleMemory(Mockito.any(), Mockito.anyLong(), Mockito.anyString()); Mockito.doNothing().when(libvirtScaleVmCommandWrapperSpy).scaleVcpus(Mockito.any(), Mockito.anyInt(), Mockito.anyString());