From 939ee9e1534a1e6013a74f92dbb151666b3887e3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Jul 2023 14:04:38 +0530 Subject: [PATCH] server,engine-orchestration: allocate vm without transaction (#7695) When deploying a VM is failed during the allocation process it may leave the resources that have been already allocated before the failure. They will get removed from the database as the whole code block is wrapped inside a transaction twice but the server would not inform the network or storage plugins to clean up the allocated resources. This PR removes Transactions during VM allocation which results in the allocated VM and its resource records being persisted in DB even during failures. When failure is encountered VM is moved to Error state. This helps VM and its resources to be properly deallocated when it is expunged either by a server task such as ExpungeTask or during manual expunge. Signed-off-by: Abhishek Kumar --- .../cloud/vm/VirtualMachineManagerImpl.java | 113 +++---- .../java/com/cloud/vm/UserVmManagerImpl.java | 288 +++++++++--------- 2 files changed, 201 insertions(+), 200 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 9b37132d07c..d3ba95061b6 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -458,72 +458,79 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throws InsufficientCapacityException { s_logger.info(String.format("allocating virtual machine from template:%s with hostname:%s and %d networks", template.getUuid(), vmInstanceName, auxiliaryNetworks.size())); + VMInstanceVO persistedVm = null; + try { + final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); + final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); - final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); - final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating entries for VM: " + vm); + } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating entries for VM: " + vm); - } + vm.setDataCenterId(plan.getDataCenterId()); + if (plan.getPodId() != null) { + vm.setPodIdToDeployIn(plan.getPodId()); + } + assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet"; + persistedVm = _vmDao.persist(vm); - vm.setDataCenterId(plan.getDataCenterId()); - if (plan.getPodId() != null) { - vm.setPodIdToDeployIn(plan.getPodId()); - } - assert plan.getClusterId() == null && plan.getPoolId() == null : "We currently don't support cluster and pool preset yet"; - final VMInstanceVO vmFinal = _vmDao.persist(vm); + final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(persistedVm, template, serviceOffering, null, null); - final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vmFinal, template, serviceOffering, null, null); + Long rootDiskSize = rootDiskOfferingInfo.getSize(); + if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) { + rootDiskSize = SystemVmRootDiskSize.value(); + } + final Long rootDiskSizeFinal = rootDiskSize; - Long rootDiskSize = rootDiskOfferingInfo.getSize(); - if (vm.getType().isUsedBySystem() && SystemVmRootDiskSize.value() != null && SystemVmRootDiskSize.value() > 0L) { - rootDiskSize = SystemVmRootDiskSize.value(); - } - final Long rootDiskSizeFinal = rootDiskSize; + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating nics for " + persistedVm); + } - Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { - @Override - public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating nics for " + vmFinal); + try { + if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) { + _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions); } + } catch (final ConcurrentOperationException e) { + throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e); + } - try { - if (!vmProfile.getBootArgs().contains("ExternalLoadBalancerVm")) { - _networkMgr.allocate(vmProfile, auxiliaryNetworks, extraDhcpOptions); - } - } catch (final ConcurrentOperationException e) { - throw new CloudRuntimeException("Concurrent operation while trying to allocate resources for the VM", e); - } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocating disks for " + persistedVm); + } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocating disks for " + vmFinal); - } + allocateRootVolume(persistedVm, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal); - allocateRootVolume(vmFinal, template, rootDiskOfferingInfo, owner, rootDiskSizeFinal); - - if (dataDiskOfferings != null) { - for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), - dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), vmFinal, template, owner, null); - } - } - if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { - int diskNumber = 1; - for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) { - DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); - long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); - VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); - volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + vmFinal.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, - vmFinal, dataDiskTemplate, owner, Long.valueOf(diskNumber)); - diskNumber++; - } + if (dataDiskOfferings != null) { + for (final DiskOfferingInfo dataDiskOfferingInfo : dataDiskOfferings) { + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId(), dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), + dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, null); + } + } + if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { + int diskNumber = 1; + for (Entry dataDiskTemplateToDiskOfferingMap : datadiskTemplateToDiskOfferingMap.entrySet()) { + DiskOffering diskOffering = dataDiskTemplateToDiskOfferingMap.getValue(); + long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); + VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); + volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf(diskNumber), diskOffering, diskOfferingSize, null, null, + persistedVm, dataDiskTemplate, owner, Long.valueOf(diskNumber)); + diskNumber++; } } - }); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Allocation completed for VM: " + vmFinal); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Allocation completed for VM: " + persistedVm); + } + } catch (InsufficientCapacityException | CloudRuntimeException e) { + // Failed VM will be in Stopped. Transition it to Error, so it can be expunged by ExpungeTask or similar + try { + if (persistedVm != null) { + stateTransitTo(persistedVm, VirtualMachine.Event.OperationFailedToError, null); + } + } catch (NoTransitionException nte) { + s_logger.error(String.format("Failed to transition %s in %s state to Error state", persistedVm, persistedVm.getState().toString())); + } + throw e; } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f0aed627f15..b4e8e137984 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -348,7 +348,6 @@ import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.db.UUIDManager; @@ -4366,161 +4365,156 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs) throws InsufficientCapacityException { - return Transaction.execute(new TransactionCallbackWithException() { - @Override - public UserVmVO doInTransaction(TransactionStatus status) throws InsufficientCapacityException { - UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), - offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); - vm.setUuid(uuidName); - vm.setDynamicallyScalable(dynamicScalingEnabled); + UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), + offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); + vm.setUuid(uuidName); + vm.setDynamicallyScalable(dynamicScalingEnabled); - Map details = template.getDetails(); - if (details != null && !details.isEmpty()) { - vm.details.putAll(details); - } + Map details = template.getDetails(); + if (details != null && !details.isEmpty()) { + vm.details.putAll(details); + } - if (StringUtils.isNotBlank(sshPublicKeys)) { - vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys); - } + if (StringUtils.isNotBlank(sshPublicKeys)) { + vm.setDetail(VmDetailConstants.SSH_PUBLIC_KEY, sshPublicKeys); + } - if (StringUtils.isNotBlank(sshkeypairs)) { - vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs); - } + if (StringUtils.isNotBlank(sshkeypairs)) { + vm.setDetail(VmDetailConstants.SSH_KEY_PAIR_NAMES, sshkeypairs); + } - if (keyboard != null && !keyboard.isEmpty()) { - vm.setDetail(VmDetailConstants.KEYBOARD, keyboard); - } + if (keyboard != null && !keyboard.isEmpty()) { + vm.setDetail(VmDetailConstants.KEYBOARD, keyboard); + } - if (!isImport && isIso) { - vm.setIsoId(template.getId()); - } + if (!isImport && isIso) { + vm.setIsoId(template.getId()); + } - long guestOSId = template.getGuestOSId(); - GuestOSVO guestOS = _guestOSDao.findById(guestOSId); - long guestOSCategoryId = guestOS.getCategoryId(); - GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); - if (hypervisorType.equals(HypervisorType.VMware)) { - updateVMDiskController(vm, customParameters, guestOS); - } + long guestOSId = template.getGuestOSId(); + GuestOSVO guestOS = _guestOSDao.findById(guestOSId); + long guestOSCategoryId = guestOS.getCategoryId(); + GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); + if (hypervisorType.equals(HypervisorType.VMware)) { + updateVMDiskController(vm, customParameters, guestOS); + } - Long rootDiskSize = null; - // custom root disk size, resizes base template to larger size - if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { - // already verified for positive number - rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); + Long rootDiskSize = null; + // custom root disk size, resizes base template to larger size + if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { + // already verified for positive number + rootDiskSize = Long.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE)); - VMTemplateVO templateVO = _templateDao.findById(template.getId()); - if (templateVO == null) { - throw new InvalidParameterValueException("Unable to look up template by id " + template.getId()); - } - - validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters); - } - - if (isDisplayVm != null) { - vm.setDisplayVm(isDisplayVm); - } else { - vm.setDisplayVm(true); - } - - if (isImport) { - vm.setDataCenterId(zone.getId()); - vm.setHostId(host.getId()); - if (lastHost != null) { - vm.setLastHostId(lastHost.getId()); - } - vm.setPowerState(powerState); - if (powerState == VirtualMachine.PowerState.PowerOn) { - vm.setState(State.Running); - } - } - - vm.setUserVmType(vmType); - _vmDao.persist(vm); - for (String key : customParameters.keySet()) { - // BIOS was explicitly passed as the boot type, so honour it - if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) { - vm.details.remove(ApiConstants.BootType.UEFI.toString()); - continue; - } - - // Deploy as is, Don't care about the boot type or template settings - if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) { - vm.details.remove(ApiConstants.BootType.UEFI.toString()); - continue; - } - - if (!hypervisorType.equals(HypervisorType.KVM)) { - if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) { - vm.details.remove(VmDetailConstants.IOTHREADS); - continue; - } - if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) { - vm.details.remove(VmDetailConstants.IO_POLICY); - continue; - } - } - - if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || - key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || - key.equalsIgnoreCase(VmDetailConstants.MEMORY)) { - // handle double byte strings. - vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key)))); - } else { - vm.setDetail(key, customParameters.get(key)); - } - } - vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); - - persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); - - List hiddenDetails = new ArrayList<>(); - if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) { - hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR); - } - _vmDao.saveDetails(vm, hiddenDetails); - if (!isImport) { - s_logger.debug("Allocating in the DB for vm"); - DataCenterDeployment plan = new DataCenterDeployment(zone.getId()); - - List computeTags = new ArrayList(); - computeTags.add(offering.getHostTag()); - - List rootDiskTags = new ArrayList(); - DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId); - rootDiskTags.add(rootDiskOfferingVO.getTags()); - - if (isIso) { - _orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName, - hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, - networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId); - } else { - _orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(), - offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap, - dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId); - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Successfully allocated DB entry for " + vm); - } - } - CallContext.current().setEventDetails("Vm Id: " + vm.getUuid()); - - if (!isImport) { - if (!offering.isDynamic()) { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), - hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); - } else { - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), - hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm()); - } - - //Update Resource Count for the given account - resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); - } - return vm; + VMTemplateVO templateVO = _templateDao.findById(template.getId()); + if (templateVO == null) { + throw new InvalidParameterValueException("Unable to look up template by id " + template.getId()); } - }); + + validateRootDiskResize(hypervisorType, rootDiskSize, templateVO, vm, customParameters); + } + + if (isDisplayVm != null) { + vm.setDisplayVm(isDisplayVm); + } else { + vm.setDisplayVm(true); + } + + if (isImport) { + vm.setDataCenterId(zone.getId()); + vm.setHostId(host.getId()); + if (lastHost != null) { + vm.setLastHostId(lastHost.getId()); + } + vm.setPowerState(powerState); + if (powerState == VirtualMachine.PowerState.PowerOn) { + vm.setState(State.Running); + } + } + + vm.setUserVmType(vmType); + _vmDao.persist(vm); + for (String key : customParameters.keySet()) { + // BIOS was explicitly passed as the boot type, so honour it + if (key.equalsIgnoreCase(ApiConstants.BootType.BIOS.toString())) { + vm.details.remove(ApiConstants.BootType.UEFI.toString()); + continue; + } + + // Deploy as is, Don't care about the boot type or template settings + if (key.equalsIgnoreCase(ApiConstants.BootType.UEFI.toString()) && template.isDeployAsIs()) { + vm.details.remove(ApiConstants.BootType.UEFI.toString()); + continue; + } + + if (!hypervisorType.equals(HypervisorType.KVM)) { + if (key.equalsIgnoreCase(VmDetailConstants.IOTHREADS)) { + vm.details.remove(VmDetailConstants.IOTHREADS); + continue; + } + if (key.equalsIgnoreCase(VmDetailConstants.IO_POLICY)) { + vm.details.remove(VmDetailConstants.IO_POLICY); + continue; + } + } + + if (key.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || + key.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || + key.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + // handle double byte strings. + vm.setDetail(key, Integer.toString(Integer.parseInt(customParameters.get(key)))); + } else { + vm.setDetail(key, customParameters.get(key)); + } + } + vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); + + persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); + + List hiddenDetails = new ArrayList<>(); + if (customParameters.containsKey(VmDetailConstants.NAME_ON_HYPERVISOR)) { + hiddenDetails.add(VmDetailConstants.NAME_ON_HYPERVISOR); + } + _vmDao.saveDetails(vm, hiddenDetails); + if (!isImport) { + s_logger.debug("Allocating in the DB for vm"); + DataCenterDeployment plan = new DataCenterDeployment(zone.getId()); + + List computeTags = new ArrayList(); + computeTags.add(offering.getHostTag()); + + List rootDiskTags = new ArrayList(); + DiskOfferingVO rootDiskOfferingVO = _diskOfferingDao.findById(rootDiskOfferingId); + rootDiskTags.add(rootDiskOfferingVO.getTags()); + + if (isIso) { + _orchSrvc.createVirtualMachineFromScratch(vm.getUuid(), Long.toString(owner.getAccountId()), vm.getIsoId().toString(), hostName, displayName, + hypervisorType.name(), guestOSCategory.getName(), offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, + networkNicMap, plan, extraDhcpOptionMap, rootDiskOfferingId); + } else { + _orchSrvc.createVirtualMachine(vm.getUuid(), Long.toString(owner.getAccountId()), Long.toString(template.getId()), hostName, displayName, hypervisorType.name(), + offering.getCpu(), offering.getSpeed(), offering.getRamSize(), diskSize, computeTags, rootDiskTags, networkNicMap, plan, rootDiskSize, extraDhcpOptionMap, + dataDiskTemplateToDiskOfferingMap, diskOfferingId, rootDiskOfferingId); + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Successfully allocated DB entry for " + vm); + } + } + CallContext.current().setEventDetails("Vm Id: " + vm.getUuid()); + + if (!isImport) { + if (!offering.isDynamic()) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), + hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); + } else { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, zone.getId(), vm.getId(), vm.getHostName(), offering.getId(), template.getId(), + hypervisorType.toString(), VirtualMachine.class.getName(), vm.getUuid(), customParameters, vm.isDisplayVm()); + } + + //Update Resource Count for the given account + resourceCountIncrement(accountId, isDisplayVm, new Long(offering.getCpu()), new Long(offering.getRamSize())); + } + return vm; } private void updateVMDiskController(UserVmVO vm, Map customParameters, GuestOSVO guestOS) {