From c56fc7f81c762a8ecdec2e445ed3d9a2c7b73337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernardo=20De=20Marco=20Gon=C3=A7alves?= Date: Thu, 28 Nov 2024 13:45:20 -0300 Subject: [PATCH 1/8] fix section divider display on auth page (#9966) --- ui/src/views/auth/Login.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/views/auth/Login.vue b/ui/src/views/auth/Login.vue index 8503f71082b..e2d3ee4c746 100644 --- a/ui/src/views/auth/Login.vue +++ b/ui/src/views/auth/Login.vue @@ -289,7 +289,6 @@ export default { if (response) { const oauthproviders = response.listoauthproviderresponse.oauthprovider || [] oauthproviders.forEach(item => { - this.socialLogin = true if (item.provider === 'google') { this.googleprovider = item.enabled this.googleclientid = item.clientid @@ -301,6 +300,7 @@ export default { this.githubredirecturi = item.redirecturi } }) + this.socialLogin = this.googleprovider || this.githubprovider } }) }, From a73841a693cb6ce0281bce8aa8de3d675480f322 Mon Sep 17 00:00:00 2001 From: dahn Date: Thu, 28 Nov 2024 17:46:01 +0100 Subject: [PATCH 2/8] get expunged VM data for job result (#9949) --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 9d5a1be894b..6e75512948a 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5611,7 +5611,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir //Update Resource Count for the given account resourceCountDecrement(vm.getAccountId(), vm.isDisplayVm(), new Long(offering.getCpu()), new Long(offering.getRamSize())); } - return _vmDao.findById(vmId); + return _vmDao.findByIdIncludingRemoved(vmId); } else { CloudRuntimeException ex = new CloudRuntimeException("Failed to destroy vm with specified vmId"); ex.addProxyObject(vm.getUuid(), "vmId"); From d54b105a037e781c45d61718c2d809ddfa3e53cd Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Thu, 28 Nov 2024 17:47:47 +0100 Subject: [PATCH 3/8] Linstor: add support for ISO block devices and direct download (#9792) --- .../resource/LibvirtComputingResource.java | 31 +++++++--- .../kvm/resource/LibvirtDomainXMLParser.java | 24 +++++++- .../hypervisor/kvm/resource/LibvirtVMDef.java | 24 ++++---- .../kvm/storage/KVMStorageProcessor.java | 5 +- .../kvm/storage/LibvirtStorageAdaptor.java | 6 +- plugins/storage/volume/linstor/CHANGELOG.md | 6 ++ .../kvm/storage/LinstorStorageAdaptor.java | 60 ++++++++++++++++--- 7 files changed, 118 insertions(+), 38 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 43923059d91..a4d7bd524ad 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -2983,6 +2983,17 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return dataPath; } + public static boolean useBLOCKDiskType(KVMPhysicalDisk physicalDisk) { + return physicalDisk != null && + physicalDisk.getPool().getType() == StoragePoolType.Linstor && + physicalDisk.getFormat() != null && + physicalDisk.getFormat()== PhysicalDiskFormat.RAW; + } + + public static DiskDef.DiskType getDiskType(KVMPhysicalDisk physicalDisk) { + return useBLOCKDiskType(physicalDisk) ? DiskDef.DiskType.BLOCK : DiskDef.DiskType.FILE; + } + public void createVbd(final Connect conn, final VirtualMachineTO vmSpec, final String vmName, final LibvirtVMDef vm) throws InternalErrorException, LibvirtException, URISyntaxException { final Map details = vmSpec.getDetails(); final List disks = Arrays.asList(vmSpec.getDisks()); @@ -3028,7 +3039,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv physicalDisk = getPhysicalDiskFromNfsStore(dataStoreUrl, data); } else if (primaryDataStoreTO.getPoolType().equals(StoragePoolType.SharedMountPoint) || primaryDataStoreTO.getPoolType().equals(StoragePoolType.Filesystem) || - primaryDataStoreTO.getPoolType().equals(StoragePoolType.StorPool)) { + primaryDataStoreTO.getPoolType().equals(StoragePoolType.StorPool) || + primaryDataStoreTO.getPoolType().equals(StoragePoolType.Linstor)) { physicalDisk = getPhysicalDiskPrimaryStore(primaryDataStoreTO, data); } } @@ -3078,8 +3090,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final DiskDef disk = new DiskDef(); int devId = volume.getDiskSeq().intValue(); if (volume.getType() == Volume.Type.ISO) { - - disk.defISODisk(volPath, devId, isUefiEnabled); + final DiskDef.DiskType diskType = getDiskType(physicalDisk); + disk.defISODisk(volPath, devId, isUefiEnabled, diskType); if (guestCpuArch != null && guestCpuArch.equals("aarch64")) { disk.setBusType(DiskDef.DiskBus.SCSI); @@ -3171,7 +3183,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (vmSpec.getType() != VirtualMachine.Type.User) { final DiskDef iso = new DiskDef(); - iso.defISODisk(sysvmISOPath); + iso.defISODisk(sysvmISOPath, DiskDef.DiskType.FILE); if (guestCpuArch != null && guestCpuArch.equals("aarch64")) { iso.setBusType(DiskDef.DiskBus.SCSI); } @@ -3384,7 +3396,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv List disks = getDisks(conn, vmName); DiskDef configdrive = null; for (DiskDef disk : disks) { - if (disk.getDeviceType() == DiskDef.DeviceType.CDROM && disk.getDiskLabel() == CONFIG_DRIVE_ISO_DISK_LABEL) { + if (disk.getDeviceType() == DiskDef.DeviceType.CDROM && CONFIG_DRIVE_ISO_DISK_LABEL.equals(disk.getDiskLabel())) { configdrive = disk; } } @@ -3414,11 +3426,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final String name = isoPath.substring(index + 1); final KVMStoragePool secondaryPool = storagePoolManager.getStoragePoolByURI(path); final KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name); + final DiskDef.DiskType diskType = getDiskType(isoVol); isoPath = isoVol.getPath(); - iso.defISODisk(isoPath, diskSeq); + iso.defISODisk(isoPath, diskSeq, diskType); } else { - iso.defISODisk(null, diskSeq); + iso.defISODisk(null, diskSeq, DiskDef.DiskType.FILE); } final String result = attachOrDetachDevice(conn, true, vmName, iso.toString()); @@ -3426,7 +3439,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final List disks = getDisks(conn, vmName); for (final DiskDef disk : disks) { if (disk.getDeviceType() == DiskDef.DeviceType.CDROM - && (diskSeq == null || disk.getDiskLabel() == iso.getDiskLabel())) { + && (diskSeq == null || disk.getDiskLabel().equals(iso.getDiskLabel()))) { cleanupDisk(disk); } } @@ -4002,7 +4015,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return stopVMInternal(conn, vmName, true); } String ret = stopVMInternal(conn, vmName, false); - if (ret == Script.ERR_TIMEOUT) { + if (Script.ERR_TIMEOUT.equals(ret)) { ret = stopVMInternal(conn, vmName, true); } else if (ret != null) { /* diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index a0dd270f999..4cd7fbf3b39 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -126,11 +126,10 @@ public class LibvirtDomainXMLParser { } def.defFileBasedDisk(diskFile, diskLabel, DiskDef.DiskBus.valueOf(bus.toUpperCase()), fmt); } else if (device.equalsIgnoreCase("cdrom")) { - def.defISODisk(diskFile, i+1, diskLabel); + def.defISODisk(diskFile, i+1, diskLabel, DiskDef.DiskType.FILE); } } else if (type.equalsIgnoreCase("block")) { - def.defBlockBasedDisk(diskDev, diskLabel, - DiskDef.DiskBus.valueOf(bus.toUpperCase())); + parseDiskBlock(def, device, diskDev, diskLabel, bus, diskFile, i); } if (StringUtils.isNotBlank(diskCacheMode)) { def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase())); @@ -449,6 +448,25 @@ public class LibvirtDomainXMLParser { return node.getAttribute(attr); } + /** + * Parse the disk block part of the libvirt XML. + * @param def + * @param device + * @param diskDev + * @param diskLabel + * @param bus + * @param diskFile + * @param curDiskIndex + */ + private void parseDiskBlock(DiskDef def, String device, String diskDev, String diskLabel, String bus, + String diskFile, int curDiskIndex) { + if (device.equalsIgnoreCase("disk")) { + def.defBlockBasedDisk(diskDev, diskLabel, DiskDef.DiskBus.valueOf(bus.toUpperCase())); + } else if (device.equalsIgnoreCase("cdrom")) { + def.defISODisk(diskFile, curDiskIndex+1, diskLabel, DiskDef.DiskType.BLOCK); + } + } + public Integer getVncPort() { return vncPort; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index cfd72c28b5a..ec940942082 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -833,8 +833,8 @@ public class LibvirtVMDef { } } - public void defISODisk(String volPath) { - _diskType = DiskType.FILE; + public void defISODisk(String volPath, DiskType diskType) { + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; _diskLabel = getDevLabel(3, DiskBus.IDE, true); @@ -843,8 +843,8 @@ public class LibvirtVMDef { _bus = DiskBus.IDE; } - public void defISODisk(String volPath, boolean isUefiEnabled) { - _diskType = DiskType.FILE; + public void defISODisk(String volPath, boolean isUefiEnabled, DiskType diskType) { + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; _bus = isUefiEnabled ? DiskBus.SATA : DiskBus.IDE; @@ -853,18 +853,18 @@ public class LibvirtVMDef { _diskCacheMode = DiskCacheMode.NONE; } - public void defISODisk(String volPath, Integer devId) { - defISODisk(volPath, devId, null); + public void defISODisk(String volPath, Integer devId, DiskType diskType) { + defISODisk(volPath, devId, null, diskType); } - public void defISODisk(String volPath, Integer devId, String diskLabel) { + public void defISODisk(String volPath, Integer devId, String diskLabel, DiskType diskType) { if (devId == null && StringUtils.isBlank(diskLabel)) { s_logger.debug(String.format("No ID or label informed for volume [%s].", volPath)); - defISODisk(volPath); + defISODisk(volPath, diskType); return; } - _diskType = DiskType.FILE; + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; @@ -881,11 +881,11 @@ public class LibvirtVMDef { _bus = DiskBus.IDE; } - public void defISODisk(String volPath, Integer devId,boolean isSecure) { + public void defISODisk(String volPath, Integer devId, boolean isSecure, DiskType diskType) { if (!isSecure) { - defISODisk(volPath, devId); + defISODisk(volPath, devId, diskType); } else { - _diskType = DiskType.FILE; + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; _diskLabel = getDevLabel(devId, DiskBus.SATA, true); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 75443605c31..8cee8434b5e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -1114,11 +1114,12 @@ public class KVMStorageProcessor implements StorageProcessor { storagePool = storagePoolMgr.getStoragePoolByURI(path); } final KVMPhysicalDisk isoVol = storagePool.getPhysicalDisk(name); + final DiskDef.DiskType isoDiskType = LibvirtComputingResource.getDiskType(isoVol); isoPath = isoVol.getPath(); - iso.defISODisk(isoPath, isUefiEnabled); + iso.defISODisk(isoPath, isUefiEnabled, isoDiskType); } else { - iso.defISODisk(null, isUefiEnabled); + iso.defISODisk(null, isUefiEnabled, DiskDef.DiskType.FILE); } final List disks = resource.getDisks(conn, vmName); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 57a492452e5..11375969b6e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -172,7 +172,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { * Checks if downloaded template is extractable * @return true if it should be extracted, false if not */ - private boolean isTemplateExtractable(String templatePath) { + public static boolean isTemplateExtractable(String templatePath) { String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); } @@ -182,7 +182,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { * @param downloadedTemplateFile * @param templateUuid */ - private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) { + public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) { if (downloadedTemplateFile.endsWith(".zip")) { return "unzip -p " + downloadedTemplateFile + " | cat > " + templateUuid; } else if (downloadedTemplateFile.endsWith(".bz2")) { @@ -197,7 +197,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { /** * Extract downloaded template into installPath, remove compressed file */ - private void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { + public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { String extractCommand = getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + downloadedTemplateFile); diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index a0a53d20119..930e139870f 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Disable discard="unmap" for ide devices and qemu < 7.0 https://bugzilla.redhat.com/show_bug.cgi?id=2029980 +## [2024-10-14] + +### Added + +- Support for ISO direct download to primary storage + ## [2024-10-04] ### Added diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index bc2a3a42d97..1f71a54a4f3 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -23,11 +23,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import javax.annotation.Nonnull; import com.cloud.storage.Storage; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; import org.apache.cloudstack.storage.datastore.util.LinstorUtil; import org.apache.cloudstack.utils.qemu.QemuImg; @@ -56,6 +58,8 @@ import com.linbit.linstor.api.model.StoragePool; import com.linbit.linstor.api.model.Volume; import com.linbit.linstor.api.model.VolumeDefinition; +import java.io.File; + @StorageAdaptorInfo(storagePoolType=Storage.StoragePoolType.Linstor) public class LinstorStorageAdaptor implements StorageAdaptor { private static final Logger s_logger = Logger.getLogger(LinstorStorageAdaptor.class); @@ -563,13 +567,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null); final DevelopersApi api = getLinstorAPI(destPools); - final String rscName = LinstorUtil.RSC_PREFIX + name; - try { - LinstorUtil.applyAuxProps(api, rscName, disk.getDispName(), disk.getVmName()); - } catch (ApiException apiExc) { - s_logger.error(String.format("Error setting aux properties for %s", rscName)); - logLinstorAnswers(apiExc.getApiCallRcList()); - } + applyAuxProps(api, name, disk.getDispName(), disk.getVmName()); s_logger.debug(String.format("Linstor.copyPhysicalDisk: dstPath: %s", dstDisk.getPath())); final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath()); @@ -620,13 +618,57 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return null; } + private void fileExistsOrThrow(String templateFilePath) { + File sourceFile = new File(templateFilePath); + if (!sourceFile.exists()) { + throw new CloudRuntimeException("Direct download template file " + sourceFile + + " does not exist on this host"); + } + } + + private String getFinalDirectDownloadPath(String templateFilePath, KVMStoragePool destPool) { + String finalSourcePath = templateFilePath; + if (LibvirtStorageAdaptor.isTemplateExtractable(templateFilePath)) { + finalSourcePath = templateFilePath.substring(0, templateFilePath.lastIndexOf('.')); + LibvirtStorageAdaptor.extractDownloadedTemplate(templateFilePath, destPool, finalSourcePath); + } + return finalSourcePath; + } + + private void applyAuxProps(DevelopersApi api, String csPath, String csName, String csVMName) { + final String rscName = getLinstorRscName(csPath); + try { + LinstorUtil.applyAuxProps(api, rscName, csName, csVMName); + } catch (ApiException apiExc) { + s_logger.error(String.format("Error setting aux properties for %s", rscName)); + logLinstorAnswers(apiExc.getApiCallRcList()); + } + } + @Override public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { - s_logger.debug("Linstor: createTemplateFromDirectDownloadFile"); - return null; + s_logger.debug(String.format("Linstor: createTemplateFromDirectDownloadFile: %s/%s", templateFilePath, format)); + fileExistsOrThrow(templateFilePath); + String name = UUID.randomUUID().toString(); + + String finalSourcePath = getFinalDirectDownloadPath(templateFilePath, destPool); + + File finalSourceFile = new File(finalSourcePath); + final KVMPhysicalDisk dstDisk = destPool.createPhysicalDisk( + name, QemuImg.PhysicalDiskFormat.RAW, Storage.ProvisioningType.THIN, finalSourceFile.length(), null); + + final DevelopersApi api = getLinstorAPI(destPool); + applyAuxProps(api, name, finalSourceFile.getName(), null); + + Script.runSimpleBashScript( + String.format("dd if=\"%s\" of=\"%s\" bs=64k conv=nocreat,sparse oflag=direct", + finalSourcePath, dstDisk.getPath())); + + Script.runSimpleBashScript("rm " + finalSourcePath); + return dstDisk; } public long getCapacity(LinstorStoragePool pool) { From 37eec061669ff63488bc2fa522806b367e47cb48 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Thu, 28 Nov 2024 22:20:08 +0530 Subject: [PATCH 4/8] Allow VMWare import via another host (#9787) Co-authored-by: Suresh Kumar Anaparti --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/command/admin/vm/ImportVmCmd.java | 12 +- .../agent/api/ConvertInstanceAnswer.java | 17 +- .../agent/api/ConvertInstanceCommand.java | 11 +- .../api/ImportConvertedInstanceAnswer.java | 40 +++ .../api/ImportConvertedInstanceCommand.java | 63 ++++ .../LibvirtConvertInstanceCommandWrapper.java | 235 +------------- ...ImportConvertedInstanceCommandWrapper.java | 302 ++++++++++++++++++ ...virtConvertInstanceCommandWrapperTest.java | 118 ------- ...rtConvertedInstanceCommandWrapperTest.java | 245 ++++++++++++++ .../vm/UnmanagedVMsManagerImpl.java | 181 ++++++++--- .../vm/UnmanagedVMsManagerImplTest.java | 288 ++++++++++++++++- ui/public/locales/en.json | 6 +- .../views/tools/ImportUnmanagedInstance.vue | 65 +++- ui/src/views/tools/ManageInstances.vue | 3 +- 15 files changed, 1159 insertions(+), 428 deletions(-) create mode 100644 core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java create mode 100644 core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java create mode 100644 plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtImportConvertedInstanceCommandWrapper.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtImportConvertedInstanceCommandWrapperTest.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index d6099ac4717..b9a4938b5b2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -225,6 +225,7 @@ public class ApiConstants { public static final String ICMP_TYPE = "icmptype"; public static final String ID = "id"; public static final String IDS = "ids"; + public static final String IMPORT_INSTANCE_HOST_ID = "importinstancehostid"; public static final String INDEX = "index"; public static final String INSTANCES_DISKS_STATS_RETENTION_ENABLED = "instancesdisksstatsretentionenabled"; public static final String INSTANCES_DISKS_STATS_RETENTION_TIME = "instancesdisksstatsretentiontime"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 945f861cd3e..5c82e5bbd97 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -146,15 +146,19 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { private String clusterName; @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, - description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") + description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v conversion from VMware to KVM.") private Long convertInstanceHostId; + @Parameter(name = ApiConstants.IMPORT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, since = "4.19.2", + description = "(only for importing VMs from VMware to KVM) optional - the host to import the converted instance from VMware to KVM.") + private Long importInstanceHostId; + @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") private Long convertStoragePoolId; @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN, - description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to import VM file(s) to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.") + description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to export OVF from VMware to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.") private Boolean forceMsToImportVmFiles; ///////////////////////////////////////////////////// @@ -201,6 +205,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return convertInstanceHostId; } + public Long getImportInstanceHostId() { + return importInstanceHostId; + } + public Long getConvertStoragePoolId() { return convertStoragePoolId; } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java index 829888570a6..8092ab9b43f 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java @@ -16,25 +16,20 @@ // under the License. package com.cloud.agent.api; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; - public class ConvertInstanceAnswer extends Answer { + private String temporaryConvertUuid; + public ConvertInstanceAnswer() { super(); } - private UnmanagedInstanceTO convertedInstance; - public ConvertInstanceAnswer(Command command, boolean success, String details) { - super(command, success, details); - } - - public ConvertInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { + public ConvertInstanceAnswer(Command command, String temporaryConvertUuid) { super(command, true, ""); - this.convertedInstance = convertedInstance; + this.temporaryConvertUuid = temporaryConvertUuid; } - public UnmanagedInstanceTO getConvertedInstance() { - return convertedInstance; + public String getTemporaryConvertUuid() { + return temporaryConvertUuid; } } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index b8250903f85..f938d0ac55d 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -20,13 +20,10 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; -import java.util.List; - public class ConvertInstanceCommand extends Command { private RemoteInstanceTO sourceInstance; private Hypervisor.HypervisorType destinationHypervisorType; - private List destinationStoragePools; private DataStoreTO conversionTemporaryLocation; private String templateDirOnConversionLocation; private boolean checkConversionSupport; @@ -36,12 +33,10 @@ public class ConvertInstanceCommand extends Command { public ConvertInstanceCommand() { } - public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, - List destinationStoragePools, DataStoreTO conversionTemporaryLocation, + public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, DataStoreTO conversionTemporaryLocation, String templateDirOnConversionLocation, boolean checkConversionSupport, boolean exportOvfToConversionLocation) { this.sourceInstance = sourceInstance; this.destinationHypervisorType = destinationHypervisorType; - this.destinationStoragePools = destinationStoragePools; this.conversionTemporaryLocation = conversionTemporaryLocation; this.templateDirOnConversionLocation = templateDirOnConversionLocation; this.checkConversionSupport = checkConversionSupport; @@ -56,10 +51,6 @@ public class ConvertInstanceCommand extends Command { return destinationHypervisorType; } - public List getDestinationStoragePools() { - return destinationStoragePools; - } - public DataStoreTO getConversionTemporaryLocation() { return conversionTemporaryLocation; } diff --git a/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java new file mode 100644 index 00000000000..2a8f8704e3f --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import org.apache.cloudstack.vm.UnmanagedInstanceTO; + +public class ImportConvertedInstanceAnswer extends Answer { + + public ImportConvertedInstanceAnswer() { + super(); + } + private UnmanagedInstanceTO convertedInstance; + + public ImportConvertedInstanceAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public ImportConvertedInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { + super(command, true, ""); + this.convertedInstance = convertedInstance; + } + + public UnmanagedInstanceTO getConvertedInstance() { + return convertedInstance; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java new file mode 100644 index 00000000000..9d50e852ced --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.RemoteInstanceTO; + +import java.util.List; + +public class ImportConvertedInstanceCommand extends Command { + + private RemoteInstanceTO sourceInstance; + private List destinationStoragePools; + private DataStoreTO conversionTemporaryLocation; + private String temporaryConvertUuid; + + public ImportConvertedInstanceCommand() { + } + + public ImportConvertedInstanceCommand(RemoteInstanceTO sourceInstance, + List destinationStoragePools, + DataStoreTO conversionTemporaryLocation, String temporaryConvertUuid) { + this.sourceInstance = sourceInstance; + this.destinationStoragePools = destinationStoragePools; + this.conversionTemporaryLocation = conversionTemporaryLocation; + this.temporaryConvertUuid = temporaryConvertUuid; + } + + public RemoteInstanceTO getSourceInstance() { + return sourceInstance; + } + + public List getDestinationStoragePools() { + return destinationStoragePools; + } + + public DataStoreTO getConversionTemporaryLocation() { + return conversionTemporaryLocation; + } + + public String getTemporaryConvertUuid() { + return temporaryConvertUuid; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index f6f6ea1082d..e6654b1ffc5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -18,44 +18,29 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.net.URLEncoder; import java.nio.charset.Charset; -import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; +import com.cloud.agent.api.ConvertInstanceAnswer; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; -import com.cloud.agent.api.ConvertInstanceAnswer; import com.cloud.agent.api.ConvertInstanceCommand; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; -import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.storage.Storage; import com.cloud.utils.FileUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; @@ -73,7 +58,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper destinationStoragePools = cmd.getDestinationStoragePools(); DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation(); long timeout = (long) cmd.getWait() * 1000; @@ -81,7 +65,7 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper temporaryDisks = xmlParser == null ? - getTemporaryDisksWithPrefixFromTemporaryPool(temporaryStoragePool, temporaryConvertPath, temporaryConvertUuid) : - getTemporaryDisksFromParsedXml(temporaryStoragePool, xmlParser, convertedBasePath); - - List destinationDisks = moveTemporaryDisksToDestination(temporaryDisks, - destinationStoragePools, storagePoolMgr); - - cleanupDisksAndDomainFromTemporaryLocation(temporaryDisks, temporaryStoragePool, temporaryConvertUuid); - - UnmanagedInstanceTO convertedInstanceTO = getConvertedUnmanagedInstance(temporaryConvertUuid, - destinationDisks, xmlParser); - return new ConvertInstanceAnswer(cmd, convertedInstanceTO); + return new ConvertInstanceAnswer(cmd, temporaryConvertUuid); } catch (Exception e) { String error = String.format("Error converting instance %s from %s, due to: %s", sourceInstanceName, sourceHypervisorType, e.getMessage()); s_logger.error(error, e); - return new ConvertInstanceAnswer(cmd, false, error); + cleanupSecondaryStorage = true; + return new Answer(cmd, false, error); } finally { if (ovfExported && StringUtils.isNotBlank(ovfTemplateDirOnConversionLocation)) { String sourceOVFDir = String.format("%s/%s", temporaryConvertPath, ovfTemplateDirOnConversionLocation); s_logger.debug("Cleaning up exported OVA at dir " + sourceOVFDir); FileUtil.deletePath(sourceOVFDir); } - if (conversionTemporaryLocation instanceof NfsTO) { + if (cleanupSecondaryStorage && conversionTemporaryLocation instanceof NfsTO) { s_logger.debug("Cleaning up secondary storage temporary location"); storagePoolMgr.deleteStoragePool(temporaryStoragePool.getType(), temporaryStoragePool.getUuid()); } @@ -209,55 +184,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { - List disksDefs = xmlParser.getDisks(); - disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && - x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksDefs)) { - String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); - s_logger.error(err); - throw new CloudRuntimeException(err); - } - sanitizeDisksPath(disksDefs); - return getPhysicalDisksFromDefPaths(disksDefs, pool); - } - - private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { - List disks = new ArrayList<>(); - for (LibvirtVMDef.DiskDef diskDef : disksDefs) { - KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); - disks.add(physicalDisk); - } - return disks; - } - - protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { - String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); - s_logger.info(msg); - pool.refresh(); - List disksWithPrefix = pool.listPhysicalDisks() - .stream() - .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) - .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksWithPrefix)) { - msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); - s_logger.error(msg); - throw new CloudRuntimeException(msg); - } - return disksWithPrefix; - } - - private void cleanupDisksAndDomainFromTemporaryLocation(List disks, - KVMStoragePool temporaryStoragePool, - String temporaryConvertUuid) { - for (KVMPhysicalDisk disk : disks) { - s_logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); - temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); - } - s_logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); - FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml"); - } - protected void sanitizeDisksPath(List disks) { for (LibvirtVMDef.DiskDef disk : disks) { String[] diskPathParts = disk.getDiskPath().split("/"); @@ -266,114 +192,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper moveTemporaryDisksToDestination(List temporaryDisks, - List destinationStoragePools, - KVMStoragePoolManager storagePoolMgr) { - List targetDisks = new ArrayList<>(); - if (temporaryDisks.size() != destinationStoragePools.size()) { - String warn = String.format("Discrepancy between the converted instance disks (%s) " + - "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); - s_logger.warn(warn); - } - for (int i = 0; i < temporaryDisks.size(); i++) { - String poolPath = destinationStoragePools.get(i); - KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); - if (destinationPool == null) { - String err = String.format("Could not find a storage pool by URI: %s", poolPath); - s_logger.error(err); - continue; - } - if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) { - String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath); - s_logger.error(err); - continue; - } - KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); - if (s_logger.isDebugEnabled()) { - String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + - " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); - s_logger.debug(msg); - } - - String destinationName = UUID.randomUUID().toString(); - - KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); - targetDisks.add(destinationDisk); - } - return targetDisks; - } - - private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, - List vmDisks, - LibvirtDomainXMLParser xmlParser) { - UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); - instanceTO.setName(baseName); - instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); - instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); - return instanceTO; - } - - private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { - List nics = new ArrayList<>(); - if (xmlParser != null) { - List interfaces = xmlParser.getInterfaces(); - for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { - UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); - nic.setMacAddress(interfaceDef.getMacAddress()); - nic.setNicId(interfaceDef.getBrName()); - nic.setAdapterType(interfaceDef.getModel().toString()); - nics.add(nic); - } - } - return nics; - } - - protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { - List instanceDisks = new ArrayList<>(); - List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; - for (int i = 0; i< vmDisks.size(); i++) { - KVMPhysicalDisk physicalDisk = vmDisks.get(i); - KVMStoragePool storagePool = physicalDisk.getPool(); - UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); - disk.setPosition(i); - Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); - disk.setDatastoreHost(storagePoolHostAndPath.first()); - disk.setDatastorePath(storagePoolHostAndPath.second()); - disk.setDatastoreName(storagePool.getUuid()); - disk.setDatastoreType(storagePool.getType().name()); - disk.setCapacity(physicalDisk.getVirtualSize()); - disk.setFileBaseName(physicalDisk.getName()); - if (CollectionUtils.isNotEmpty(diskDefs)) { - LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); - disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } else { - // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver - disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } - instanceDisks.add(disk); - } - return instanceDisks; - } - - protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { - String sourceHostIp = null; - String sourcePath = null; - List commands = new ArrayList<>(); - commands.add(new String[]{Script.getExecutableAbsolutePath("mount")}); - commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()}); - String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second(); - s_logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint)); - if (StringUtils.isNotEmpty(storagePoolMountPoint)) { - String[] res = storagePoolMountPoint.strip().split(" "); - res = res[0].split(":"); - if (res.length > 1) { - sourceHostIp = res[0].strip(); - sourcePath = res[1].strip(); - } - } - return new Pair<>(sourceHostIp, sourcePath); - } - private boolean exportOVAFromVMOnVcenter(String vmExportUrl, String targetOvfDir, int noOfThreads, @@ -416,27 +234,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtImportConvertedInstanceCommandWrapper.class); + + @Override + public Answer execute(ImportConvertedInstanceCommand cmd, LibvirtComputingResource serverResource) { + RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); + Hypervisor.HypervisorType sourceHypervisorType = sourceInstance.getHypervisorType(); + String sourceInstanceName = sourceInstance.getInstanceName(); + List destinationStoragePools = cmd.getDestinationStoragePools(); + DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation(); + final String temporaryConvertUuid = cmd.getTemporaryConvertUuid(); + + final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + KVMStoragePool temporaryStoragePool = getTemporaryStoragePool(conversionTemporaryLocation, storagePoolMgr); + final String temporaryConvertPath = temporaryStoragePool.getLocalPath(); + + try { + String convertedBasePath = String.format("%s/%s", temporaryConvertPath, temporaryConvertUuid); + LibvirtDomainXMLParser xmlParser = parseMigratedVMXmlDomain(convertedBasePath); + + List temporaryDisks = xmlParser == null ? + getTemporaryDisksWithPrefixFromTemporaryPool(temporaryStoragePool, temporaryConvertPath, temporaryConvertUuid) : + getTemporaryDisksFromParsedXml(temporaryStoragePool, xmlParser, convertedBasePath); + + List destinationDisks = moveTemporaryDisksToDestination(temporaryDisks, + destinationStoragePools, storagePoolMgr); + + cleanupDisksAndDomainFromTemporaryLocation(temporaryDisks, temporaryStoragePool, temporaryConvertUuid); + + UnmanagedInstanceTO convertedInstanceTO = getConvertedUnmanagedInstance(temporaryConvertUuid, + destinationDisks, xmlParser); + return new ImportConvertedInstanceAnswer(cmd, convertedInstanceTO); + } catch (Exception e) { + String error = String.format("Error converting instance %s from %s, due to: %s", + sourceInstanceName, sourceHypervisorType, e.getMessage()); + s_logger.error(error, e); + return new ImportConvertedInstanceAnswer(cmd, false, error); + } finally { + if (conversionTemporaryLocation instanceof NfsTO) { + s_logger.debug("Cleaning up secondary storage temporary location"); + storagePoolMgr.deleteStoragePool(temporaryStoragePool.getType(), temporaryStoragePool.getUuid()); + } + } + } + + protected KVMStoragePool getTemporaryStoragePool(DataStoreTO conversionTemporaryLocation, KVMStoragePoolManager storagePoolMgr) { + if (conversionTemporaryLocation instanceof NfsTO) { + NfsTO nfsTO = (NfsTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePoolByURI(nfsTO.getUrl()); + } else { + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + } + } + + protected List getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { + List disksDefs = xmlParser.getDisks(); + disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && + x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksDefs)) { + String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + sanitizeDisksPath(disksDefs); + return getPhysicalDisksFromDefPaths(disksDefs, pool); + } + + private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { + List disks = new ArrayList<>(); + for (LibvirtVMDef.DiskDef diskDef : disksDefs) { + KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); + disks.add(physicalDisk); + } + return disks; + } + + protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { + String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); + s_logger.info(msg); + pool.refresh(); + List disksWithPrefix = pool.listPhysicalDisks() + .stream() + .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksWithPrefix)) { + msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + return disksWithPrefix; + } + + private void cleanupDisksAndDomainFromTemporaryLocation(List disks, + KVMStoragePool temporaryStoragePool, + String temporaryConvertUuid) { + for (KVMPhysicalDisk disk : disks) { + s_logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); + temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); + } + s_logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); + FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml"); + } + + protected void sanitizeDisksPath(List disks) { + for (LibvirtVMDef.DiskDef disk : disks) { + String[] diskPathParts = disk.getDiskPath().split("/"); + String relativePath = diskPathParts[diskPathParts.length - 1]; + disk.setDiskPath(relativePath); + } + } + + protected List moveTemporaryDisksToDestination(List temporaryDisks, + List destinationStoragePools, + KVMStoragePoolManager storagePoolMgr) { + List targetDisks = new ArrayList<>(); + if (temporaryDisks.size() != destinationStoragePools.size()) { + String warn = String.format("Discrepancy between the converted instance disks (%s) " + + "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); + s_logger.warn(warn); + } + for (int i = 0; i < temporaryDisks.size(); i++) { + String poolPath = destinationStoragePools.get(i); + KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); + if (destinationPool == null) { + String err = String.format("Could not find a storage pool by URI: %s", poolPath); + s_logger.error(err); + continue; + } + if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) { + String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath); + s_logger.error(err); + continue; + } + KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); + if (s_logger.isDebugEnabled()) { + String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + + " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); + s_logger.debug(msg); + } + + String destinationName = UUID.randomUUID().toString(); + + KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); + targetDisks.add(destinationDisk); + } + return targetDisks; + } + + private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, + List vmDisks, + LibvirtDomainXMLParser xmlParser) { + UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); + instanceTO.setName(baseName); + instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); + instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); + return instanceTO; + } + + private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { + List nics = new ArrayList<>(); + if (xmlParser != null) { + List interfaces = xmlParser.getInterfaces(); + for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { + UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); + nic.setMacAddress(interfaceDef.getMacAddress()); + nic.setNicId(interfaceDef.getBrName()); + nic.setAdapterType(interfaceDef.getModel().toString()); + nics.add(nic); + } + } + return nics; + } + + protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { + List instanceDisks = new ArrayList<>(); + List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; + for (int i = 0; i< vmDisks.size(); i++) { + KVMPhysicalDisk physicalDisk = vmDisks.get(i); + KVMStoragePool storagePool = physicalDisk.getPool(); + UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); + disk.setPosition(i); + Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); + disk.setDatastoreHost(storagePoolHostAndPath.first()); + disk.setDatastorePath(storagePoolHostAndPath.second()); + disk.setDatastoreName(storagePool.getUuid()); + disk.setDatastoreType(storagePool.getType().name()); + disk.setCapacity(physicalDisk.getVirtualSize()); + disk.setFileBaseName(physicalDisk.getName()); + if (CollectionUtils.isNotEmpty(diskDefs)) { + LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); + disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } else { + // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver + disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } + instanceDisks.add(disk); + } + return instanceDisks; + } + + protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { + String sourceHostIp = null; + String sourcePath = null; + List commands = new ArrayList<>(); + commands.add(new String[]{Script.getExecutableAbsolutePath("mount")}); + commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()}); + String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second(); + s_logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint)); + if (StringUtils.isNotEmpty(storagePoolMountPoint)) { + String[] res = storagePoolMountPoint.strip().split(" "); + res = res[0].split(":"); + if (res.length > 1) { + sourceHostIp = res[0].strip(); + sourcePath = res[1].strip(); + } + } + return new Pair<>(sourceHostIp, sourcePath); + } + + protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException { + String xmlPath = String.format("%s.xml", installPath); + if (!new File(xmlPath).exists()) { + String err = String.format("Conversion failed. Unable to find the converted XML domain, expected %s", xmlPath); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + InputStream is = new BufferedInputStream(new FileInputStream(xmlPath)); + String xml = IOUtils.toString(is, Charset.defaultCharset()); + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + try { + parser.parseDomainXML(xml); + return parser; + } catch (RuntimeException e) { + String err = String.format("Error parsing the converted instance XML domain at %s: %s", xmlPath, e.getMessage()); + s_logger.error(err, e); + s_logger.debug(xml); + return null; + } + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java index 1cc2a60e380..b369cf25f3d 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -18,12 +18,10 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; -import java.util.Arrays; import java.util.List; import java.util.UUID; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -41,13 +39,10 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; -import com.cloud.storage.Storage; -import com.cloud.utils.Pair; import com.cloud.utils.script.Script; @RunWith(MockitoJUnitRunner.class) @@ -80,7 +75,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { Mockito.when(storagePoolManager.getStoragePoolByURI(secondaryPoolUrl)).thenReturn(temporaryPool); KVMPhysicalDisk physicalDisk1 = Mockito.mock(KVMPhysicalDisk.class); KVMPhysicalDisk physicalDisk2 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2)); } @Test @@ -107,51 +101,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { Assert.assertNotNull(temporaryStoragePool); } - @Test - public void testGetTemporaryDisksWithPrefixFromTemporaryPool() { - String convertPath = "/xyz"; - String convertPrefix = UUID.randomUUID().toString(); - KVMPhysicalDisk physicalDisk1 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(physicalDisk1.getName()).thenReturn("disk1"); - KVMPhysicalDisk physicalDisk2 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(physicalDisk2.getName()).thenReturn("disk2"); - - KVMPhysicalDisk convertedDisk1 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedDisk1.getName()).thenReturn(String.format("%s-sda", convertPrefix)); - KVMPhysicalDisk convertedDisk2 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedDisk2.getName()).thenReturn(String.format("%s-sdb", convertPrefix)); - KVMPhysicalDisk convertedXml = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedXml.getName()).thenReturn(String.format("%s.xml", convertPrefix)); - Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2, - convertedDisk1, convertedDisk2, convertedXml)); - - List convertedDisks = convertInstanceCommandWrapper.getTemporaryDisksWithPrefixFromTemporaryPool(temporaryPool, convertPath, convertPrefix); - Assert.assertEquals(2, convertedDisks.size()); - } - - @Test - public void testGetTemporaryDisksFromParsedXml() { - String relativePath = UUID.randomUUID().toString(); - String fullPath = String.format("/mnt/xyz/%s", relativePath); - - LibvirtVMDef.DiskDef diskDef = new LibvirtVMDef.DiskDef(); - LibvirtVMDef.DiskDef.DiskBus bus = LibvirtVMDef.DiskDef.DiskBus.VIRTIO; - LibvirtVMDef.DiskDef.DiskFmtType type = LibvirtVMDef.DiskDef.DiskFmtType.QCOW2; - diskDef.defFileBasedDisk(fullPath, "test", bus, type); - - LibvirtDomainXMLParser parser = Mockito.mock(LibvirtDomainXMLParser.class); - Mockito.when(parser.getDisks()).thenReturn(List.of(diskDef)); - - KVMPhysicalDisk convertedDisk1 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedDisk1.getName()).thenReturn("disk1"); - Mockito.when(temporaryPool.getPhysicalDisk(relativePath)).thenReturn(convertedDisk1); - - List disks = convertInstanceCommandWrapper.getTemporaryDisksFromParsedXml(temporaryPool, parser, ""); - Mockito.verify(convertInstanceCommandWrapper).sanitizeDisksPath(List.of(diskDef)); - Assert.assertEquals(1, disks.size()); - Assert.assertEquals("disk1", disks.get(0).getName()); - } - @Test public void testSanitizeDisksPath() { String relativePath = UUID.randomUUID().toString(); @@ -165,73 +114,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { Assert.assertEquals(relativePath, diskDef.getDiskPath()); } - @Test - public void testMoveTemporaryDisksToDestination() { - KVMPhysicalDisk sourceDisk = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(sourceDisk.getPool()).thenReturn(temporaryPool); - List disks = List.of(sourceDisk); - String destinationPoolUuid = UUID.randomUUID().toString(); - List destinationPools = List.of(destinationPoolUuid); - - KVMPhysicalDisk destDisk = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(destDisk.getPath()).thenReturn("xyz"); - Mockito.when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, destinationPoolUuid)) - .thenReturn(destinationPool); - Mockito.when(destinationPool.getType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); - Mockito.when(storagePoolManager.copyPhysicalDisk(Mockito.eq(sourceDisk), Mockito.anyString(), Mockito.eq(destinationPool), Mockito.anyInt())) - .thenReturn(destDisk); - - List movedDisks = convertInstanceCommandWrapper.moveTemporaryDisksToDestination(disks, destinationPools, storagePoolManager); - Assert.assertEquals(1, movedDisks.size()); - Assert.assertEquals("xyz", movedDisks.get(0).getPath()); - } - - @Test - public void testGetUnmanagedInstanceDisks() { - try (MockedStatic