From cbd2b5a022476e8b85f8ce745d01db5370fb185f Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 19 Jun 2025 04:03:58 -0400 Subject: [PATCH 01/83] Add check for ldap truststore password (#11055) --- .../cloudstack/ldap/LdapContextFactory.java | 34 +++++++++++++++++-- .../cloudstack/ldap/LdapManagerImpl.java | 5 +++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java index 0161adf9fda..ee48e8cc027 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.ldap; +import java.io.FileInputStream; import java.io.IOException; import java.util.Hashtable; @@ -24,6 +25,7 @@ import javax.naming.Context; import javax.naming.NamingException; import javax.naming.ldap.InitialLdapContext; import javax.naming.ldap.LdapContext; +import java.security.KeyStore; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -72,8 +74,36 @@ public class LdapContextFactory { if (sslStatus) { s_logger.info("LDAP SSL enabled."); environment.put(Context.SECURITY_PROTOCOL, "ssl"); - System.setProperty("javax.net.ssl.trustStore", _ldapConfiguration.getTrustStore(domainId)); - System.setProperty("javax.net.ssl.trustStorePassword", _ldapConfiguration.getTrustStorePassword(domainId)); + String trustStore = _ldapConfiguration.getTrustStore(domainId); + String trustStorePassword = _ldapConfiguration.getTrustStorePassword(domainId); + + if (!validateTrustStore(trustStore, trustStorePassword)) { + throw new RuntimeException("Invalid truststore or truststore password"); + } + + System.setProperty("javax.net.ssl.trustStore", trustStore); + System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); + } + } + + private boolean validateTrustStore(String trustStore, String trustStorePassword) { + if (trustStore == null) { + return true; + } + + if (trustStorePassword == null) { + return false; + } + + try { + KeyStore.getInstance("JKS").load( + new FileInputStream(trustStore), + trustStorePassword.toCharArray() + ); + return true; + } catch (Exception e) { + s_logger.warn("Failed to validate truststore: " + e.getMessage()); + return false; } } diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java index 6ed79a0c69f..352e439b50c 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java @@ -186,6 +186,11 @@ public class LdapManagerImpl extends ComponentLifecycleBase implements LdapManag } catch (NamingException | IOException e) { LOGGER.debug("NamingException while doing an LDAP bind", e); throw new InvalidParameterValueException("Unable to bind to the given LDAP server"); + } catch (RuntimeException e) { + if (e.getMessage().contains("Invalid truststore")) { + throw new InvalidParameterValueException("Invalid truststore or truststore password"); + } + throw e; } finally { closeContext(context); } From 75147b781163958200287aa9979e60bf602b2f89 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Mon, 23 Jun 2025 07:49:51 -0300 Subject: [PATCH 02/83] [Vmware to KVM Migration] Display virt-v2v and ovftool versions for supported hosts for migration (#11019) * [Vmware to KVM Migration] Display virt-v2v and ovftool versions for supported hosts for migration * Fix UI display * Address review comments * Fix ovftool and version display - also display versions on host details view --- api/src/main/java/com/cloud/host/Host.java | 9 ++++-- .../cloud/agent/manager/AgentManagerImpl.java | 17 ++++++++++- .../resource/LibvirtComputingResource.java | 29 +++++++++++++++++-- .../wrapper/LibvirtReadyCommandWrapper.java | 8 +++++ ui/public/locales/en.json | 2 ++ ui/src/views/infra/HostInfo.vue | 16 ++++++++++ .../views/tools/ImportUnmanagedInstance.vue | 6 ++++ 7 files changed, 81 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 56b4ed75a31..a3b6ccadc01 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -53,9 +53,12 @@ public interface Host extends StateObject, Identity, Partition, HAResour return strs; } } - public static final String HOST_UEFI_ENABLE = "host.uefi.enable"; - public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; - public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + + String HOST_UEFI_ENABLE = "host.uefi.enable"; + String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; + String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + String HOST_OVFTOOL_VERSION = "host.ovftool.version"; + String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; /** * @return name of the machine. diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 9137d1c1dff..aff528efede 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -54,6 +54,7 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.ThreadContext; @@ -703,11 +704,25 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl Map detailsMap = readyAnswer.getDetailsMap(); if (detailsMap != null) { String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); + String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); + String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (uefiEnabled != null) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion)) { _hostDao.loadDetails(host); + boolean updateNeeded = false; if (!uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { host.getDetails().put(Host.HOST_UEFI_ENABLE, uefiEnabled); + updateNeeded = true; + } + if (StringUtils.isNotBlank(virtv2vVersion) && !virtv2vVersion.equals(host.getDetails().get(Host.HOST_VIRTV2V_VERSION))) { + host.getDetails().put(Host.HOST_VIRTV2V_VERSION, virtv2vVersion); + updateNeeded = true; + } + if (StringUtils.isNotBlank(ovftoolVersion) && !ovftoolVersion.equals(host.getDetails().get(Host.HOST_OVFTOOL_VERSION))) { + host.getDetails().put(Host.HOST_OVFTOOL_VERSION, ovftoolVersion); + updateNeeded = true; + } + if (updateNeeded) { _hostDao.saveDetails(host); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 13518de5cb3..7fb3839860f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -17,6 +17,8 @@ package com.cloud.hypervisor.kvm.resource; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; +import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_VIRTV2V_VERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import java.io.BufferedReader; @@ -3766,7 +3768,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cmd.setIqn(getIqn()); cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption())); cmd.setHostTags(getHostTags()); - cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(hostSupportsInstanceConversion())); + boolean instanceConversionSupported = hostSupportsInstanceConversion(); + cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); + if (instanceConversionSupported) { + cmd.getHostDetails().put(HOST_VIRTV2V_VERSION, getHostVirtV2vVersion()); + } + if (hostSupportsOvfExport()) { + cmd.getHostDetails().put(HOST_OVFTOOL_VERSION, getHostOvfToolVersion()); + } HealthCheckResult healthCheckResult = getHostHealthCheckResult(); if (healthCheckResult != HealthCheckResult.IGNORE) { cmd.setHostHealthCheckResult(healthCheckResult == HealthCheckResult.SUCCESS); @@ -5368,8 +5377,24 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + public String getHostVirtV2vVersion() { + if (!hostSupportsInstanceConversion()) { + return ""; + } + String cmd = String.format("%s | awk '{print $2}'", INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD); + String version = Script.runSimpleBashScript(cmd); + return StringUtils.isNotBlank(version) ? version.split(",")[0] : ""; + } + + public String getHostOvfToolVersion() { + if (!hostSupportsOvfExport()) { + return ""; + } + return Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + } + public boolean ovfExportToolSupportsParallelThreads() { - String ovfExportToolVersion = Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + String ovfExportToolVersion = getHostOvfToolVersion(); if (StringUtils.isBlank(ovfExportToolVersion)) { return false; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index 8f23e79e4a3..485254c6bb9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -47,6 +47,14 @@ public final class LibvirtReadyCommandWrapper extends CommandWrapper + +
+ {{ $t('label.host.virtv2v.version') }} +
+ {{ host.details['host.virtv2v.version'] }} +
+
+ + +
+ {{ $t('label.host.ovftool.version') }} +
+ {{ host.details['host.ovftool.version'] }} +
+
+
{{ $t('label.hosttags') }} diff --git a/ui/src/views/tools/ImportUnmanagedInstance.vue b/ui/src/views/tools/ImportUnmanagedInstance.vue index 50a0d631808..95c14f141d2 100644 --- a/ui/src/views/tools/ImportUnmanagedInstance.vue +++ b/ui/src/views/tools/ImportUnmanagedInstance.vue @@ -944,6 +944,12 @@ export default { } else { host.name = host.name + ' (' + this.$t('label.not.supported') + ')' } + if (host.details['host.virtv2v.version']) { + host.name = host.name + ' (virt-v2v=' + host.details['host.virtv2v.version'] + ')' + } + if (host.details['host.ovftool.version']) { + host.name = host.name + ' (ovftool=' + host.details['host.ovftool.version'] + ')' + } }) }) }, From ba0204f8edd6140d893e92bf33522f9a50637d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jandre?= <48719461+JoaoJandre@users.noreply.github.com> Date: Tue, 24 Jun 2025 09:23:57 -0300 Subject: [PATCH 03/83] Block volume shrink on Xen (#11004) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: João Jandre --- .../wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java | 7 +++++-- .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 3 +++ ui/src/views/storage/ChangeOfferingForVolume.vue | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java index 2ddf1bd1851..abf46b2771c 100755 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java @@ -49,9 +49,12 @@ public final class CitrixResizeVolumeCommandWrapper extends CommandWrapper= newSize) { - logger.info("No need to resize volume: " + volId +", current size " + toHumanReadableSize(command.getCurrentSize()) + " is same as new size " + toHumanReadableSize(newSize)); + if (command.getCurrentSize() == newSize) { + logger.info("No need to resize volume [{}], current size [{}] is same as new size [{}].", volId, toHumanReadableSize(command.getCurrentSize()), toHumanReadableSize(newSize)); return new ResizeVolumeAnswer(command, true, "success", newSize); + } else if (command.getCurrentSize() > newSize) { + logger.error("XenServer does not support volume shrink. Volume [{}] current size [{}] is smaller than new size [{}]", volId, toHumanReadableSize(command.getCurrentSize()), toHumanReadableSize(newSize)); + return new ResizeVolumeAnswer(command, false, "operation not supported"); } if (command.isManaged()) { resizeSr(conn, command); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 6a2aa09c38a..84e4bbf258f 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2429,6 +2429,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Going from existing size of " + currentSize + " to size of " + newSize + " would shrink the volume." + "Need to sign off by supplying the shrinkok parameter with value of true."); } + if (ApiDBUtils.getHypervisorTypeFromFormat(volume.getDataCenterId(), volume.getFormat()) == HypervisorType.XenServer) { + throw new InvalidParameterValueException("Shrink volume is not supported for the XenServer hypervisor."); + } } } /* Check resource limit for this account */ diff --git a/ui/src/views/storage/ChangeOfferingForVolume.vue b/ui/src/views/storage/ChangeOfferingForVolume.vue index 5ab80390999..dedc2e8dc04 100644 --- a/ui/src/views/storage/ChangeOfferingForVolume.vue +++ b/ui/src/views/storage/ChangeOfferingForVolume.vue @@ -90,7 +90,7 @@ :checked="autoMigrate" @change="val => { autoMigrate = val }"/> - + Date: Wed, 2 Jul 2025 15:45:32 +0530 Subject: [PATCH 04/83] Do not rely on Memory engine even transiently in DB setup scripts (#11106) It is not safe for use with replication, and is straight up incompatible with highly-available active-active type MySQL distributions such as Galera Co-authored-by: Tristan Deloche --- setup/db/create-schema.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 3f14fccd010..10141185eb4 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -398,7 +398,7 @@ CREATE TABLE `cloud`.`op_lock` ( `waiters` int NOT NULL DEFAULT 0 COMMENT 'How many have the thread acquired this lock (reentrant)', PRIMARY KEY (`key`), INDEX `i_op_lock__mac_ip_thread`(`mac`, `ip`, `thread`) -) ENGINE=Memory DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`configuration` ( `category` varchar(255) NOT NULL DEFAULT 'Advanced', @@ -1793,7 +1793,7 @@ CREATE TABLE `cloud`.`op_nwgrp_work` ( INDEX `i_op_nwgrp_work__taken`(`taken`), INDEX `i_op_nwgrp_work__step`(`step`), INDEX `i_op_nwgrp_work__seq_no`(`seq_no`) -) ENGINE=MEMORY DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`op_vm_ruleset_log` ( `id` bigint unsigned UNIQUE NOT NULL AUTO_INCREMENT COMMENT 'id', From 3b54194aef18c44a37489614ce3391eceab55b2f Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Thu, 3 Jul 2025 13:15:35 +0530 Subject: [PATCH 05/83] Correct quota type indexes (#11085) --- ui/src/utils/quota.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/utils/quota.js b/ui/src/utils/quota.js index 485c99131d2..b8adbb93518 100644 --- a/ui/src/utils/quota.js +++ b/ui/src/utils/quota.js @@ -107,7 +107,7 @@ export const QUOTA_TYPES = [ }, { id: 29, - type: 'VPC' + type: 'BUCKET' }, { id: 30, @@ -115,7 +115,7 @@ export const QUOTA_TYPES = [ }, { id: 31, - type: 'BACKUP_OBJECT' + type: 'VPC' } ] From c24e4eea855bd21e72ce7aa182fada2671dbe857 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 3 Jul 2025 13:20:36 +0530 Subject: [PATCH 06/83] server: fix orphan db transaction issue (#11095) Signed-off-by: Abhishek Kumar --- .../com/cloud/alert/AlertManagerImpl.java | 32 ++++++++++++----- .../com/cloud/storage/StorageManagerImpl.java | 36 +++++++++++-------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index f35a0664a85..377b2134d78 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -89,6 +89,10 @@ import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; + import org.jetbrains.annotations.Nullable; public class AlertManagerImpl extends ManagerBase implements AlertManager, Configurable { @@ -290,8 +294,13 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size()))); for (Long hostId : hostIds) { futures.put(hostId, executorService.submit(() -> { - final HostVO host = hostDao.findById(hostId); - _capacityMgr.updateCapacityForHost(host); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final HostVO host = hostDao.findById(hostId); + _capacityMgr.updateCapacityForHost(host); + } + }); return null; })); } @@ -316,13 +325,18 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size()))); for (Long poolId: storagePoolIds) { futures.put(poolId, executorService.submit(() -> { - final StoragePoolVO pool = _storagePoolDao.findById(poolId); - long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null); - if (pool.isShared()) { - _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk); - } else { - _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk); - } + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final StoragePoolVO pool = _storagePoolDao.findById(poolId); + long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null); + if (pool.isShared()) { + _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk); + } else { + _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk); + } + } + }); return null; })); } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index f9199b94288..90113f66aaf 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -257,6 +257,7 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -1591,22 +1592,27 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (exceptionOccurred.get()) { return null; } - HostVO host = _hostDao.findById(hostId); - try { - connectHostToSharedPool(host, primaryStore.getId()); - poolHostIds.add(hostId); - } catch (Exception e) { - if (handleExceptionsPartially && e.getCause() instanceof StorageConflictException) { - exceptionOccurred.set(true); - throw e; + Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) throws Exception { + HostVO host = _hostDao.findById(hostId); + try { + connectHostToSharedPool(host, primaryStore.getId()); + poolHostIds.add(hostId); + } catch (Exception e) { + if (handleExceptionsPartially && e.getCause() instanceof StorageConflictException) { + exceptionOccurred.set(true); + throw e; + } + logger.warn("Unable to establish a connection between {} and {}", host, primaryStore, e); + String reason = getStoragePoolMountFailureReason(e.getMessage()); + if (handleExceptionsPartially && reason != null) { + exceptionOccurred.set(true); + throw new CloudRuntimeException(reason); + } + } } - logger.warn("Unable to establish a connection between {} and {}", host, primaryStore, e); - String reason = getStoragePoolMountFailureReason(e.getMessage()); - if (handleExceptionsPartially && reason != null) { - exceptionOccurred.set(true); - throw new CloudRuntimeException(reason); - } - } + }); return null; })); } From 1a251c8b7899f5cee9d586047aaa469d9bec4ea6 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Thu, 3 Jul 2025 12:55:21 +0200 Subject: [PATCH 07/83] merge forward fix --- .../org/apache/cloudstack/ldap/LdapContextFactory.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java index 4199b0757d8..1c759c6a45e 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java @@ -54,14 +54,14 @@ public class LdapContextFactory { return createInitialDirContext(bindPrincipal, bindPassword, providerUrl, true, domainId); } - private LdapContext createInitialDirContext(final String principal, final String password, final boolean isSystemContext, Long domainId) throws NamingException, IOException { + private LdapContext createInitialDirContext(final String principal, final String password, final boolean isSystemContext, Long domainId) throws NamingException { return createInitialDirContext(principal, password, null, isSystemContext, domainId); } private LdapContext createInitialDirContext(final String principal, final String password, final String providerUrl, final boolean isSystemContext, Long domainId) - throws NamingException, IOException { + throws NamingException { Hashtable environment = getEnvironment(principal, password, providerUrl, isSystemContext, domainId); - logger.debug("initializing ldap with provider url: " + environment.get(Context.PROVIDER_URL)); + logger.debug("initializing ldap with provider url: {}", environment.get(Context.PROVIDER_URL)); return new InitialLdapContext(environment, null); } @@ -103,7 +103,7 @@ public class LdapContextFactory { ); return true; } catch (Exception e) { - s_logger.warn("Failed to validate truststore: " + e.getMessage()); + logger.warn("Failed to validate truststore: {}", e.getMessage()); return false; } } From ed7bd5e5804c2d834b061f64bfd2d711e1ae5715 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 3 Jul 2025 19:00:42 +0530 Subject: [PATCH 08/83] ui: fix handler for deploy button menu (#11116) Signed-off-by: Abhishek Kumar --- ui/src/views/compute/DeployVM.vue | 4 +- ui/src/views/compute/DeployVnfAppliance.vue | 4 +- ui/src/views/compute/wizard/DeployButtons.vue | 47 +++++++++---------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index 73ea3fcc59d..d61c9e08ea9 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -845,7 +845,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" />
@@ -860,7 +860,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" /> diff --git a/ui/src/views/compute/DeployVnfAppliance.vue b/ui/src/views/compute/DeployVnfAppliance.vue index 49a2a9ef8f3..b7e797ee57c 100644 --- a/ui/src/views/compute/DeployVnfAppliance.vue +++ b/ui/src/views/compute/DeployVnfAppliance.vue @@ -825,7 +825,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" /> @@ -840,7 +840,7 @@ :deployButtonMenuOptions="deployMenuOptions" @handle-cancel="() => $router.back()" @handle-deploy="handleSubmit" - @handle-deploy-menu="handleSubmitAndStay" /> + @handle-deploy-menu="(index, e) => handleSubmitAndStay(e)" /> diff --git a/ui/src/views/compute/wizard/DeployButtons.vue b/ui/src/views/compute/wizard/DeployButtons.vue index 2bd2408e4ed..6fc9aee096d 100644 --- a/ui/src/views/compute/wizard/DeployButtons.vue +++ b/ui/src/views/compute/wizard/DeployButtons.vue @@ -86,7 +86,7 @@ export default { this.$emit('handle-deploy', e) }, handleMenu (e) { - this.$emit('handle-deploy-menu', e.key - 1) + this.$emit('handle-deploy-menu', e.key - 1, e) } } } @@ -94,37 +94,36 @@ export default { From 80f46ad55d63bea88da0d1d38b00b045499aa105 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 4 Jul 2025 13:54:54 +0530 Subject: [PATCH 09/83] [VMware to KVM Migration] Fix for converted instance npe issue when source vmware instance ovf is exported from management server (#11003) --- .../com/cloud/storage/VolumeApiService.java | 7 + .../agent/api/ConvertInstanceAnswer.java | 16 -- .../agent/api/ConvertInstanceCommand.java | 11 +- .../LibvirtConvertInstanceCommandWrapper.java | 205 +----------------- ...virtConvertInstanceCommandWrapperTest.java | 70 ------ .../vm/UnmanagedVMsManagerImpl.java | 59 +---- .../vm/UnmanagedVMsManagerImplTest.java | 2 - 7 files changed, 20 insertions(+), 350 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index bb69b5b6650..4182728c204 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -171,6 +171,13 @@ public interface VolumeApiService { * */ boolean doesStoragePoolSupportDiskOffering(StoragePool destPool, DiskOffering diskOffering); + + /** + * Checks if the storage pool supports the required disk offering tags + * destPool the storage pool to check the disk offering tags + * diskOfferingTags the tags that should be supported + * return whether the tags are supported in the storage pool + */ boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags); Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge); diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java index 174348f4f18..8092ab9b43f 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java @@ -16,8 +16,6 @@ // under the License. package com.cloud.agent.api; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; - public class ConvertInstanceAnswer extends Answer { private String temporaryConvertUuid; @@ -25,16 +23,6 @@ public class ConvertInstanceAnswer extends Answer { public ConvertInstanceAnswer() { super(); } - private UnmanagedInstanceTO convertedInstance; - - public ConvertInstanceAnswer(Command command, boolean success, String details) { - super(command, success, details); - } - - public ConvertInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { - super(command, true, ""); - this.convertedInstance = convertedInstance; - } public ConvertInstanceAnswer(Command command, String temporaryConvertUuid) { super(command, true, ""); @@ -44,8 +32,4 @@ public class ConvertInstanceAnswer extends Answer { public String getTemporaryConvertUuid() { return temporaryConvertUuid; } - - public UnmanagedInstanceTO getConvertedInstance() { - return convertedInstance; - } } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index b8250903f85..f938d0ac55d 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -20,13 +20,10 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; -import java.util.List; - public class ConvertInstanceCommand extends Command { private RemoteInstanceTO sourceInstance; private Hypervisor.HypervisorType destinationHypervisorType; - private List destinationStoragePools; private DataStoreTO conversionTemporaryLocation; private String templateDirOnConversionLocation; private boolean checkConversionSupport; @@ -36,12 +33,10 @@ public class ConvertInstanceCommand extends Command { public ConvertInstanceCommand() { } - public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, - List destinationStoragePools, DataStoreTO conversionTemporaryLocation, + public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, DataStoreTO conversionTemporaryLocation, String templateDirOnConversionLocation, boolean checkConversionSupport, boolean exportOvfToConversionLocation) { this.sourceInstance = sourceInstance; this.destinationHypervisorType = destinationHypervisorType; - this.destinationStoragePools = destinationStoragePools; this.conversionTemporaryLocation = conversionTemporaryLocation; this.templateDirOnConversionLocation = templateDirOnConversionLocation; this.checkConversionSupport = checkConversionSupport; @@ -56,10 +51,6 @@ public class ConvertInstanceCommand extends Command { return destinationHypervisorType; } - public List getDestinationStoragePools() { - return destinationStoragePools; - } - public DataStoreTO getConversionTemporaryLocation() { return conversionTemporaryLocation; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index a11730a1240..e79a8da9dda 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -18,22 +18,12 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.net.URLEncoder; import java.nio.charset.Charset; -import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.agent.api.Answer; @@ -44,17 +34,12 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; -import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.storage.Storage; import com.cloud.utils.FileUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; @@ -77,7 +62,7 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { - List disksDefs = xmlParser.getDisks(); - disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && - x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksDefs)) { - String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); - logger.error(err); - throw new CloudRuntimeException(err); - } - sanitizeDisksPath(disksDefs); - return getPhysicalDisksFromDefPaths(disksDefs, pool); - } - - private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { - List disks = new ArrayList<>(); - for (LibvirtVMDef.DiskDef diskDef : disksDefs) { - KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); - disks.add(physicalDisk); - } - return disks; - } - - protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { - String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); - logger.info(msg); - pool.refresh(); - List disksWithPrefix = pool.listPhysicalDisks() - .stream() - .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) - .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksWithPrefix)) { - msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); - logger.error(msg); - throw new CloudRuntimeException(msg); - } - return disksWithPrefix; - } - - private void cleanupDisksAndDomainFromTemporaryLocation(List disks, - KVMStoragePool temporaryStoragePool, - String temporaryConvertUuid) { - for (KVMPhysicalDisk disk : disks) { - logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); - temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); - } - logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); - FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml"); - } - protected void sanitizeDisksPath(List disks) { for (LibvirtVMDef.DiskDef disk : disks) { String[] diskPathParts = disk.getDiskPath().split("/"); @@ -262,114 +198,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper moveTemporaryDisksToDestination(List temporaryDisks, - List destinationStoragePools, - KVMStoragePoolManager storagePoolMgr) { - List targetDisks = new ArrayList<>(); - if (temporaryDisks.size() != destinationStoragePools.size()) { - String warn = String.format("Discrepancy between the converted instance disks (%s) " + - "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); - logger.warn(warn); - } - for (int i = 0; i < temporaryDisks.size(); i++) { - String poolPath = destinationStoragePools.get(i); - KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); - if (destinationPool == null) { - String err = String.format("Could not find a storage pool by URI: %s", poolPath); - logger.error(err); - continue; - } - if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) { - String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath); - logger.error(err); - continue; - } - KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); - if (logger.isDebugEnabled()) { - String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + - " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); - logger.debug(msg); - } - - String destinationName = UUID.randomUUID().toString(); - - KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); - targetDisks.add(destinationDisk); - } - return targetDisks; - } - - private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, - List vmDisks, - LibvirtDomainXMLParser xmlParser) { - UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); - instanceTO.setName(baseName); - instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); - instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); - return instanceTO; - } - - private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { - List nics = new ArrayList<>(); - if (xmlParser != null) { - List interfaces = xmlParser.getInterfaces(); - for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { - UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); - nic.setMacAddress(interfaceDef.getMacAddress()); - nic.setNicId(interfaceDef.getBrName()); - nic.setAdapterType(interfaceDef.getModel().toString()); - nics.add(nic); - } - } - return nics; - } - - protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { - List instanceDisks = new ArrayList<>(); - List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; - for (int i = 0; i< vmDisks.size(); i++) { - KVMPhysicalDisk physicalDisk = vmDisks.get(i); - KVMStoragePool storagePool = physicalDisk.getPool(); - UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); - disk.setPosition(i); - Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); - disk.setDatastoreHost(storagePoolHostAndPath.first()); - disk.setDatastorePath(storagePoolHostAndPath.second()); - disk.setDatastoreName(storagePool.getUuid()); - disk.setDatastoreType(storagePool.getType().name()); - disk.setCapacity(physicalDisk.getVirtualSize()); - disk.setFileBaseName(physicalDisk.getName()); - if (CollectionUtils.isNotEmpty(diskDefs)) { - LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); - disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } else { - // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver - disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } - instanceDisks.add(disk); - } - return instanceDisks; - } - - protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { - String sourceHostIp = null; - String sourcePath = null; - List commands = new ArrayList<>(); - commands.add(new String[]{Script.getExecutableAbsolutePath("mount")}); - commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()}); - String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second(); - logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint)); - if (StringUtils.isNotEmpty(storagePoolMountPoint)) { - String[] res = storagePoolMountPoint.strip().split(" "); - res = res[0].split(":"); - if (res.length > 1) { - sourceHostIp = res[0].strip(); - sourcePath = res[1].strip(); - } - } - return new Pair<>(sourceHostIp, sourcePath); - } - private boolean exportOVAFromVMOnVcenter(String vmExportUrl, String targetOvfDir, int noOfThreads, @@ -412,27 +240,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper disks = List.of(sourceDisk); - String destinationPoolUuid = UUID.randomUUID().toString(); - List destinationPools = List.of(destinationPoolUuid); - - KVMPhysicalDisk destDisk = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(destDisk.getPath()).thenReturn("xyz"); - Mockito.when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, destinationPoolUuid)) - .thenReturn(destinationPool); - Mockito.when(destinationPool.getType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); - Mockito.when(storagePoolManager.copyPhysicalDisk(Mockito.eq(sourceDisk), Mockito.anyString(), Mockito.eq(destinationPool), Mockito.anyInt())) - .thenReturn(destDisk); - - List movedDisks = convertInstanceCommandWrapper.moveTemporaryDisksToDestination(disks, destinationPools, storagePoolManager); - Assert.assertEquals(1, movedDisks.size()); - Assert.assertEquals("xyz", movedDisks.get(0).getPath()); - } - - @Test - public void testGetUnmanagedInstanceDisks() { - try (MockedStatic