From 0dcbe57a47875d45ee7effa92451a331c5bccc10 Mon Sep 17 00:00:00 2001 From: Edward-x <30854794+YLChen-007@users.noreply.github.com> Date: Wed, 28 Jan 2026 14:56:44 +0800 Subject: [PATCH 01/23] Fix that Sensitive information logged in SshHelper.sshExecute method (#12026) * Sensitive information logged in SshHelper.sshExecute method * Fix that Sensitive information logged in SshHelper.sshExecute method2 * Fix sensitive information handling in SshHelper and its tests --------- Co-authored-by: chenyoulong20g@ict.ac.cn --- .../java/com/cloud/utils/ssh/SshHelper.java | 73 ++++++++++++++++++- .../com/cloud/utils/ssh/SshHelperTest.java | 60 +++++++++++++++ 2 files changed, 129 insertions(+), 4 deletions(-) diff --git a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java index 87221ab5ac8..caf2b28c52f 100644 --- a/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java +++ b/utils/src/main/java/com/cloud/utils/ssh/SshHelper.java @@ -23,6 +23,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -40,6 +42,23 @@ public class SshHelper { private static final int DEFAULT_CONNECT_TIMEOUT = 180000; private static final int DEFAULT_KEX_TIMEOUT = 60000; private static final int DEFAULT_WAIT_RESULT_TIMEOUT = 120000; + private static final String MASKED_VALUE = "*****"; + + private static final Pattern[] SENSITIVE_COMMAND_PATTERNS = new Pattern[] { + Pattern.compile("(?i)(\\s+-p\\s+['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-p\\s+)([^\\s]+)"), + Pattern.compile("(?i)(\\s+-p=['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-p=)([^\\s]+)"), + Pattern.compile("(?i)(--password=['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(--password=)([^\\s]+)"), + Pattern.compile("(?i)(--password\\s+['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(--password\\s+)([^\\s]+)"), + Pattern.compile("(?i)(\\s+-u\\s+['\"][^,'\":]+[,:])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-u\\s+[^\\s,:]+[,:])([^\\s]+)"), + Pattern.compile("(?i)(\\s+-s\\s+['\"])([^'\"]*)(['\"])"), + Pattern.compile("(?i)(\\s+-s\\s+)([^\\s]+)"), + + }; protected static Logger LOGGER = LogManager.getLogger(SshHelper.class); @@ -145,7 +164,7 @@ public class SshHelper { } public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, String[] localFiles, String fileMode, - int connectTimeoutInMs, int kexTimeoutInMs) throws Exception { + int connectTimeoutInMs, int kexTimeoutInMs) throws Exception { com.trilead.ssh2.Connection conn = null; com.trilead.ssh2.SCPClient scpClient = null; @@ -291,13 +310,16 @@ public class SshHelper { } if (sess.getExitStatus() == null) { - //Exit status is NOT available. Returning failure result. - LOGGER.error(String.format("SSH execution of command %s has no exit status set. Result output: %s", command, result)); + // Exit status is NOT available. Returning failure result. + LOGGER.error(String.format("SSH execution of command %s has no exit status set. Result output: %s", + sanitizeForLogging(command), sanitizeForLogging(result))); return new Pair(false, result); } if (sess.getExitStatus() != null && sess.getExitStatus().intValue() != 0) { - LOGGER.error(String.format("SSH execution of command %s has an error status code in return. Result output: %s", command, result)); + LOGGER.error(String.format( + "SSH execution of command %s has an error status code in return. Result output: %s", + sanitizeForLogging(command), sanitizeForLogging(result))); return new Pair(false, result); } return new Pair(true, result); @@ -366,4 +388,47 @@ public class SshHelper { throw new SshException(msg); } } + + private static String sanitizeForLogging(String value) { + if (value == null) { + return null; + } + String masked = maskSensitiveValue(value); + String cleaned = com.cloud.utils.StringUtils.cleanString(masked); + if (StringUtils.isBlank(cleaned)) { + return masked; + } + return cleaned; + } + + private static String maskSensitiveValue(String value) { + String masked = value; + for (Pattern pattern : SENSITIVE_COMMAND_PATTERNS) { + masked = replaceWithMask(masked, pattern); + } + return masked; + } + + private static String replaceWithMask(String value, Pattern pattern) { + Matcher matcher = pattern.matcher(value); + if (!matcher.find()) { + return value; + } + + StringBuffer buffer = new StringBuffer(); + do { + StringBuilder replacement = new StringBuilder(); + replacement.append(matcher.group(1)); + if (matcher.groupCount() >= 3) { + replacement.append(MASKED_VALUE); + replacement.append(matcher.group(matcher.groupCount())); + } else { + replacement.append(MASKED_VALUE); + } + matcher.appendReplacement(buffer, Matcher.quoteReplacement(replacement.toString())); + } while (matcher.find()); + + matcher.appendTail(buffer); + return buffer.toString(); + } } diff --git a/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java b/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java index 61d746bc12d..8a14f60527b 100644 --- a/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java +++ b/utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java @@ -21,6 +21,7 @@ package com.cloud.utils.ssh; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.Method; import org.junit.Assert; import org.junit.Test; @@ -140,4 +141,63 @@ public class SshHelperTest { Mockito.verify(conn).openSession(); } + + @Test + public void sanitizeForLoggingMasksShortPasswordFlag() throws Exception { + String command = "/opt/cloud/bin/script -v 10.0.0.1 -p superSecret"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain flag", sanitized.contains("-p *****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("superSecret")); + } + + @Test + public void sanitizeForLoggingMasksQuotedPasswordFlag() throws Exception { + String command = "/opt/cloud/bin/script -v 10.0.0.1 -p \"super Secret\""; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain quoted flag", sanitized.contains("-p *****")); + Assert.assertFalse("Sanitized command should not contain original password", + sanitized.contains("super Secret")); + } + + @Test + public void sanitizeForLoggingMasksLongPasswordAssignments() throws Exception { + String command = "tool --password=superSecret"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain assignment", sanitized.contains("--password=*****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("superSecret")); + } + + @Test + public void sanitizeForLoggingMasksUsernamePasswordPairs() throws Exception { + String command = "/opt/cloud/bin/vpn_l2tp.sh -u alice,topSecret"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain username and mask password", + sanitized.contains("-u alice,*****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("topSecret")); + } + + @Test + public void sanitizeForLoggingMasksUsernamePasswordPairsWithColon() throws Exception { + String command = "curl -u alice:topSecret https://example.com"; + String sanitized = invokeSanitizeForLogging(command); + + Assert.assertTrue("Sanitized command should retain username and mask password", + sanitized.contains("-u alice:*****")); + Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("topSecret")); + } + + @Test + public void sanitizeForLoggingHandlesNullValues() throws Exception { + Assert.assertNull(invokeSanitizeForLogging(null)); + } + + private String invokeSanitizeForLogging(String value) throws Exception { + Method method = SshHelper.class.getDeclaredMethod("sanitizeForLogging", String.class); + method.setAccessible(true); + return (String) method.invoke(null, value); + } } From 1b2ae13df74ef3f22f8e5b40b4cf45a10863205f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 28 Jan 2026 12:40:34 +0530 Subject: [PATCH 02/23] ui: add cache for oslogo request using osId (#11422) When OsLogo component is used in the items of a list having same OS type it was causing listOsTypes API call multiple time. This change allows caching request and response value for 30 seconds. Caching behaviour is controlled using `useCache` flag. Signed-off-by: Abhishek Kumar --- ui/src/components/widgets/OsLogo.vue | 78 ++++++++++--------- .../compute/wizard/OsBasedImageRadioGroup.vue | 3 +- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/ui/src/components/widgets/OsLogo.vue b/ui/src/components/widgets/OsLogo.vue index 643953012c1..f19aac56a1a 100644 --- a/ui/src/components/widgets/OsLogo.vue +++ b/ui/src/components/widgets/OsLogo.vue @@ -31,6 +31,9 @@ - - diff --git a/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue b/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue index 2518ed0c042..45ea347553c 100644 --- a/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue +++ b/ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue @@ -42,7 +42,8 @@ class="radio-group__os-logo" size="2x" :osId="item.ostypeid" - :os-name="item.osName" /> + :os-name="item.osName" + :use-cache="true" />   {{ item.displaytext }} From 6932cacabc187cf3d76e53c7979ed10067aff2f2 Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Wed, 28 Jan 2026 16:00:30 +0530 Subject: [PATCH 03/23] Allow copy of templates from secondary storages of other zone when adding a new secondary storage (#12296) * Allow copy of templates from secondary storages of other zone when adding a new secondary storage * Add API param and UI changes on add secondary storage page * Make copy template across zones non blocking * Code fixes * unused imports * Add copy template flag in zone wizard and remove NFS checks * Fix UI * Label fixes * code optimizations * code refactoring * missing changes * Combine template copy and download into a single asynchronous operation * unused import and fixed conflicts * unused code * update config message * Fix configuration setting value on add secondary storage page * Removed unused code * Update unit tests --- .../admin/host/AddSecondaryStorageCmd.java | 24 ++- .../service/StorageOrchestrationService.java | 3 +- .../api/storage/TemplateService.java | 4 +- .../com/cloud/storage/StorageManager.java | 5 +- .../orchestration/StorageOrchestrator.java | 45 +++-- .../storage/image/TemplateServiceImpl.java | 163 ++++++++++++++--- .../image/TemplateServiceImplTest.java | 171 +++++++++++++++++- .../cloud/storage/ImageStoreDetailsUtil.java | 11 ++ .../com/cloud/storage/StorageManagerImpl.java | 2 +- .../cloud/template/TemplateManagerImpl.java | 14 +- ui/public/locales/en.json | 4 +- ui/src/views/infra/AddSecondaryStorage.vue | 82 ++++++++- .../infra/zone/ZoneWizardAddResources.vue | 25 ++- .../views/infra/zone/ZoneWizardLaunchZone.vue | 5 + 14 files changed, 490 insertions(+), 68 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java index 9a7eff7e2e5..585fd1b87a8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java @@ -29,6 +29,11 @@ import org.apache.cloudstack.api.response.ZoneResponse; import com.cloud.exception.DiscoveryException; import com.cloud.storage.ImageStore; import com.cloud.user.Account; +import org.apache.commons.collections.MapUtils; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; @APICommand(name = "addSecondaryStorage", description = "Adds secondary storage.", responseObject = ImageStoreResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -44,6 +49,9 @@ public class AddSecondaryStorageCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "The Zone ID for the secondary storage") protected Long zoneId; + @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].copytemplatesfromothersecondarystorages=true") + protected Map details; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -56,6 +64,20 @@ public class AddSecondaryStorageCmd extends BaseCmd { return zoneId; } + public Map getDetails() { + Map detailsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(details)) { + Collection props = details.values(); + for (Object prop : props) { + HashMap detail = (HashMap) prop; + for (Map.Entry entry: detail.entrySet()) { + detailsMap.put(entry.getKey(),entry.getValue()); + } + } + } + return detailsMap; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -68,7 +90,7 @@ public class AddSecondaryStorageCmd extends BaseCmd { @Override public void execute(){ try{ - ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null); + ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), getDetails()); ImageStoreResponse storeResponse = null; if (result != null ) { storeResponse = _responseGenerator.createImageStoreResponse(result); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java index 8be2015bfef..4af0c806060 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java @@ -22,7 +22,6 @@ import java.util.concurrent.Future; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; @@ -31,5 +30,5 @@ public interface StorageOrchestrationService { MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List templateIdList, List snapshotIdList); - Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore); + Future orchestrateTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index a8861d5acc6..269eb4f1c21 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -80,4 +80,6 @@ public interface TemplateService { List getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId); AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore); -} + + void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); + } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index de0cb34d63e..4ce1f4a9638 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -220,8 +220,9 @@ public interface StorageManager extends StorageService { "storage.pool.host.connect.workers", "1", "Number of worker threads to be used to connect hosts to a primary storage", true); - ConfigKey COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES = new ConfigKey<>(Boolean.class, "copy.public.templates.from.other.storages", - "Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.", + ConfigKey COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES = new ConfigKey<>(Boolean.class, "copy.templates.from.other.secondary.storages", + "Storage", "true", "When enabled, this feature allows templates to be copied from existing Secondary Storage servers (within the same zone or across zones) " + + "while adding a new Secondary Storage. If the copy operation fails, the system falls back to downloading the template from the source URL.", true, ConfigKey.Scope.Zone, null); /** diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 37a1f8dc196..933b4e0c5ce 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -36,6 +36,9 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.TemplateManager; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -45,6 +48,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageServic import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; @@ -103,6 +107,15 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra VolumeDataStoreDao volumeDataStoreDao; @Inject DataMigrationUtility migrationHelper; + @Inject + TemplateManager templateManager; + @Inject + VMTemplateDao templateDao; + @Inject + TemplateDataFactory templateDataFactory; + @Inject + DataCenterDao dcDao; + ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, "image.store.imbalance.threshold", @@ -304,8 +317,9 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra } @Override - public Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore) { - return submit(destStore.getScope().getScopeId(), new CopyTemplateTask(source, destStore)); + public Future orchestrateTemplateCopyFromSecondaryStores(long srcTemplateId, DataStore destStore) { + Long dstZoneId = destStore.getScope().getScopeId(); + return submit(dstZoneId, new CopyTemplateFromSecondaryStorageTask(srcTemplateId, destStore)); } protected Pair migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List files, MigrationPolicy migrationPolicy, int skipped) { @@ -624,13 +638,13 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra } } - private class CopyTemplateTask implements Callable { - private TemplateInfo sourceTmpl; - private DataStore destStore; - private String logid; + private class CopyTemplateFromSecondaryStorageTask implements Callable { + private final long srcTemplateId; + private final DataStore destStore; + private final String logid; - public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) { - this.sourceTmpl = sourceTmpl; + CopyTemplateFromSecondaryStorageTask(long srcTemplateId, DataStore destStore) { + this.srcTemplateId = srcTemplateId; this.destStore = destStore; this.logid = ThreadContext.get(LOGCONTEXTID); } @@ -639,17 +653,16 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra public TemplateApiResult call() { ThreadContext.put(LOGCONTEXTID, logid); TemplateApiResult result; - AsyncCallFuture future = templateService.copyTemplateToImageStore(sourceTmpl, destStore); + long destZoneId = destStore.getScope().getScopeId(); + TemplateInfo sourceTmpl = templateDataFactory.getTemplate(srcTemplateId, DataStoreRole.Image); try { - result = future.get(); - } catch (ExecutionException | InterruptedException e) { - logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", - sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + templateService.handleTemplateCopyFromSecondaryStores(srcTemplateId, destStore); result = new TemplateApiResult(sourceTmpl); - result.setResult(e.getMessage()); + } finally { + tryCleaningUpExecutor(destZoneId); + ThreadContext.clearAll(); } - tryCleaningUpExecutor(destStore.getScope().getScopeId()); - ThreadContext.clearAll(); + return result; } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index bee62955051..5fc9bbac352 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -31,6 +31,8 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; +import com.cloud.exception.StorageUnavailableException; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -67,9 +69,11 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.ThreadContext; import org.springframework.stereotype.Component; import com.cloud.agent.api.Answer; @@ -567,10 +571,7 @@ public class TemplateServiceImpl implements TemplateService { } if (availHypers.contains(tmplt.getHypervisorType())) { - boolean copied = isCopyFromOtherStoragesEnabled(zoneId) && tryCopyingTemplateToImageStore(tmplt, store); - if (!copied) { - tryDownloadingTemplateToImageStore(tmplt, store); - } + storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(tmplt.getId(), store); } else { logger.info("Skip downloading template {} since current data center does not have hypervisor {}", tmplt, tmplt.getHypervisorType()); } @@ -617,6 +618,16 @@ public class TemplateServiceImpl implements TemplateService { } + @Override + public void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore) { + VMTemplateVO template = _templateDao.findById(templateId); + long zoneId = destStore.getScope().getScopeId(); + boolean copied = imageStoreDetailsUtil.isCopyTemplatesFromOtherStoragesEnabled(destStore.getId(), zoneId) && tryCopyingTemplateToImageStore(template, destStore); + if (!copied) { + tryDownloadingTemplateToImageStore(template, destStore); + } + } + protected void tryDownloadingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { if (tmplt.getUrl() == null) { logger.info("Not downloading template [{}] to image store [{}], as it has no URL.", tmplt.getUniqueName(), @@ -634,28 +645,134 @@ public class TemplateServiceImpl implements TemplateService { } protected boolean tryCopyingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { - Long zoneId = destStore.getScope().getScopeId(); - List storesInZone = _storeMgr.getImageStoresByZoneIds(zoneId); - for (DataStore sourceStore : storesInZone) { - Map existingTemplatesInSourceStore = listTemplate(sourceStore); - if (existingTemplatesInSourceStore == null || !existingTemplatesInSourceStore.containsKey(tmplt.getUniqueName())) { - logger.debug("Template [{}] does not exist on image store [{}]; searching on another one.", - tmplt.getUniqueName(), sourceStore.getName()); - continue; - } - TemplateObject sourceTmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), sourceStore); - if (sourceTmpl.getInstallPath() == null) { - logger.warn("Can not copy template [{}] from image store [{}], as it returned a null install path.", tmplt.getUniqueName(), - sourceStore.getName()); - continue; - } - storageOrchestrator.orchestrateTemplateCopyToImageStore(sourceTmpl, destStore); + if (searchAndCopyWithinZone(tmplt, destStore)) { return true; } - logger.debug("Can't copy template [{}] from another image store.", tmplt.getUniqueName()); + + Long destZoneId = destStore.getScope().getScopeId(); + logger.debug("Template [{}] not found in any image store of zone [{}]. Checking other zones.", + tmplt.getUniqueName(), destZoneId); + + return searchAndCopyAcrossZones(tmplt, destStore, destZoneId); + } + + private boolean searchAndCopyAcrossZones(VMTemplateVO tmplt, DataStore destStore, Long destZoneId) { + List allZoneIds = _dcDao.listAllIds(); + for (Long otherZoneId : allZoneIds) { + if (otherZoneId.equals(destZoneId)) { + continue; + } + + List storesInOtherZone = _storeMgr.getImageStoresByZoneIds(otherZoneId); + logger.debug("Checking zone [{}] for template [{}]...", otherZoneId, tmplt.getUniqueName()); + + if (CollectionUtils.isEmpty(storesInOtherZone)) { + logger.debug("Zone [{}] has no image stores. Skipping.", otherZoneId); + continue; + } + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInOtherZone); + if (sourceTmpl == null) { + logger.debug("Template [{}] not found with a valid install path in any image store of zone [{}].", + tmplt.getUniqueName(), otherZoneId); + continue; + } + + logger.info("Template [{}] found in zone [{}]. Initiating cross-zone copy to zone [{}].", + tmplt.getUniqueName(), otherZoneId, destZoneId); + + return copyTemplateAcrossZones(destStore, sourceTmpl); + } + + logger.debug("Template [{}] was not found in any zone. Cannot perform zone-to-zone copy.", tmplt.getUniqueName()); return false; } + protected TemplateObject findUsableTemplate(VMTemplateVO tmplt, List imageStores) { + for (DataStore store : imageStores) { + + Map templates = listTemplate(store); + if (templates == null || !templates.containsKey(tmplt.getUniqueName())) { + continue; + } + + TemplateObject tmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), store); + if (tmpl.getInstallPath() == null) { + logger.debug("Template [{}] found in image store [{}] but install path is null. Skipping.", + tmplt.getUniqueName(), store.getName()); + continue; + } + return tmpl; + } + return null; + } + + private boolean searchAndCopyWithinZone(VMTemplateVO tmplt, DataStore destStore) { + Long destZoneId = destStore.getScope().getScopeId(); + List storesInSameZone = _storeMgr.getImageStoresByZoneIds(destZoneId); + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInSameZone); + if (sourceTmpl == null) { + return false; + } + + TemplateApiResult result; + AsyncCallFuture future = copyTemplateToImageStore(sourceTmpl, destStore); + try { + result = future.get(); + } catch (ExecutionException | InterruptedException e) { + logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", + sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } + return result.isSuccess(); + } + + private boolean copyTemplateAcrossZones(DataStore destStore, TemplateObject sourceTmpl) { + Long dstZoneId = destStore.getScope().getScopeId(); + DataCenterVO dstZone = _dcDao.findById(dstZoneId); + + if (dstZone == null) { + logger.warn("Destination zone [{}] not found for template [{}].", dstZoneId, sourceTmpl.getUniqueName()); + return false; + } + + TemplateApiResult result; + try { + VMTemplateVO template = _templateDao.findById(sourceTmpl.getId()); + try { + DataStore sourceStore = sourceTmpl.getDataStore(); + long userId = CallContext.current().getCallingUserId(); + boolean success = _tmpltMgr.copy(userId, template, sourceStore, dstZone); + + result = new TemplateApiResult(sourceTmpl); + if (!success) { + result.setResult("Cross-zone template copy failed"); + } + } catch (StorageUnavailableException | ResourceAllocationException e) { + logger.error("Exception while copying template [{}] from zone [{}] to zone [{}]", + template, + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZone.getId(), + e); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } finally { + ThreadContext.clearAll(); + } + } catch (Exception e) { + logger.error("Failed to copy template [{}] from zone [{}] to zone [{}].", + sourceTmpl.getUniqueName(), + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZoneId, + e); + return false; + } + + return result.isSuccess(); + } + @Override public AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore) { TemplateObject sourceTmpl = (TemplateObject) source; @@ -699,10 +816,6 @@ public class TemplateServiceImpl implements TemplateService { return null; } - protected boolean isCopyFromOtherStoragesEnabled(Long zoneId) { - return StorageManager.COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES.valueIn(zoneId); - } - protected void publishTemplateCreation(TemplateInfo tmplt) { VMTemplateVO tmpltVo = _templateDao.findById(tmplt.getId()); diff --git a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java index cb7994915b3..e9eac045869 100644 --- a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java +++ b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java @@ -18,13 +18,20 @@ */ package org.apache.cloudstack.storage.image; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.template.TemplateProp; import com.cloud.template.TemplateManager; +import com.cloud.user.Account; +import com.cloud.user.User; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; -import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.store.TemplateObject; @@ -46,6 +53,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.mockito.Mockito.mock; + @RunWith(MockitoJUnitRunner.class) public class TemplateServiceImplTest { @@ -89,6 +98,12 @@ public class TemplateServiceImplTest { @Mock TemplateManager templateManagerMock; + @Mock + VMTemplateDao templateDao; + + @Mock + DataCenterDao _dcDao; + Map templatesInSourceStore = new HashMap<>(); @Before @@ -101,7 +116,6 @@ public class TemplateServiceImplTest { Mockito.doReturn(List.of(sourceStoreMock, destStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(zoneId); Mockito.doReturn(templatesInSourceStore).when(templateService).listTemplate(sourceStoreMock); Mockito.doReturn(null).when(templateService).listTemplate(destStoreMock); - Mockito.doReturn("install-path").when(templateInfoMock).getInstallPath(); Mockito.doReturn(templateInfoMock).when(templateDataFactoryMock).getTemplate(2L, sourceStoreMock); Mockito.doReturn(3L).when(dataStoreMock).getId(); Mockito.doReturn(zoneScopeMock).when(dataStoreMock).getScope(); @@ -166,7 +180,7 @@ public class TemplateServiceImplTest { boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertFalse(result); - Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any()); } @Test @@ -174,20 +188,161 @@ public class TemplateServiceImplTest { templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock); Mockito.doReturn(null).when(templateInfoMock).getInstallPath(); + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(List.of(1L)).when(_dcDao).listAllIds(); + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertFalse(result); - Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any()); } @Test - public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherStorageAndTaskWasScheduled() { - templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock); - Mockito.doReturn(new AsyncCallFuture<>()).when(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherZone() throws StorageUnavailableException, ResourceAllocationException { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + Mockito.doReturn(null).when(templateService).listTemplate(sourceStoreMock); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templatesInOtherZone = new HashMap<>(); + templatesInOtherZone.put("unique-name", tmpltPropMock); + Mockito.doReturn(templatesInOtherZone).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + + DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class); + Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L); + Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any()); boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertTrue(result); - Mockito.verify(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenDestinationZoneIsMissing() { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templates = new HashMap<>(); + templates.put("unique-name", tmpltPropMock); + Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + Mockito.doReturn(null).when(_dcDao).findById(1L); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertFalse(result); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenCrossZoneCopyTaskIsScheduled() throws StorageUnavailableException, ResourceAllocationException { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templates = new HashMap<>(); + templates.put("unique-name", tmpltPropMock); + Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + Mockito.doReturn(100L).when(sourceTmplMock).getId(); + + DataStore sourceStoreMock = Mockito.mock(DataStore.class); + Scope sourceScopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(sourceStoreMock).when(sourceTmplMock).getDataStore(); + + DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class); + Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L); + VMTemplateVO templateVoMock = Mockito.mock(VMTemplateVO.class); + Mockito.doReturn(templateVoMock).when(templateDao).findById(100L); + + Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any()); + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext callContext = mock(CallContext.class); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertTrue(result); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenTemplateNotFoundInAnyZone() { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(Mockito.anyLong()); + Mockito.doReturn(null).when(templateService).listTemplate(Mockito.any()); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertFalse(result); + } + + @Test + public void testFindUsableTemplateReturnsTemplateWithNonNullInstallPath() { + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.getId()).thenReturn(10L); + Mockito.when(template.getUniqueName()).thenReturn("test-template"); + + DataStore storeWithNullPath = Mockito.mock(DataStore.class); + Mockito.when(storeWithNullPath.getName()).thenReturn("store-null"); + + DataStore storeWithValidPath = Mockito.mock(DataStore.class); + TemplateObject tmplWithNullPath = Mockito.mock(TemplateObject.class); + Mockito.when(tmplWithNullPath.getInstallPath()).thenReturn(null); + + TemplateObject tmplWithValidPath = Mockito.mock(TemplateObject.class); + Mockito.when(tmplWithValidPath.getInstallPath()).thenReturn("/mnt/secondary/template.qcow2"); + + Mockito.doReturn(tmplWithNullPath).when(templateDataFactoryMock).getTemplate(10L, storeWithNullPath); + Mockito.doReturn(tmplWithValidPath).when(templateDataFactoryMock).getTemplate(10L, storeWithValidPath); + + Map templates = new HashMap<>(); + templates.put("test-template", Mockito.mock(TemplateProp.class)); + + Mockito.doReturn(templates).when(templateService).listTemplate(storeWithNullPath); + Mockito.doReturn(templates).when(templateService).listTemplate(storeWithValidPath); + + List imageStores = List.of(storeWithNullPath, storeWithValidPath); + + TemplateObject result = templateService.findUsableTemplate(template, imageStores); + + Assert.assertNotNull(result); + Assert.assertEquals(tmplWithValidPath, result); } } diff --git a/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java b/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java index baf5ef8902d..9f5aa660f4f 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java @@ -78,4 +78,15 @@ public class ImageStoreDetailsUtil { return getGlobalDefaultNfsVersion(); } + public boolean isCopyTemplatesFromOtherStoragesEnabled(Long storeId, Long zoneId) { + final Map storeDetails = imageStoreDetailsDao.getDetails(storeId); + final String keyWithoutDots = StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.key() + .replace(".", ""); + + if (storeDetails != null && storeDetails.containsKey(keyWithoutDots)) { + return Boolean.parseBoolean(storeDetails.get(keyWithoutDots)); + } + + return StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.valueIn(zoneId); + } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 13b7fbb00c2..d1dca0fa901 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -4206,7 +4206,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStoreDownloadFollowRedirects, AllowVolumeReSizeBeyondAllocation, StoragePoolHostConnectWorkers, - COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES + COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES }; } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 5773410c35a..78265021c0a 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -842,6 +842,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, // Copy will just find one eligible image store for the destination zone // and copy template there, not propagate to all image stores // for that zone + + boolean copied = false; + for (DataStore dstSecStore : dstSecStores) { TemplateDataStoreVO dstTmpltStore = _tmplStoreDao.findByStoreTemplate(dstSecStore.getId(), tmpltId); if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOADED) { @@ -856,9 +859,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiResult result = future.get(); if (result.isFailed()) { logger.debug("Copy Template failed for image store {}: {}", dstSecStore, result.getResult()); + _tmplStoreDao.removeByTemplateStore(tmpltId, dstSecStore.getId()); continue; // try next image store } + copied = true; + _tmpltDao.addTemplateToZone(template, dstZoneId); if (account.getId() != Account.ACCOUNT_ID_SYSTEM) { @@ -886,12 +892,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } } + + return true; + } catch (Exception ex) { - logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore); + logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore, ex); } } - return true; - + return copied; } @Override diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index b2465fa325f..99873820d53 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -591,6 +591,8 @@ "label.copy.consoleurl": "Copy console URL to clipboard", "label.copyid": "Copy ID", "label.copy.password": "Copy password", +"label.copy.templates.from.other.secondary.storages": "Copy Templates from other storages instead of fetching from URLs", +"label.copy.templates.from.other.secondary.storages.add.zone": "Copy Templates from other storages", "label.core": "Core", "label.core.zone.type": "Core Zone type", "label.counter": "Counter", @@ -3019,7 +3021,7 @@ "message.desc.importmigratefromvmwarewizard": "By selecting an existing or external VMware Datacenter and an instance to import, CloudStack migrates the selected instance from VMware to KVM on a conversion host using virt-v2v and imports it into a KVM Cluster", "message.desc.primary.storage": "Each Cluster must contain one or more primary storage servers. We will add the first one now. Primary storage contains the disk volumes for all the Instances running on hosts in the cluster. Use any standards-compliant protocol that is supported by the underlying hypervisor.", "message.desc.reset.ssh.key.pair": "Please specify a ssh key pair that you would like to add to this Instance.", -"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

Provide the IP address and exported path.", +"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

Provide the IP address and exported path.

\"Copy templates from other secondary storages\" switch can be used to automatically copy existing templates from secondary storages in other zones instead of fetching from their URLs.", "message.desc.register.user.data": "Please fill in the following to register new User Data.", "message.desc.registered.user.data": "Registered a User Data.", "message.desc.zone": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", diff --git a/ui/src/views/infra/AddSecondaryStorage.vue b/ui/src/views/infra/AddSecondaryStorage.vue index 746af5b959d..db4893115a6 100644 --- a/ui/src/views/infra/AddSecondaryStorage.vue +++ b/ui/src/views/infra/AddSecondaryStorage.vue @@ -48,6 +48,7 @@ +
+ + + +
{{ $t('label.cancel') }} {{ $t('label.ok') }} @@ -191,7 +204,9 @@ export default { providers: ['NFS', 'SMB/CIFS', 'S3', 'Swift'], zones: [], loading: false, - secondaryStorageNFSStaging: false + secondaryStorageNFSStaging: false, + showCopyTemplatesToggle: false, + copyTemplatesTouched: false } }, created () { @@ -203,7 +218,8 @@ export default { this.formRef = ref() this.form = reactive({ provider: 'NFS', - secondaryStorageHttps: true + secondaryStorageHttps: true, + copyTemplatesFromOtherSecondaryStorages: true }) this.rules = reactive({ zone: [{ required: true, message: this.$t('label.required') }], @@ -225,20 +241,56 @@ export default { }, fetchData () { this.listZones() + this.checkOtherSecondaryStorages() }, closeModal () { this.$emit('close-action') }, + fetchCopyTemplatesConfig () { + if (!this.form.zone) { + return + } + + api('listConfigurations', { + name: 'copy.templates.from.other.secondary.storages', + zoneid: this.form.zone + }).then(json => { + const items = + json?.listconfigurationsresponse?.configuration || [] + + items.forEach(item => { + if (item.name === 'copy.templates.from.other.secondary.storages') { + this.form.copyTemplatesFromOtherSecondaryStorages = + item.value === 'true' + } + }) + }) + }, + onZoneChange (val) { + this.form.zone = val + this.copyTemplatesTouched = false + this.fetchCopyTemplatesConfig() + }, listZones () { api('listZones', { showicon: true }).then(json => { - if (json && json.listzonesresponse && json.listzonesresponse.zone) { - this.zones = json.listzonesresponse.zone - if (this.zones.length > 0) { - this.form.zone = this.zones[0].id || '' - } + this.zones = json.listzonesresponse.zone || [] + + if (this.zones.length > 0) { + this.form.zone = this.zones[0].id + this.fetchCopyTemplatesConfig() } }) }, + checkOtherSecondaryStorages () { + api('listImageStores', { listall: true }).then(json => { + const stores = json?.listimagestoresresponse?.imagestore || [] + + this.showCopyTemplatesToggle = stores.length > 0 + }) + }, + onCopyTemplatesToggleChanged (val) { + this.copyTemplatesTouched = true + }, nfsURL (server, path) { var url if (path.substring(0, 1) !== '/') { @@ -362,6 +414,22 @@ export default { nfsParams.url = nfsUrl } + if ( + this.showCopyTemplatesToggle && + this.copyTemplatesTouched + ) { + const copyTemplatesKey = 'copytemplatesfromothersecondarystorages' + + const detailIdx = Object.keys(data) + .filter(k => k.startsWith('details[')) + .map(k => parseInt(k.match(/details\[(\d+)\]/)[1])) + .reduce((a, b) => Math.max(a, b), -1) + 1 + + data[`details[${detailIdx}].key`] = copyTemplatesKey + data[`details[${detailIdx}].value`] = + values.copyTemplatesFromOtherSecondaryStorages.toString() + } + this.loading = true try { diff --git a/ui/src/views/infra/zone/ZoneWizardAddResources.vue b/ui/src/views/infra/zone/ZoneWizardAddResources.vue index 4bd602f0aca..298cc7fec9d 100644 --- a/ui/src/views/infra/zone/ZoneWizardAddResources.vue +++ b/ui/src/views/infra/zone/ZoneWizardAddResources.vue @@ -840,6 +840,13 @@ export default { display: { secondaryStorageProvider: ['Swift'] } + }, + { + title: 'label.copy.templates.from.other.secondary.storages.add.zone', + key: 'copyTemplatesFromOtherSecondaryStorages', + required: false, + switch: true, + checked: this.copytemplate } ] } @@ -860,7 +867,8 @@ export default { }], storageProviders: [], currentStep: null, - options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider'] + options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider'], + copytemplate: true } }, created () { @@ -885,6 +893,7 @@ export default { primaryStorageScope: null }) } + this.applyCopyTemplatesOptionFromGlobalSettingDuringSecondaryStorageAddition() } }, watch: { @@ -1108,6 +1117,20 @@ export default { this.storageProviders = storageProviders }) }, + applyCopyTemplatesOptionFromGlobalSettingDuringSecondaryStorageAddition () { + api('listConfigurations', { + name: 'copy.templates.from.other.secondary.storages' + }).then(json => { + const config = json?.listconfigurationsresponse?.configuration?.[0] + + if (!config || config.value === undefined) { + return + } + + const value = String(config.value).toLowerCase() === 'true' + this.copytemplate = value + }) + }, fetchPrimaryStorageProvider () { this.primaryStorageProviders = [] api('listStorageProviders', { type: 'primary' }).then(json => { diff --git a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue index a787ad839cd..fbf5e6f5c20 100644 --- a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue +++ b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue @@ -1580,6 +1580,11 @@ export default { params.provider = this.prefillContent.secondaryStorageProvider params.zoneid = this.stepData.zoneReturned.id params.url = url + if (this.prefillContent.copyTemplatesFromOtherSecondaryStorages !== undefined) { + params['details[0].key'] = 'copytemplatesfromothersecondarystorages' + params['details[0].value'] = + this.prefillContent.copyTemplatesFromOtherSecondaryStorages + } } else if (this.prefillContent.secondaryStorageProvider === 'SMB') { const nfsServer = this.prefillContent.secondaryStorageServer const path = this.prefillContent.secondaryStoragePath From ff7ec0cd229fc829f3f978e3573366cd20fec5f2 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 28 Jan 2026 16:15:48 +0530 Subject: [PATCH 04/23] Update alert id for VR public and private interface (#12527) --- .../main/java/org/apache/cloudstack/alert/AlertService.java | 4 ++-- .../src/main/java/com/cloud/alert/AlertManager.java | 1 - .../src/main/resources/META-INF/db/schema-42020to42030.sql | 3 +++ server/src/main/java/com/cloud/event/AlertGenerator.java | 5 +++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java index 1250284b5c2..4ae6288efce 100644 --- a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java +++ b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java @@ -71,8 +71,8 @@ public interface AlertService { public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true); public static final AlertType ALERT_TYPE_CA_CERT = new AlertType((short)31, "ALERT.CA.CERT", true); public static final AlertType ALERT_TYPE_VM_SNAPSHOT = new AlertType((short)32, "ALERT.VM.SNAPSHOT", true); - public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true); - public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true); + public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)33, "ALERT.VR.PUBLIC.IFACE.MTU", true); + public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)34, "ALERT.VR.PRIVATE.IFACE.MTU", true); public short getType() { return type; diff --git a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java index 3d4e6579f7c..7fe19c3ba9f 100644 --- a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java +++ b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java @@ -54,5 +54,4 @@ public interface AlertManager extends Manager, AlertService { void recalculateCapacity(); void sendAlert(AlertType alertType, long dataCenterId, Long podId, String subject, String body); - } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql index 598fdb7adc4..567e623564e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql @@ -20,3 +20,6 @@ --; ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048); + +UPDATE `cloud`.`alert` SET type = 33 WHERE name = 'ALERT.VR.PUBLIC.IFACE.MTU'; +UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; diff --git a/server/src/main/java/com/cloud/event/AlertGenerator.java b/server/src/main/java/com/cloud/event/AlertGenerator.java index f1b23e87308..601bf5e831a 100644 --- a/server/src/main/java/com/cloud/event/AlertGenerator.java +++ b/server/src/main/java/com/cloud/event/AlertGenerator.java @@ -67,12 +67,13 @@ public class AlertGenerator { } public static void publishAlertOnEventBus(String alertType, long dataCenterId, Long podId, String subject, String body) { - String configKey = Config.PublishAlertEvent.key(); String value = s_configDao.getValue(configKey); boolean configValue = Boolean.parseBoolean(value); - if(!configValue) + if (!configValue) { return; + } + try { eventDistributor = ComponentContext.getComponent(EventDistributor.class); } catch (NoSuchBeanDefinitionException nbe) { From 83ce0067b82bc39c7c91667c6ac4d2dd144ce450 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 28 Jan 2026 16:37:57 +0530 Subject: [PATCH 05/23] Update the snapshot physical size for the primary storage resource after snapshot creation and during resource count recalculation (#12481) * Update snapshot size for the primary storage resource after snapshot creation and during resource count recalculation * Update snapshot physical size * review * review --- .../user/snapshot/CreateSnapshotCmd.java | 3 +- .../datastore/db/SnapshotDataStoreDao.java | 14 +++++++++ .../db/SnapshotDataStoreDaoImpl.java | 29 +++++++++++++++++- .../ResourceLimitManagerImpl.java | 10 +++---- .../storage/snapshot/SnapshotManagerImpl.java | 30 +++++++++++-------- .../ResourceLimitManagerImplTest.java | 10 +++++-- 6 files changed, 73 insertions(+), 23 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index bd541b69183..078d4517f95 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -244,8 +244,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { } private Snapshot.LocationType getLocationType() { - - if (Snapshot.LocationType.values() == null || Snapshot.LocationType.values().length == 0 || locationType == null) { + if (locationType == null) { return null; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index ef0a5d0ebff..96df4928773 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -110,4 +110,18 @@ StateDao snapshotIds, Long batchSize); + + /** + * Returns the total physical size, in bytes, of all snapshots stored on primary + * storage for the specified account that have not yet been backed up to + * secondary storage. + * + *

If no such snapshots are found, this method returns {@code 0}.

+ * + * @param accountId the ID of the account whose snapshots on primary storage + * should be considered + * @return the total physical size in bytes of matching snapshots on primary + * storage, or {@code 0} if none are found + */ + long getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(long accountId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index ba76a6b3f41..c68316dd1fe 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -78,6 +78,15 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase params) throws ConfigurationException { super.configure(name, params); @@ -118,7 +127,6 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase) status -> { - long newResourceCount = 0L; List domainIdList = childDomains.stream().map(DomainVO::getId).collect(Collectors.toList()); domainIdList.add(domainId); List accountIdList = accounts.stream().map(AccountVO::getId).collect(Collectors.toList()); @@ -1189,6 +1188,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim List resourceCounts = _resourceCountDao.lockRows(rowIdsToLock); long oldResourceCount = 0L; + long newResourceCount = 0L; ResourceCountVO domainRC = null; // calculate project count here @@ -1210,7 +1210,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (oldResourceCount != newResourceCount) { domainRC.setCount(newResourceCount); _resourceCountDao.update(domainRC.getId(), domainRC); - logger.warn("Discrepency in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.", + logger.warn("Discrepancy in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.", oldResourceCount, newResourceCount, type, domainId); } return newResourceCount; @@ -1436,16 +1436,17 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } protected long calculatePrimaryStorageForAccount(long accountId, String tag) { + long snapshotsPhysicalSizeOnPrimaryStorage = _snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId); if (StringUtils.isEmpty(tag)) { List virtualRouters = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId); - return _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); + return snapshotsPhysicalSizeOnPrimaryStorage + _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); } long storage = 0; List volumes = getVolumesWithAccountAndTag(accountId, tag); for (VolumeVO volume : volumes) { storage += volume.getSize() == null ? 0L : volume.getSize(); } - return storage; + return snapshotsPhysicalSizeOnPrimaryStorage + storage; } @Override @@ -2143,7 +2144,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim protected class ResourceCountCheckTask extends ManagedContextRunnable { public ResourceCountCheckTask() { - } @Override diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index e7606572a07..19cde4da0f1 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -276,6 +276,15 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement return !DataCenter.Type.Edge.equals(zone.getType()); } + private ResourceType getStoreResourceType(long dataCenterId, Snapshot.LocationType locationType) { + ResourceType storeResourceType = ResourceType.secondary_storage; + if (!isBackupSnapshotToSecondaryForZone(dataCenterId) || + Snapshot.LocationType.PRIMARY.equals(locationType)) { + storeResourceType = ResourceType.primary_storage; + } + return storeResourceType; + } + @Override public String getConfigComponentName() { return SnapshotManager.class.getSimpleName(); @@ -614,7 +623,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _snapshotDao.update(snapshot.getId(), snapshot); snapshotInfo = this.snapshotFactory.getSnapshot(snapshotId, store); - Long snapshotOwnerId = vm.getAccountId(); + long snapshotOwnerId = vm.getAccountId(); try { SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.BACKUP); @@ -622,7 +631,6 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement throw new CloudRuntimeException(String.format("Unable to find Snapshot strategy to handle Snapshot [%s]", snapshot)); } snapshotInfo = snapshotStrategy.backupSnapshot(snapshotInfo); - } catch (Exception e) { logger.debug("Failed to backup Snapshot from Instance Snapshot", e); _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.snapshot); @@ -771,12 +779,11 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _accountMgr.checkAccess(caller, null, true, snapshotCheck); SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshotCheck, zoneId, SnapshotOperation.DELETE); - if (snapshotStrategy == null) { logger.error("Unable to find snapshot strategy to handle snapshot [{}]", snapshotCheck); - return false; } + Pair, List> storeRefAndZones = getStoreRefsAndZonesForSnapshotDelete(snapshotId, zoneId); List snapshotStoreRefs = storeRefAndZones.first(); List zoneIds = storeRefAndZones.second(); @@ -1472,8 +1479,9 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); + ResourceType storeResourceType = dataStoreRole == DataStoreRole.Image ? ResourceType.secondary_storage : ResourceType.primary_storage; // Correct the resource count of snapshot in case of delta snapshots. - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); if (!payload.getAsyncBackup() && backupSnapToSecondary) { copyNewSnapshotToZones(snapshotId, snapshot.getDataCenterId(), payload.getZoneIds()); @@ -1485,15 +1493,17 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement if (logger.isDebugEnabled()) { logger.debug("Failed to create snapshot" + cre.getLocalizedMessage()); } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType()); _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize())); throw cre; } catch (Exception e) { if (logger.isDebugEnabled()) { logger.debug("Failed to create snapshot", e); } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType()); _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize())); throw new CloudRuntimeException("Failed to create snapshot", e); } return snapshot; @@ -1695,11 +1705,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement Type snapshotType = getSnapshotType(policyId); Account owner = _accountMgr.getAccount(volume.getAccountId()); - ResourceType storeResourceType = ResourceType.secondary_storage; - if (!isBackupSnapshotToSecondaryForZone(volume.getDataCenterId()) || - Snapshot.LocationType.PRIMARY.equals(locationType)) { - storeResourceType = ResourceType.primary_storage; - } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), locationType); try { _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); _resourceLimitMgr.checkResourceLimit(owner, storeResourceType, volume.getSize()); diff --git a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index 34030626d22..53ccc830dd2 100644 --- a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -118,6 +119,8 @@ public class ResourceLimitManagerImplTest extends TestCase { VolumeDao volumeDao; @Mock UserVmDao userVmDao; + @Mock + SnapshotDataStoreDao snapshotDataStoreDao; private List hostTags = List.of("htag1", "htag2", "htag3"); private List storageTags = List.of("stag1", "stag2"); @@ -840,12 +843,13 @@ public class ResourceLimitManagerImplTest extends TestCase { String tag = null; Mockito.when(vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId)) .thenReturn(List.of(1L)); + Mockito.when(snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId)).thenReturn(100L); Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(100L); - Assert.assertEquals(100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); tag = ""; Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(200L); - Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals(300L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); tag = "tag"; VolumeVO vol = Mockito.mock(VolumeVO.class); @@ -853,7 +857,7 @@ public class ResourceLimitManagerImplTest extends TestCase { Mockito.when(vol.getSize()).thenReturn(size); List vols = List.of(vol, vol); Mockito.doReturn(vols).when(resourceLimitManager).getVolumesWithAccountAndTag(accountId, tag); - Assert.assertEquals(vols.size() * size, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals((vols.size() * size) + 100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); } @Test From 8c2ba2b34119f570bdedee750b5aa762a634a3a5 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 28 Jan 2026 16:46:34 +0530 Subject: [PATCH 06/23] ui: bump nodejs v24 LTS usage (#12471) --- ui/README.md | 6 +++--- ui/package.json | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ui/README.md b/ui/README.md index 170232b574e..3f7bcb8120e 100644 --- a/ui/README.md +++ b/ui/README.md @@ -27,18 +27,18 @@ A modern role-based progressive CloudStack UI based on Vue.js and Ant Design. Install node: (Debian/Ubuntu) - curl -sL https://deb.nodesource.com/setup_20.x | sudo -E bash - + curl -sL https://deb.nodesource.com/setup_24.x | sudo -E bash - sudo apt-get install -y nodejs # Or use distro provided: sudo apt-get install npm nodejs Install node: (CentOS/Fedora/RHEL) - curl -sL https://rpm.nodesource.com/setup_20.x | sudo bash - + curl -sL https://rpm.nodesource.com/setup_24.x | sudo bash - sudo yum install nodejs Install node: (Mac OS) - brew install node@20 + brew install node@24 Optionally, you may also install system-wide dev tools: diff --git a/ui/package.json b/ui/package.json index 48f337500bd..9801c1b1815 100644 --- a/ui/package.json +++ b/ui/package.json @@ -101,15 +101,18 @@ "eslint-plugin-vue": "^7.0.0", "less": "^3.0.4", "less-loader": "^5.0.0", + "nan": "2.18.0", + "node-gyp": "10.0.1", "sass": "^1.49.9", "sass-loader": "^8.0.2", "uglifyjs-webpack-plugin": "^2.2.0", "vue-jest": "^5.0.0-0", "vue-svg-loader": "^0.17.0-beta.2", - "webpack": "^4.46.0", - "node-gyp": "10.0.1", "nan": "2.18.0" + "webpack": "^4.46.0" + }, + "resolutions": { + "nan": "2.18.0" }, - "resolutions": { "nan": "2.18.0" }, "eslintConfig": { "root": true, "env": { From 35e6d7c5ba8f921c7611ab6c0c84d7b3db557456 Mon Sep 17 00:00:00 2001 From: Edward-x <30854794+YLChen-007@users.noreply.github.com> Date: Wed, 28 Jan 2026 19:16:59 +0800 Subject: [PATCH 07/23] fix that log sensitive infomation in cmd of script (#12024) * fix that log sensitive infomation in cmd of script * Remove unnecessary line break in Script.java * Update utils/src/main/java/com/cloud/utils/script/Script.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Refactor logging in Script class to simplify handling of sensitive arguments * Improve command logging in Script class to include full command line when debugging * Remove unused _passwordCommand flag from Script class to simplify code * Update utils/src/main/java/com/cloud/utils/script/Script.java Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Remove unused import for KeyStoreUtils * Update utils/src/main/java/com/cloud/utils/script/Script.java --------- Co-authored-by: chenyoulong20g@ict.ac.cn Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: dahn Co-authored-by: dahn --- ...bvirtUpdateHostPasswordCommandWrapper.java | 3 +- ...itrixUpdateHostPasswordCommandWrapper.java | 2 +- .../java/com/cloud/utils/script/Script.java | 99 +++++++++++-------- .../com/cloud/utils/script/ScriptTest.java | 30 ++++++ 4 files changed, 93 insertions(+), 41 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateHostPasswordCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateHostPasswordCommandWrapper.java index b8fe0ded716..80c723b5a6e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateHostPasswordCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateHostPasswordCommandWrapper.java @@ -37,7 +37,8 @@ public final class LibvirtUpdateHostPasswordCommandWrapper extends CommandWrappe final String newPassword = command.getNewPassword(); final Script script = libvirtUtilitiesHelper.buildScript(libvirtComputingResource.getUpdateHostPasswdPath()); - script.add(username, newPassword); + script.add(username); + script.addSensitive(newPassword); final String result = script.execute(); if (result != null) { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java index af868d8c1c7..e3ee0ca13ca 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java @@ -47,7 +47,7 @@ public final class CitrixUpdateHostPasswordCommandWrapper extends CommandWrapper try { logger.debug("Executing password update command on host: {} for user: {}", hostIp, username); final String hostPassword = citrixResourceBase.getPwdFromQueue(); - result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine.toString()); + result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine); } catch (final Exception e) { return new Answer(command, false, e.getMessage()); } diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index ffda782edda..09c58dce9a8 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -30,8 +30,10 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Properties; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -43,7 +45,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; @@ -66,8 +67,8 @@ public class Script implements Callable { private static final int DEFAULT_TIMEOUT = 3600 * 1000; /* 1 hour */ private volatile boolean _isTimeOut = false; - private boolean _passwordCommand = false; private boolean avoidLoggingCommand = false; + private final Set sensitiveArgIndices = new HashSet<>(); private static final ScheduledExecutorService s_executors = Executors.newScheduledThreadPool(10, new NamedThreadFactory("Script")); @@ -145,6 +146,11 @@ public class Script implements Callable { _command.add(param); } + public void addSensitive(String param) { + _command.add(param); + sensitiveArgIndices.add(_command.size() - 1); + } + public Script set(String name, String value) { _command.add(name); _command.add(value); @@ -163,7 +169,7 @@ public class Script implements Callable { if (sanitizeViCmdParameter(cmd, builder) || sanitizeRbdFileFormatCmdParameter(cmd, builder)) { continue; } - if (obscureParam) { + if (obscureParam || sensitiveArgIndices.contains(i)) { builder.append("******").append(" "); obscureParam = false; } else { @@ -172,7 +178,6 @@ public class Script implements Callable { if ("-y".equals(cmd) || "-z".equals(cmd)) { obscureParam = true; - _passwordCommand = true; } } return builder.toString(); @@ -240,8 +245,8 @@ public class Script implements Callable { public String execute(OutputInterpreter interpreter) { String[] command = _command.toArray(new String[_command.size()]); String commandLine = buildCommandLine(command); - if (_logger.isDebugEnabled() && !avoidLoggingCommand) { - _logger.debug(String.format("Executing command [%s].", commandLine.split(KeyStoreUtils.KS_FILENAME)[0])); + if (_logger.isDebugEnabled() ) { + _logger.debug(String.format("Executing command [%s].", commandLine)); } try { @@ -263,48 +268,62 @@ public class Script implements Callable { _thread = Thread.currentThread(); ScheduledFuture future = null; if (_timeout > 0) { - _logger.trace(String.format("Scheduling the execution of command [%s] with a timeout of [%s] milliseconds.", commandLine, _timeout)); + _logger.trace(String.format( + "Scheduling the execution of command [%s] with a timeout of [%s] milliseconds.", + commandLine, _timeout)); future = s_executors.schedule(this, _timeout, TimeUnit.MILLISECONDS); } long processPid = _process.pid(); Task task = null; if (interpreter != null && interpreter.drain()) { - _logger.trace(String.format("Executing interpreting task of process [%s] for command [%s].", processPid, commandLine)); + _logger.trace(String.format("Executing interpreting task of process [%s] for command [%s].", + processPid, commandLine)); task = new Task(interpreter, ir); s_executors.execute(task); } while (true) { - _logger.trace(String.format("Attempting process [%s] execution for command [%s] with timeout [%s].", processPid, commandLine, _timeout)); + _logger.trace(String.format("Attempting process [%s] execution for command [%s] with timeout [%s].", + processPid, commandLine, _timeout)); try { if (_process.waitFor(_timeout, TimeUnit.MILLISECONDS)) { - _logger.trace(String.format("Process [%s] execution for command [%s] completed within timeout period [%s].", processPid, commandLine, + _logger.trace(String.format( + "Process [%s] execution for command [%s] completed within timeout period [%s].", + processPid, commandLine, _timeout)); if (_process.exitValue() == 0) { - _logger.debug(String.format("Successfully executed process [%s] for command [%s].", processPid, commandLine)); + _logger.debug(String.format("Successfully executed process [%s] for command [%s].", + processPid, commandLine)); if (interpreter != null) { if (interpreter.drain()) { - _logger.trace(String.format("Returning task result of process [%s] for command [%s].", processPid, commandLine)); + _logger.trace( + String.format("Returning task result of process [%s] for command [%s].", + processPid, commandLine)); return task.getResult(); } - _logger.trace(String.format("Returning interpretation of process [%s] for command [%s].", processPid, commandLine)); + _logger.trace( + String.format("Returning interpretation of process [%s] for command [%s].", + processPid, commandLine)); return interpreter.interpret(ir); } else { // null return exitValue apparently - _logger.trace(String.format("Process [%s] for command [%s] exited with value [%s].", processPid, commandLine, + _logger.trace(String.format("Process [%s] for command [%s] exited with value [%s].", + processPid, commandLine, _process.exitValue())); return String.valueOf(_process.exitValue()); } } else { - _logger.warn(String.format("Execution of process [%s] for command [%s] failed.", processPid, commandLine)); + _logger.warn(String.format("Execution of process [%s] for command [%s] failed.", + processPid, commandLine)); break; } } } catch (InterruptedException e) { if (!_isTimeOut) { - _logger.debug(String.format("Exception [%s] occurred; however, it was not a timeout. Therefore, proceeding with the execution of process [%s] for command " - + "[%s].", e.getMessage(), processPid, commandLine), e); + _logger.debug(String.format( + "Exception [%s] occurred; however, it was not a timeout. Therefore, proceeding with the execution of process [%s] for command [%s].", + e.getMessage(), processPid, commandLine), e); continue; } } finally { @@ -317,18 +336,17 @@ public class Script implements Callable { TimedOutLogger log = new TimedOutLogger(_process); Task timedoutTask = new Task(log, ir); - _logger.trace(String.format("Running timed out task of process [%s] for command [%s].", processPid, commandLine)); + _logger.trace(String.format("Running timed out task of process [%s] for command [%s].", processPid, + commandLine)); timedoutTask.run(); - if (!_passwordCommand) { - _logger.warn(String.format("Process [%s] for command [%s] timed out. Output is [%s].", processPid, commandLine, timedoutTask.getResult())); - } else { - _logger.warn(String.format("Process [%s] for command [%s] timed out.", processPid, commandLine)); - } + _logger.warn(String.format("Process [%s] for command [%s] timed out. Output is [%s].", processPid, + commandLine, timedoutTask.getResult())); return ERR_TIMEOUT; } - _logger.debug(String.format("Exit value of process [%s] for command [%s] is [%s].", processPid, commandLine, _process.exitValue())); + _logger.debug(String.format("Exit value of process [%s] for command [%s] is [%s].", processPid, + commandLine, _process.exitValue())); BufferedReader reader = new BufferedReader(new InputStreamReader(_process.getInputStream()), 128); @@ -339,19 +357,24 @@ public class Script implements Callable { error = String.valueOf(_process.exitValue()); } - _logger.warn(String.format("Process [%s] for command [%s] encountered the error: [%s].", processPid, commandLine, error)); + _logger.warn(String.format("Process [%s] for command [%s] encountered the error: [%s].", processPid, + commandLine, error)); return error; } catch (SecurityException ex) { - _logger.warn(String.format("Exception [%s] occurred. This may be due to an attempt of executing command [%s] as non root.", ex.getMessage(), commandLine), + _logger.warn(String.format( + "Exception [%s] occurred. This may be due to an attempt of executing command [%s] as non root.", + ex.getMessage(), commandLine), ex); return stackTraceAsString(ex); } catch (Exception ex) { - _logger.warn(String.format("Exception [%s] occurred when attempting to run command [%s].", ex.getMessage(), commandLine), ex); + _logger.warn(String.format("Exception [%s] occurred when attempting to run command [%s].", + ex.getMessage(), commandLine), ex); return stackTraceAsString(ex); } finally { if (_process != null) { - _logger.trace(String.format("Destroying process [%s] for command [%s].", _process.pid(), commandLine)); + _logger.trace( + String.format("Destroying process [%s] for command [%s].", _process.pid(), commandLine)); IOUtils.closeQuietly(_process.getErrorStream()); IOUtils.closeQuietly(_process.getOutputStream()); IOUtils.closeQuietly(_process.getInputStream()); @@ -362,9 +385,10 @@ public class Script implements Callable { public String executeIgnoreExitValue(OutputInterpreter interpreter, int exitValue) { String[] command = _command.toArray(new String[_command.size()]); + String commandLine = buildCommandLine(command); if (_logger.isDebugEnabled()) { - _logger.debug(String.format("Executing: %s", buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0])); + _logger.debug(String.format("Executing: %s", commandLine)); } try { @@ -375,7 +399,7 @@ public class Script implements Callable { _process = pb.start(); if (_process == null) { - _logger.warn(String.format("Unable to execute: %s", buildCommandLine(command))); + _logger.warn(String.format("Unable to execute: %s", commandLine)); return String.format("Unable to execute the command: %s", command[0]); } @@ -439,11 +463,8 @@ public class Script implements Callable { Task timedoutTask = new Task(log, ir); timedoutTask.run(); - if (!_passwordCommand) { - _logger.warn(String.format("Timed out: %s. Output is: %s", buildCommandLine(command), timedoutTask.getResult())); - } else { - _logger.warn(String.format("Timed out: %s", buildCommandLine(command))); - } + _logger.warn(String.format("Timed out: %s. Output is: %s", commandLine, + timedoutTask.getResult())); return ERR_TIMEOUT; } @@ -467,7 +488,7 @@ public class Script implements Callable { _logger.warn("Security Exception....not running as root?", ex); return stackTraceAsString(ex); } catch (Exception ex) { - _logger.warn(String.format("Exception: %s", buildCommandLine(command)), ex); + _logger.warn(String.format("Exception: %s", commandLine), ex); return stackTraceAsString(ex); } finally { if (_process != null) { @@ -516,9 +537,9 @@ public class Script implements Callable { } catch (Exception ex) { result = stackTraceAsString(ex); } finally { - done = true; - notifyAll(); - IOUtils.closeQuietly(reader); + done = true; + notifyAll(); + IOUtils.closeQuietly(reader); } } } diff --git a/utils/src/test/java/com/cloud/utils/script/ScriptTest.java b/utils/src/test/java/com/cloud/utils/script/ScriptTest.java index cc6047959da..a52f3840bea 100644 --- a/utils/src/test/java/com/cloud/utils/script/ScriptTest.java +++ b/utils/src/test/java/com/cloud/utils/script/ScriptTest.java @@ -78,4 +78,34 @@ public class ScriptTest { String result = Script.getExecutableAbsolutePath("ls"); Assert.assertTrue(List.of("/usr/bin/ls", "/bin/ls").contains(result)); } + + @Test + public void testBuildCommandLineWithSensitiveData() { + Script script = new Script("test.sh"); + script.add("normal-arg"); + script.addSensitive("sensitive-arg"); + String commandLine = script.toString(); + Assert.assertEquals("test.sh normal-arg ****** ", commandLine); + } + + @Test + public void testBuildCommandLineWithMultipleSensitiveData() { + Script script = new Script("test.sh"); + script.add("normal-arg"); + script.addSensitive("sensitive-arg1"); + script.add("another-normal-arg"); + script.addSensitive("sensitive-arg2"); + String commandLine = script.toString(); + Assert.assertEquals("test.sh normal-arg ****** another-normal-arg ****** ", commandLine); + } + + @Test + public void testBuildCommandLineWithLegacyPasswordOption() { + Script script = new Script("test.sh"); + script.add("-y"); + script.add("legacy-password"); + String commandLine = script.toString(); + Assert.assertEquals("test.sh -y ****** ", commandLine); + } + } From 4d35d68e4ef77a6e034f2464ab92976a175bc11c Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 28 Jan 2026 17:17:50 +0530 Subject: [PATCH 08/23] Fix url in password reset email (#12078) --- .../org/apache/cloudstack/ServerDaemon.java | 15 +++----- .../user/UserPasswordResetManager.java | 4 ++- .../user/UserPasswordResetManagerImpl.java | 26 +++++++++++--- .../cloud/utils/server/ServerProperties.java | 36 +++++++++++++++++-- 4 files changed, 64 insertions(+), 17 deletions(-) diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 09bdb11a6b3..06477fff898 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -71,11 +71,6 @@ public class ServerDaemon implements Daemon { private static final String BIND_INTERFACE = "bind.interface"; private static final String CONTEXT_PATH = "context.path"; private static final String SESSION_TIMEOUT = "session.timeout"; - private static final String HTTP_ENABLE = "http.enable"; - private static final String HTTP_PORT = "http.port"; - private static final String HTTPS_ENABLE = "https.enable"; - private static final String HTTPS_PORT = "https.port"; - private static final String KEYSTORE_FILE = "https.keystore"; private static final String KEYSTORE_PASSWORD = "https.keystore.password"; private static final String WEBAPP_DIR = "webapp.dir"; private static final String ACCESS_LOG = "access.log"; @@ -137,11 +132,11 @@ public class ServerDaemon implements Daemon { } setBindInterface(properties.getProperty(BIND_INTERFACE, null)); setContextPath(properties.getProperty(CONTEXT_PATH, "/client")); - setHttpEnable(Boolean.valueOf(properties.getProperty(HTTP_ENABLE, "true"))); - setHttpPort(Integer.valueOf(properties.getProperty(HTTP_PORT, "8080"))); - setHttpsEnable(Boolean.valueOf(properties.getProperty(HTTPS_ENABLE, "false"))); - setHttpsPort(Integer.valueOf(properties.getProperty(HTTPS_PORT, "8443"))); - setKeystoreFile(properties.getProperty(KEYSTORE_FILE)); + setHttpEnable(Boolean.valueOf(properties.getProperty(ServerProperties.HTTP_ENABLE, "true"))); + setHttpPort(Integer.valueOf(properties.getProperty(ServerProperties.HTTP_PORT, "8080"))); + setHttpsEnable(Boolean.valueOf(properties.getProperty(ServerProperties.HTTPS_ENABLE, "false"))); + setHttpsPort(Integer.valueOf(properties.getProperty(ServerProperties.HTTPS_PORT, "8443"))); + setKeystoreFile(properties.getProperty(ServerProperties.KEYSTORE_FILE)); setKeystorePassword(properties.getProperty(KEYSTORE_PASSWORD)); setWebAppLocation(properties.getProperty(WEBAPP_DIR)); setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log")); diff --git a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java index 377e57b31e9..ca14f6a1654 100644 --- a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java +++ b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java @@ -78,7 +78,9 @@ public interface UserPasswordResetManager { ConfigKey UserPasswordResetDomainURL = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, String.class, "user.password.reset.mail.domain.url", null, - "Domain URL for reset password links sent to the user via email", true, + "Domain URL (along with scheme - http:// or https:// and port as applicable) for reset password links sent to the user via email. " + + "If this is not set, CloudStack would determine the domain url based on the first management server from 'host' setting " + + "and http scheme based on the https.enabled flag from server.properties file in the management server.", true, ConfigKey.Scope.Global); void setResetTokenAndSend(UserAccount userAccount); diff --git a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java index 618ad5c8657..c62bca8eca4 100644 --- a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java @@ -23,6 +23,7 @@ import com.cloud.user.UserVO; import com.cloud.user.dao.UserDao; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.server.ServerProperties; import com.github.mustachejava.DefaultMustacheFactory; import com.github.mustachejava.Mustache; import com.github.mustachejava.MustacheFactory; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import static org.apache.cloudstack.config.ApiServiceConfiguration.ManagementServerAddresses; import static org.apache.cloudstack.resourcedetail.UserDetailVO.PasswordResetToken; import static org.apache.cloudstack.resourcedetail.UserDetailVO.PasswordResetTokenExpiryDate; @@ -68,7 +70,7 @@ public class UserPasswordResetManagerImpl extends ManagerBase implements UserPas new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, String.class, "user.password.reset.mail.template", "Hello {{username}}!\n" + "You have requested to reset your password. Please click the following link to reset your password:\n" + - "{{{domainUrl}}}{{{resetLink}}}\n" + + "{{{resetLink}}}\n" + "If you did not request a password reset, please ignore this email.\n" + "\n" + "Regards,\n" + @@ -179,10 +181,26 @@ public class UserPasswordResetManagerImpl extends ManagerBase implements UserPas final String email = userAccount.getEmail(); final String username = userAccount.getUsername(); final String subject = "Password Reset Request"; - final String domainUrl = UserPasswordResetDomainURL.value(); + String domainUrl = UserPasswordResetDomainURL.value(); + if (StringUtils.isBlank(domainUrl)) { + String mgmtServerAddr = ManagementServerAddresses.value().split(",")[0]; + if (ServerProperties.isHttpsEnabled()) { + domainUrl = "https://" + mgmtServerAddr + ":" + ServerProperties.getHttpsPort(); + } else { + domainUrl = "http://" + mgmtServerAddr + ":" + ServerProperties.getHttpPort(); + } + } else if (!domainUrl.startsWith("http://") && !domainUrl.startsWith("https://")) { + if (ServerProperties.isHttpsEnabled()) { + domainUrl = "https://" + domainUrl; + } else { + domainUrl = "http://" + domainUrl; + } + } - String resetLink = String.format("/client/#/user/resetPassword?username=%s&token=%s", - username, resetToken); + domainUrl = domainUrl.replaceAll("/+$", ""); + + String resetLink = String.format("%s/client/#/user/resetPassword?username=%s&token=%s", + domainUrl, username, resetToken); String content = getMessageBody(userAccount, resetToken, resetLink); SMTPMailProperties mailProperties = new SMTPMailProperties(); diff --git a/utils/src/main/java/com/cloud/utils/server/ServerProperties.java b/utils/src/main/java/com/cloud/utils/server/ServerProperties.java index 36d8614e68f..9e81fff90f0 100644 --- a/utils/src/main/java/com/cloud/utils/server/ServerProperties.java +++ b/utils/src/main/java/com/cloud/utils/server/ServerProperties.java @@ -17,10 +17,12 @@ package com.cloud.utils.server; import com.cloud.utils.crypt.EncryptionSecretKeyChecker; +import com.cloud.utils.StringUtils; import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Properties; @@ -28,9 +30,20 @@ import java.util.Properties; public class ServerProperties { protected Logger logger = LogManager.getLogger(getClass()); + public static final String HTTP_ENABLE = "http.enable"; + public static final String HTTP_PORT = "http.port"; + public static final String HTTPS_ENABLE = "https.enable"; + public static final String HTTPS_PORT = "https.port"; + public static final String KEYSTORE_FILE = "https.keystore"; + public static final String PASSWORD_ENCRYPTION_TYPE = "password.encryption.type"; + private static Properties properties = new Properties(); private static boolean loaded = false; - public static final String passwordEncryptionType = "password.encryption.type"; + + private static int httpPort = 8080; + + private static boolean httpsEnable = false; + private static int httpsPort = 8443; public synchronized static Properties getServerProperties(InputStream inputStream) { if (!loaded) { @@ -39,7 +52,7 @@ public class ServerProperties { serverProps.load(inputStream); EncryptionSecretKeyChecker checker = new EncryptionSecretKeyChecker(); - checker.check(serverProps, passwordEncryptionType); + checker.check(serverProps, PASSWORD_ENCRYPTION_TYPE); if (EncryptionSecretKeyChecker.useEncryption()) { EncryptionSecretKeyChecker.decryptAnyProperties(serverProps); @@ -50,10 +63,29 @@ public class ServerProperties { IOUtils.closeQuietly(inputStream); } + httpPort = Integer.parseInt(serverProps.getProperty(ServerProperties.HTTP_PORT, "8080")); + + boolean httpsEnabled = Boolean.parseBoolean(serverProps.getProperty(ServerProperties.HTTPS_ENABLE, "false")); + String keystoreFile = serverProps.getProperty(KEYSTORE_FILE); + httpsEnable = httpsEnabled && StringUtils.isNotEmpty(keystoreFile) && new File(keystoreFile).exists(); + httpsPort = Integer.parseInt(serverProps.getProperty(ServerProperties.HTTPS_PORT, "8443")); + properties = serverProps; loaded = true; } return properties; } + + public static int getHttpPort() { + return httpPort; + } + + public static boolean isHttpsEnabled() { + return httpsEnable; + } + + public static int getHttpsPort() { + return httpsPort; + } } From 6a04e14f8767a90b5d709e3105f7f47500b4d21c Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 28 Jan 2026 13:09:10 +0100 Subject: [PATCH 09/23] VR: fix dns list in redundant VPC VRs (#12161) --- systemvm/debian/opt/cloud/bin/cs/CsDhcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index e15714af212..a2309067289 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -110,7 +110,7 @@ class CsDhcp(CsDataBag): if gn.get_dns() and device: sline = "dhcp-option=tag:interface-%s-%s,6" % (device, idx) dns_list = [x for x in gn.get_dns() if x] - if (self.config.is_vpc() or self.config.is_router()) and ('is_vr_guest_gateway' in gn.data and gn.data['is_vr_guest_gateway']): + if self.config.is_vpc() and not gn.is_vr_guest_gateway(): if gateway in dns_list: dns_list.remove(gateway) if gn.data['router_guest_ip'] != ip: From 9fc93af85fb34b480065b975257992b1c5631fcd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 28 Jan 2026 19:36:04 +0530 Subject: [PATCH 10/23] ui: allow actions for other users of root admin (#11319) Fixes #10306 Signed-off-by: Abhishek Kumar --- ui/src/config/section/user.js | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/ui/src/config/section/user.js b/ui/src/config/section/user.js index a18994fd6ce..65c1a17f760 100644 --- a/ui/src/config/section/user.js +++ b/ui/src/config/section/user.js @@ -105,9 +105,10 @@ export default { message: 'message.enable.user', dataView: true, show: (record, store) => { - return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault && - !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) && - ['disabled', 'locked'].includes(record.state) + if (!['disabled', 'locked'].includes(record.state) || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + return false + } + return ![1, 4].includes(record.accounttype) || store.userInfo.roletype === 'Admin' } }, { @@ -117,9 +118,10 @@ export default { message: 'message.disable.user', dataView: true, show: (record, store) => { - return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault && - !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) && - record.state === 'enabled' + if (record.state !== 'enabled' || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + return false + } + return ![1, 4].includes(record.accounttype) || (store.userInfo.roletype === 'Admin' && record.id !== store.userInfo.id) } }, { @@ -131,9 +133,10 @@ export default { dataView: true, popup: true, show: (record, store) => { - return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault && - !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) && - record.state === 'enabled' + if (record.state !== 'enabled' || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + return false + } + return ![1, 4].includes(record.accounttype) || (store.userInfo.roletype === 'Admin' && record.id !== store.userInfo.id) } }, { From 95de88a8ffee115a98fd34818394d420b01f8cdf Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 28 Jan 2026 19:38:25 +0530 Subject: [PATCH 11/23] Usage server should takeover immediately if the other Usage server has been stopped gracefully (#12507) --- .../src/main/java/com/cloud/usage/dao/UsageJobDao.java | 2 ++ .../src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java | 3 ++- usage/src/main/java/com/cloud/usage/UsageManagerImpl.java | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java index d4038d4ceeb..b22ce69d94e 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java @@ -28,6 +28,8 @@ public interface UsageJobDao extends GenericDao { UsageJobVO getLastJob(); + UsageJobVO getNextRecurringJob(); + UsageJobVO getNextImmediateJob(); long getLastJobSuccessDateMillis(); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java index 44a7d1a8b72..6f340501cf1 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java @@ -156,7 +156,8 @@ public class UsageJobDaoImpl extends GenericDaoBase implements return jobs.get(0); } - private UsageJobVO getNextRecurringJob() { + @Override + public UsageJobVO getNextRecurringJob() { Filter filter = new Filter(UsageJobVO.class, "id", false, Long.valueOf(0), Long.valueOf(1)); SearchCriteria sc = createSearchCriteria(); sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0)); diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 99de98f56e4..30cdfcf21f0 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -2257,6 +2257,11 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna } } + if (_usageJobDao.getNextRecurringJob() == null) { + // Own the usage processing immediately if no other node is owning it + _usageJobDao.createNewJob(_hostname, _pid, UsageJobVO.JOB_TYPE_RECURRING); + } + Long jobId = _usageJobDao.checkHeartbeat(_hostname, _pid, _aggregationDuration); if (jobId != null) { // if I'm taking over the job...see how long it's been since the last job, and if it's more than the From 059debf212500c789d062abe7be00448a1744824 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 28 Jan 2026 19:39:37 +0530 Subject: [PATCH 12/23] Add the procedure files for insert extensions and update guest os category (#12482) * Add the procedure files for insert extensions and update guestos category * fixed indentation * Apply suggestions from code review Co-authored-by: Vishesh <8760112+vishesh92@users.noreply.github.com> --------- Co-authored-by: Vishesh <8760112+vishesh92@users.noreply.github.com> --- .../cloud.insert_category_if_not_exists.sql | 27 +++++++++++ ...on_custom_action_details_if_not_exists.sql | 46 +++++++++++++++++++ ..._extension_custom_action_if_not_exists.sql | 46 +++++++++++++++++++ ....insert_extension_detail_if_not_exists.sql | 39 ++++++++++++++++ .../cloud.insert_extension_if_not_exists.sql | 38 +++++++++++++++ .../cloud.update_category_for_guest_oses.sql | 33 +++++++++++++ ...w_and_delete_old_category_for_guest_os.sql | 35 ++++++++++++++ 7 files changed, 264 insertions(+) create mode 100644 engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql new file mode 100644 index 00000000000..a82dc7204c2 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Add new OS categories if not present +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`; +CREATE PROCEDURE `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`(IN os_name VARCHAR(255)) +BEGIN + IF NOT EXISTS ((SELECT 1 FROM `cloud`.`guest_os_category` WHERE name = os_name)) + THEN + INSERT INTO `cloud`.`guest_os_category` (name, uuid) + VALUES (os_name, UUID()) +; END IF +; END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql new file mode 100644 index 00000000000..77b16223626 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_DETAILS_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_DETAILS_IF_NOT_EXISTS` ( + IN ext_name VARCHAR(255), + IN action_name VARCHAR(255), + IN param_json TEXT +) +BEGIN + DECLARE action_id BIGINT UNSIGNED +; SELECT `eca`.`id` INTO action_id FROM `cloud`.`extension_custom_action` `eca` + JOIN `cloud`.`extension` `e` ON `e`.`id` = `eca`.`extension_id` + WHERE `eca`.`name` = action_name AND `e`.`name` = ext_name LIMIT 1 +; IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension_custom_action_details` + WHERE `extension_custom_action_id` = action_id + AND `name` = 'parameters' + ) THEN + INSERT INTO `cloud`.`extension_custom_action_details` ( + `extension_custom_action_id`, + `name`, + `value`, + `display` + ) VALUES ( + action_id, + 'parameters', + param_json, + 0 + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql new file mode 100644 index 00000000000..9dbffa630f8 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_IF_NOT_EXISTS`( + IN ext_name VARCHAR(255), + IN action_name VARCHAR(255), + IN action_desc VARCHAR(4096), + IN resource_type VARCHAR(255), + IN allowed_roles INT UNSIGNED, + IN success_msg VARCHAR(4096), + IN error_msg VARCHAR(4096), + IN timeout_seconds INT UNSIGNED +) +BEGIN + DECLARE ext_id BIGINT +; SELECT `id` INTO ext_id FROM `cloud`.`extension` WHERE `name` = ext_name LIMIT 1 +; IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension_custom_action` WHERE `name` = action_name AND `extension_id` = ext_id + ) THEN + INSERT INTO `cloud`.`extension_custom_action` ( + `uuid`, `name`, `description`, `extension_id`, `resource_type`, + `allowed_role_types`, `success_message`, `error_message`, + `enabled`, `timeout`, `created`, `removed` + ) + VALUES ( + UUID(), action_name, action_desc, ext_id, resource_type, + allowed_roles, success_msg, error_msg, + 1, timeout_seconds, NOW(), NULL + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql new file mode 100644 index 00000000000..f9d6c5da951 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`( + IN ext_name VARCHAR(255), + IN detail_key VARCHAR(255), + IN detail_value TEXT, + IN display TINYINT(1) +) +BEGIN + DECLARE ext_id BIGINT +; SELECT `id` INTO ext_id FROM `cloud`.`extension` WHERE `name` = ext_name LIMIT 1 +; IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension_details` + WHERE `extension_id` = ext_id AND `name` = detail_key + ) THEN + INSERT INTO `cloud`.`extension_details` ( + `extension_id`, `name`, `value`, `display` + ) + VALUES ( + ext_id, detail_key, detail_value, display + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql new file mode 100644 index 00000000000..8d74f9b2a98 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_IF_NOT_EXISTS`( + IN ext_name VARCHAR(255), + IN ext_desc VARCHAR(255), + IN ext_path VARCHAR(255) +) +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension` WHERE `name` = ext_name + ) THEN + INSERT INTO `cloud`.`extension` ( + `uuid`, `name`, `description`, `type`, + `relative_path`, `path_ready`, + `is_user_defined`, `state`, `created`, `removed` + ) + VALUES ( + UUID(), ext_name, ext_desc, 'Orchestrator', + ext_path, 1, 0, 'Enabled', NOW(), NULL + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql new file mode 100644 index 00000000000..87f3a85d27e --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Move existing guest OS to new categories +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`; +CREATE PROCEDURE `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`(IN category_name VARCHAR(255), IN os_name VARCHAR(255)) +BEGIN + DECLARE category_id BIGINT +; SELECT `id` INTO category_id + FROM `cloud`.`guest_os_category` + WHERE `name` = category_name + LIMIT 1 +; IF category_id IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Category not found' +; END IF +; UPDATE `cloud`.`guest_os` + SET `category_id` = category_id + WHERE `display_name` LIKE CONCAT('%', os_name, '%') +; END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql new file mode 100644 index 00000000000..42f7aa738cf --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Move existing guest OS whose category will be deleted to Other category +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`; +CREATE PROCEDURE `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`(IN to_category_name VARCHAR(255), IN from_category_name VARCHAR(255)) +BEGIN + DECLARE done INT DEFAULT 0 +; DECLARE to_category_id BIGINT +; SELECT id INTO to_category_id + FROM `cloud`.`guest_os_category` + WHERE `name` = to_category_name + LIMIT 1 +; IF to_category_id IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'ToCategory not found' +; END IF +; UPDATE `cloud`.`guest_os` + SET `category_id` = to_category_id + WHERE `category_id` = (SELECT `id` FROM `cloud`.`guest_os_category` WHERE `name` = from_category_name) +; UPDATE `cloud`.`guest_os_category` SET `removed`=now() WHERE `name` = from_category_name +; END; From 9956d325488246a1b6dd0d33398471c0c698de49 Mon Sep 17 00:00:00 2001 From: Daman Arora <61474540+Damans227@users.noreply.github.com> Date: Wed, 28 Jan 2026 09:11:14 -0500 Subject: [PATCH 13/23] Fix delete snapshot policy expunged volume (#12474) * use findByIdIncludingRemoved for volume retrieval in snapshot policy validation * add unit tests * add cleanup for orphan snapshot policies * delete snapshot policies when expunging volumes * update orphan cleanup to remove policies for volumes that are in expunged state or null --------- Co-authored-by: Daman Arora --- .../storage/volume/VolumeServiceImpl.java | 4 +- .../storage/snapshot/SnapshotManagerImpl.java | 18 +++- .../snapshot/SnapshotManagerImplTest.java | 92 +++++++++++++++++++ 3 files changed, 112 insertions(+), 2 deletions(-) diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index c59ae5b18e3..5e0eb7529ac 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -387,6 +387,7 @@ public class VolumeServiceImpl implements VolumeService { logger.info("Expunge volume with no data store specified"); if (canVolumeBeRemoved(volume.getId())) { logger.info("Volume {} is not referred anywhere, remove it from volumes table", volume); + snapshotMgr.deletePoliciesForVolume(volume.getId()); volDao.remove(volume.getId()); } future.complete(result); @@ -422,6 +423,7 @@ public class VolumeServiceImpl implements VolumeService { } VMTemplateVO template = templateDao.findById(vol.getTemplateId()); if (template != null && !template.isDeployAsIs()) { + snapshotMgr.deletePoliciesForVolume(vol.getId()); volDao.remove(vol.getId()); future.complete(result); return future; @@ -493,6 +495,7 @@ public class VolumeServiceImpl implements VolumeService { if (canVolumeBeRemoved(vo.getId())) { logger.info("Volume {} is not referred anywhere, remove it from volumes table", vo); + snapshotMgr.deletePoliciesForVolume(vo.getId()); volDao.remove(vo.getId()); } @@ -1657,7 +1660,6 @@ public class VolumeServiceImpl implements VolumeService { // mark volume entry in volumes table as destroy state VolumeInfo vol = volFactory.getVolume(volumeId); vol.stateTransit(Volume.Event.DestroyRequested); - snapshotMgr.deletePoliciesForVolume(volumeId); annotationDao.removeByEntityType(AnnotationService.EntityType.VOLUME.name(), vol.getUuid()); vol.stateTransit(Volume.Event.OperationSucceeded); diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index ff9989acac3..886feea19f2 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -1889,9 +1889,25 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement logger.debug("Failed to delete snapshot in destroying state: {}", snapshotVO); } } + cleanupOrphanSnapshotPolicies(); + return true; } + private void cleanupOrphanSnapshotPolicies() { + List policies = _snapshotPolicyDao.listActivePolicies(); + if (CollectionUtils.isEmpty(policies)) { + return; + } + for (SnapshotPolicyVO policy : policies) { + VolumeVO volume = _volsDao.findByIdIncludingRemoved(policy.getVolumeId()); + if (volume == null || volume.getState() == Volume.State.Expunged) { + logger.info("Removing orphan snapshot policy {} for non-existent volume {}", policy.getId(), policy.getVolumeId()); + deletePolicy(policy.getId()); + } + } + } + @Override public boolean stop() { backupSnapshotExecutor.shutdown(); @@ -1924,7 +1940,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement if (snapshotPolicyVO == null) { throw new InvalidParameterValueException("Policy id given: " + policy + " does not exist"); } - VolumeVO volume = _volsDao.findById(snapshotPolicyVO.getVolumeId()); + VolumeVO volume = _volsDao.findByIdIncludingRemoved(snapshotPolicyVO.getVolumeId()); if (volume == null) { throw new InvalidParameterValueException("Policy id given: " + policy + " does not belong to a valid volume"); } diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java index 367a49a801f..2d3cb04ab96 100644 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java @@ -30,6 +30,7 @@ import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.VolumeVO; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.SnapshotZoneDao; @@ -44,6 +45,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -100,6 +102,10 @@ public class SnapshotManagerImplTest { VolumeDao volumeDao; @Mock SnapshotPolicyDao snapshotPolicyDao; + @Mock + SnapshotScheduler snapshotScheduler; + @Mock + TaggedResourceService taggedResourceService; @InjectMocks SnapshotManagerImpl snapshotManager = new SnapshotManagerImpl(); @@ -108,6 +114,8 @@ public class SnapshotManagerImplTest { snapshotManager._snapshotPolicyDao = snapshotPolicyDao; snapshotManager._volsDao = volumeDao; snapshotManager._accountMgr = accountManager; + snapshotManager._snapSchedMgr = snapshotScheduler; + snapshotManager.taggedResourceService = taggedResourceService; } @After @@ -520,4 +528,88 @@ public class SnapshotManagerImplTest { Assert.assertEquals(1, result.first().size()); Assert.assertEquals(Integer.valueOf(1), result.second()); } + + @Test + public void testDeleteSnapshotPoliciesForRemovedVolume() { + Long policyId = 1L; + Long volumeId = 10L; + Long accountId = 2L; + + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(policyId); + Mockito.when(cmd.getIds()).thenReturn(null); + + Account caller = Mockito.mock(Account.class); + Mockito.when(caller.getId()).thenReturn(accountId); + CallContext.register(Mockito.mock(User.class), caller); + + SnapshotPolicyVO policyVO = Mockito.mock(SnapshotPolicyVO.class); + Mockito.when(policyVO.getId()).thenReturn(policyId); + Mockito.when(policyVO.getVolumeId()).thenReturn(volumeId); + Mockito.when(policyVO.getUuid()).thenReturn("policy-uuid"); + Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(policyVO); + + // Volume is removed (expunged) but findByIdIncludingRemoved should still return it + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeDao.findByIdIncludingRemoved(volumeId)).thenReturn(volumeVO); + + Mockito.when(snapshotPolicyDao.remove(policyId)).thenReturn(true); + + boolean result = snapshotManager.deleteSnapshotPolicies(cmd); + + Assert.assertTrue(result); + Mockito.verify(volumeDao).findByIdIncludingRemoved(volumeId); + Mockito.verify(snapshotScheduler).removeSchedule(volumeId, policyId); + Mockito.verify(snapshotPolicyDao).remove(policyId); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesNoPolicyId() { + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(null); + Mockito.when(cmd.getIds()).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesPolicyNotFound() { + Long policyId = 1L; + + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(policyId); + Mockito.when(cmd.getIds()).thenReturn(null); + + Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesVolumeNotFound() { + Long policyId = 1L; + Long volumeId = 10L; + + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(policyId); + Mockito.when(cmd.getIds()).thenReturn(null); + + SnapshotPolicyVO policyVO = Mockito.mock(SnapshotPolicyVO.class); + Mockito.when(policyVO.getVolumeId()).thenReturn(volumeId); + Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(policyVO); + + // Volume doesn't exist at all (even when including removed) + Mockito.when(volumeDao.findByIdIncludingRemoved(volumeId)).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesManualPolicyId() { + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(Snapshot.MANUAL_POLICY_ID); + Mockito.when(cmd.getIds()).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } } From 7786cf93c28fec5e5baf5d069f4f4d08e1750ab8 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Wed, 28 Jan 2026 09:13:56 -0500 Subject: [PATCH 14/23] Veeam: Use restore timeout as an interval as opposed to a counter (#11772) * Veeam: Use restore timeout as a time interval as opposed to a counter * fix log * fix unit test * remove unused imports * fix comment * unused import * change to while - issure refactoring --- .../org/apache/cloudstack/backup/veeam/VeeamClient.java | 7 +++++-- .../apache/cloudstack/backup/veeam/VeeamClientTest.java | 5 ++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index fe3633dab16..8a111f92868 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -364,7 +364,9 @@ public class VeeamClient { * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}. */ protected void checkIfRestoreSessionFinished(String type, String path) throws IOException { - for (int j = 0; j < restoreTimeout; j++) { + long startTime = System.currentTimeMillis(); + long timeoutMs = restoreTimeout * 1000L; + while (System.currentTimeMillis() - startTime < timeoutMs) { HttpResponse relatedResponse = get(path); RestoreSession session = parseRestoreSessionResponse(relatedResponse); if (session.getResult().equals("Success")) { @@ -378,7 +380,8 @@ public class VeeamClient { getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":")))); throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid)); } - logger.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout)); + logger.debug("Waiting {} seconds, out of a total of {} seconds, for the restore backup process to finish.", + (System.currentTimeMillis() - startTime) / 1000, restoreTimeout); try { Thread.sleep(1000); diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 0c70c75939e..333c3e16053 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -25,7 +25,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.junit.Assert.fail; -import static org.mockito.Mockito.times; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -157,7 +156,7 @@ public class VeeamClientTest { @Test public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOException { try { - ReflectionTestUtils.setField(mockClient, "restoreTimeout", 10); + ReflectionTestUtils.setField(mockClient, "restoreTimeout", 2); RestoreSession restoreSession = Mockito.mock(RestoreSession.class); HttpResponse httpResponse = Mockito.mock(HttpResponse.class); Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse); @@ -169,7 +168,7 @@ public class VeeamClientTest { } catch (Exception e) { Assert.assertEquals("Related job type: RestoreTest was not successful", e.getMessage()); } - Mockito.verify(mockClient, times(10)).get(Mockito.anyString()); + Mockito.verify(mockClient, Mockito.atLeastOnce()).get(Mockito.anyString()); } @Test From 1300fc5e91ac0c6ab57dbd35bb083944e9f94cef Mon Sep 17 00:00:00 2001 From: Jeevan Date: Wed, 28 Jan 2026 20:56:37 +0530 Subject: [PATCH 15/23] Fix keyword parameter filtering in listBackupOfferings API (#12540) Signed-off-by: Jeevan Yewale Co-authored-by: Jeevan Yewale --- .../java/org/apache/cloudstack/backup/BackupManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 1e6ef1a7852..dc2677a507f 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -239,7 +239,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final Filter searchFilter = new Filter(BackupOfferingVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); SearchBuilder sb = backupOfferingDao.createSearchBuilder(); sb.and("zone_id", sb.entity().getZoneId(), SearchCriteria.Op.EQ); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE); + CallContext ctx = CallContext.current(); final Account caller = ctx.getCallingAccount(); if (Account.Type.NORMAL == caller.getType()) { From 243872a77103494b3c030439e4ee24071a03ef2a Mon Sep 17 00:00:00 2001 From: Vishesh <8760112+vishesh92@users.noreply.github.com> Date: Thu, 29 Jan 2026 11:40:23 +0530 Subject: [PATCH 16/23] Use infinite scroll select (#11991) * addresses the domain selection (listed after the page size) with keyword search --- ui/src/components/view/DedicateDomain.vue | 133 +++++------ .../widgets/InfiniteScrollSelect.vue | 91 ++++++- ui/src/views/iam/AddUser.vue | 121 ++++------ ui/src/views/infra/UsageRecords.vue | 112 ++++----- ui/src/views/storage/CreateTemplate.vue | 111 ++++----- ui/src/views/storage/UploadLocalVolume.vue | 225 +++++++----------- ui/src/views/storage/UploadVolume.vue | 216 +++++++---------- ui/src/views/tools/CreateWebhook.vue | 124 ++++------ ui/src/views/tools/ManageVolumes.vue | 157 +++++------- 9 files changed, 560 insertions(+), 730 deletions(-) diff --git a/ui/src/components/view/DedicateDomain.vue b/ui/src/components/view/DedicateDomain.vue index 0b3645ce418..4b8cc31ae46 100644 --- a/ui/src/components/view/DedicateDomain.vue +++ b/ui/src/components/view/DedicateDomain.vue @@ -18,52 +18,44 @@ + + From 664f76c7e4b61489927abb6b5e859d79c47662a4 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 29 Jan 2026 05:24:58 -0300 Subject: [PATCH 22/23] Fix KvmSshToAgentEnabled setting description and make it dynamic (#12533) --- .../java/com/cloud/resource/ResourceManager.java | 4 ++-- .../resources/META-INF/db/schema-42020to42030.sql | 3 +++ .../com/cloud/resource/ResourceManagerImpl.java | 3 +-- .../cloud/resource/ResourceManagerImplTest.java | 14 +++++++++++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 89dc4badcbc..936e8b3448e 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -51,8 +51,8 @@ public interface ResourceManager extends ResourceService, Configurable { ConfigKey KvmSshToAgentEnabled = new ConfigKey<>("Advanced", Boolean.class, "kvm.ssh.to.agent","true", - "Number of retries when preparing a host into Maintenance Mode is faulty before failing", - false); + "True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations", + true); ConfigKey HOST_MAINTENANCE_LOCAL_STRATEGY = new ConfigKey<>(String.class, "host.maintenance.local.storage.strategy", "Advanced","Error", diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql index 567e623564e..5eec97278ba 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql @@ -23,3 +23,6 @@ ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(20 UPDATE `cloud`.`alert` SET type = 33 WHERE name = 'ALERT.VR.PUBLIC.IFACE.MTU'; UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; + +-- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields +UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index a77ecfcb7fe..c076ab7c893 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -2920,8 +2920,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (!isAgentOnHost || vmsMigrating || host.getStatus() == Status.Up) { return; } - final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(KvmSshToAgentEnabled.key())); - if (sshToAgent) { + if (KvmSshToAgentEnabled.value()) { Ternary credentials = getHostCredentials(host); connectAndRestartAgentOnHost(host, credentials.first(), credentials.second(), credentials.third()); } else { diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 5b7353bded6..1669d7a47d9 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -45,6 +45,7 @@ import com.cloud.vm.dao.VMInstanceDao; import com.trilead.ssh2.Connection; import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.After; import org.junit.Assert; @@ -61,6 +62,7 @@ import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -152,6 +154,12 @@ public class ResourceManagerImplTest { private MockedConstruction getVncPortCommandMockedConstruction; private AutoCloseable closeable; + private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException { + Field f = ConfigKey.class.getDeclaredField(name); + f.setAccessible(true); + f.set(configKey, o); + } + @Before public void setup() throws Exception { closeable = MockitoAnnotations.openMocks(this); @@ -194,7 +202,7 @@ public class ResourceManagerImplTest { eq("service cloudstack-agent restart"))). willReturn(new SSHCmdHelper.SSHCmdResult(0,"","")); - when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("true"); + overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "true"); rootDisks = Arrays.asList(rootDisk1, rootDisk2); dataDisks = Collections.singletonList(dataDisk); @@ -372,9 +380,9 @@ public class ResourceManagerImplTest { } @Test(expected = CloudRuntimeException.class) - public void testHandleAgentSSHDisabledNotConnectedAgent() { + public void testHandleAgentSSHDisabledNotConnectedAgent() throws NoSuchFieldException, IllegalAccessException { when(host.getStatus()).thenReturn(Status.Disconnected); - when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("false"); + overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "false"); resourceManager.handleAgentIfNotConnected(host, false); } From 26b57655ecea10d65689084a9bf7a2285b744697 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 29 Jan 2026 13:59:41 +0530 Subject: [PATCH 23/23] Deployment plan fixes for VM with last host, and last host in maintenance (#12062) * Deployment plan fixes for VM with last host - Consider last host when it is not in maintenance - Fail deployment when user requests for last host consideration and last host doesn't exists or in maintenance * changes * msg update with vm/host name * address comments * Exclude last hosts with error or degraded state as well, for vm deploy * review changes --- .../deploy/DeploymentPlanningManagerImpl.java | 80 ++++++++++--------- .../cloud/ha/HighAvailabilityManagerImpl.java | 4 +- .../cloud/resource/ResourceManagerImpl.java | 2 +- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index e7b926eb4e4..6881fbab98c 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -36,6 +36,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.resource.ResourceState; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -378,22 +379,12 @@ StateListener, Configurable { planner = getDeploymentPlannerByName(plannerName); } - Host lastHost = null; - - String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost); - boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null && - (considerLastHostStr == null || Boolean.TRUE.toString().equalsIgnoreCase(considerLastHostStr)); - if (considerLastHost) { - HostVO host = _hostDao.findById(vm.getLastHostId()); - logger.debug("This VM has last host_id specified, trying to choose the same host: " + host); - lastHost = host; - - DeployDestination deployDestination = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption); - if (deployDestination != null) { - return deployDestination; - } + DeployDestination deployDestinationForVmLasthost = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption); + if (deployDestinationForVmLasthost != null) { + return deployDestinationForVmLasthost; } + HostVO lastHost = _hostDao.findById(vm.getLastHostId()); avoidOtherClustersForDeploymentIfMigrationDisabled(vm, lastHost, avoids); DeployDestination dest = null; @@ -475,47 +466,56 @@ StateListener, Configurable { private DeployDestination deployInVmLastHost(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoids, DeploymentPlanner planner, VirtualMachine vm, DataCenter dc, ServiceOffering offering, int cpuRequested, long ramRequested, boolean volumesRequireEncryption) throws InsufficientServerCapacityException { - HostVO host = _hostDao.findById(vm.getLastHostId()); - if (canUseLastHost(host, avoids, plan, vm, offering, volumesRequireEncryption)) { - _hostDao.loadHostTags(host); - _hostDao.loadDetails(host); - if (host.getStatus() != Status.Up) { + String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost); + String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null && + !(Boolean.FALSE.toString().equalsIgnoreCase(considerLastHostStr)); + if (!considerLastHost) { + return null; + } + + logger.debug("This VM has last host_id: {}", vm.getLastHostId()); + HostVO lastHost = _hostDao.findById(vm.getLastHostId()); + if (canUseLastHost(lastHost, avoids, plan, vm, offering, volumesRequireEncryption)) { + _hostDao.loadHostTags(lastHost); + _hostDao.loadDetails(lastHost); + if (lastHost.getStatus() != Status.Up) { logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host is not in UP state or is not enabled. Host current status [{}] and resource status [{}].", - vm, host, host.getState().name(), host.getResourceState()); + vm, lastHost, lastHost.getState().name(), lastHost.getResourceState()); return null; } - if (checkVmProfileAndHost(vmProfile, host)) { - long cluster_id = host.getClusterId(); + if (checkVmProfileAndHost(vmProfile, lastHost)) { + long cluster_id = lastHost.getClusterId(); ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio"); ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio"); float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); boolean hostHasCpuCapability, hostHasCapacity = false; - hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed()); + hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(lastHost, offering.getCpu(), offering.getSpeed()); if (hostHasCpuCapability) { // first check from reserved capacity - hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true); + hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true); // if not reserved, check the free capacity if (!hostHasCapacity) - hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true); + hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true); } boolean displayStorage = getDisplayStorageFromVmProfile(vmProfile); if (!hostHasCapacity || !hostHasCpuCapability) { - logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, host); + logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, lastHost); return null; } - Pod pod = _podDao.findById(host.getPodId()); - Cluster cluster = _clusterDao.findById(host.getClusterId()); + Pod pod = _podDao.findById(lastHost.getPodId()); + Cluster cluster = _clusterDao.findById(lastHost.getClusterId()); logger.debug("Last host [{}] of VM [{}] is UP and has enough capacity. Checking for suitable pools for this host under zone [{}], pod [{}] and cluster [{}].", - host, vm, dc, pod, cluster); + lastHost, vm, dc, pod, cluster); if (vm.getHypervisorType() == HypervisorType.BareMetal) { - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap<>(), displayStorage); + DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, new HashMap<>(), displayStorage); logger.debug("Returning Deployment Destination: {}.", dest); return dest; } @@ -523,8 +523,8 @@ StateListener, Configurable { // search for storage under the zone, pod, cluster // of // the last host. - DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(), - host.getPodId(), host.getClusterId(), host.getId(), plan.getPoolId(), null); + DataCenterDeployment lastPlan = new DataCenterDeployment(lastHost.getDataCenterId(), + lastHost.getPodId(), lastHost.getClusterId(), lastHost.getId(), plan.getPoolId(), null); Pair>, List> result = findSuitablePoolsForVolumes( vmProfile, lastPlan, avoids, HostAllocator.RETURN_UPTO_ALL); Map> suitableVolumeStoragePools = result.first(); @@ -533,11 +533,11 @@ StateListener, Configurable { // choose the potential pool for this VM for this // host if (suitableVolumeStoragePools.isEmpty()) { - logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", host, vm); + logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", lastHost, vm); return null; } List suitableHosts = new ArrayList<>(); - suitableHosts.add(host); + suitableHosts.add(lastHost); Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts(), vm); @@ -550,7 +550,7 @@ StateListener, Configurable { for (Volume vol : readyAndReusedVolumes) { storageVolMap.remove(vol); } - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap, displayStorage); + DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, storageVolMap, displayStorage); logger.debug("Returning Deployment Destination: {}", dest); return dest; } @@ -562,7 +562,7 @@ StateListener, Configurable { private boolean canUseLastHost(HostVO host, ExcludeList avoids, DeploymentPlan plan, VirtualMachine vm, ServiceOffering offering, boolean volumesRequireEncryption) { if (host == null) { - logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping this and trying other available hosts.", vm, vm.getLastHostId()); + logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping it", vm, vm.getLastHostId()); return false; } @@ -576,6 +576,12 @@ StateListener, Configurable { return false; } + logger.debug("VM's last host is {}, trying to choose the same host if it is not in maintenance, error or degraded state", host); + if (host.isInMaintenanceStates() || Arrays.asList(ResourceState.Error, ResourceState.Degraded).contains(host.getResourceState())) { + logger.debug("Unable to deploy VM {} in the last host, last host {} is in {} state", vm.getName(), host.getName(), host.getResourceState()); + return false; + } + if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Cannot deploy VM [{}] in the last host [{}] because this host already has the max number of running VMs (users and system VMs). Skipping this and trying other available hosts.", vm, host); @@ -1474,7 +1480,7 @@ StateListener, Configurable { protected Pair> findPotentialDeploymentResources(List suitableHosts, Map> suitableVolumeStoragePools, ExcludeList avoid, PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes, List preferredHosts, VirtualMachine vm) { - logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM"); + logger.debug("Trying to find a potential host and associated storage pools from the suitable host/pool lists for this VM"); boolean hostCanAccessPool = false; boolean haveEnoughSpace = false; diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index b0f3e6d8d69..e3f67420a2a 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -833,7 +833,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur if (checkAndCancelWorkIfNeeded(work)) { return null; } - logger.info("Migration attempt: for VM {}from host {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries); + logger.info("Migration attempt: for {} from {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries); if (VirtualMachine.State.Stopped.equals(vm.getState())) { logger.info(String.format("vm %s is Stopped, skipping migrate.", vm)); @@ -843,8 +843,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur logger.info(String.format("VM %s is running on a different host %s, skipping migration", vm, vm.getHostId())); return null; } - logger.info("Migration attempt: for VM " + vm.getUuid() + "from host id " + srcHostId + - ". Starting attempt: " + (1 + work.getTimesTried()) + "/" + _maxRetries + " times."); try { work.setStep(Step.Migrating); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index c076ab7c893..12ceac21322 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -1417,7 +1417,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } } else { - logger.info("Maintenance: scheduling migration of VM {} from host {}", vm, host); + logger.info("Maintenance: scheduling migration of {} from {}", vm, host); _haMgr.scheduleMigration(vm, HighAvailabilityManager.ReasonType.HostMaintenance); } }