From 131ea9f7aceb4c12d8cbf0fe92bcc0419f40a2bc Mon Sep 17 00:00:00 2001 From: owsferraro Date: Fri, 27 Mar 2026 11:22:08 +0100 Subject: [PATCH 01/26] Fix PowerFlex 4.x issues with take & revert instance snapshots (#12880) * fixed database update on snapshot with multiple volumes and an api change * changed overwritevolumecontent based on powerflex version and removed unnecessary comments * Update plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java Co-authored-by: Suresh Kumar Anaparti * Update plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java Co-authored-by: Suresh Kumar Anaparti * Update plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java Co-authored-by: Suresh Kumar Anaparti --------- Co-authored-by: Suresh Kumar Anaparti --- .../vmsnapshot/ScaleIOVMSnapshotStrategy.java | 33 ++++++++-- .../client/ScaleIOGatewayClientImpl.java | 61 ++++++++++++++++++- 2 files changed, 89 insertions(+), 5 deletions(-) diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java index 7199fce1d34..aced750bd32 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.datastore.api.Volume; import com.cloud.agent.api.VMSnapshotTO; import com.cloud.alert.AlertManager; @@ -200,11 +201,35 @@ public class ScaleIOVMSnapshotStrategy extends ManagerBase implements VMSnapshot if (volumeIds != null && !volumeIds.isEmpty()) { List vmSnapshotDetails = new ArrayList(); vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "SnapshotGroupId", snapshotGroupId, false)); + Map snapshotNameToSrcPathMap = new HashMap<>(); + for (Map.Entry entry : srcVolumeDestSnapshotMap.entrySet()) { + snapshotNameToSrcPathMap.put(entry.getValue(), entry.getKey()); + } - for (int index = 0; index < volumeIds.size(); index++) { - String volumeSnapshotName = srcVolumeDestSnapshotMap.get(ScaleIOUtil.getVolumePath(volumeTOs.get(index).getPath())); - String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(volumeIds.get(index), volumeSnapshotName); - vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "Vol_" + volumeTOs.get(index).getId() + "_Snapshot", pathWithScaleIOVolumeName, false)); + for (String snapshotVolumeId : volumeIds) { + // Use getVolume() to fetch snapshot volume details and get its name + Volume snapshotVolume = client.getVolume(snapshotVolumeId); + if (snapshotVolume == null) { + throw new CloudRuntimeException("Cannot find snapshot volume with id: " + snapshotVolumeId); + } + String snapshotName = snapshotVolume.getName(); + + // Match back to source volume path + String srcVolumePath = snapshotNameToSrcPathMap.get(snapshotName); + if (srcVolumePath == null) { + throw new CloudRuntimeException("Cannot match snapshot " + snapshotName + " to a source volume"); + } + + // Find the matching VolumeObjectTO by path + VolumeObjectTO matchedVolume = volumeTOs.stream() + .filter(v -> ScaleIOUtil.getVolumePath(v.getPath()).equals(srcVolumePath)) + .findFirst() + .orElseThrow(() -> new CloudRuntimeException("Cannot find source volume for path: " + srcVolumePath)); + + String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, snapshotName); + vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), + "Vol_" + matchedVolume.getId() + "_Snapshot", + pathWithScaleIOVolumeName, false)); } vmSnapshotDetailsDao.saveDetails(vmSnapshotDetails); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java index c6a61c35b8b..8e23bc159a4 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java @@ -94,6 +94,9 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient { private String password; private String sessionKey; + private String gatewayVersion = null; + private int[] parsedVersion = null; + // The session token is valid for 8 hours from the time it was created, unless there has been no activity for 10 minutes // Reference: https://cpsdocs.dellemc.com/bundle/PF_REST_API_RG/page/GUID-92430F19-9F44-42B6-B898-87D5307AE59B.html private static final long MAX_VALID_SESSION_TIME_IN_HRS = 8; @@ -621,15 +624,26 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient { throw new CloudRuntimeException("Unable to revert, source snapshot volume and destination volume doesn't belong to same volume tree"); } + String requestBody = buildOverwriteVolumeContentRequest(sourceSnapshotVolumeId); + Boolean overwriteVolumeContentStatus = post( "/instances/Volume::" + destVolumeId + "/action/overwriteVolumeContent", - String.format("{\"srcVolumeId\":\"%s\",\"allowOnExtManagedVol\":\"TRUE\"}", sourceSnapshotVolumeId), Boolean.class); + requestBody, Boolean.class); if (overwriteVolumeContentStatus != null) { return overwriteVolumeContentStatus; } return false; } + private String buildOverwriteVolumeContentRequest(final String srcVolumeId) { + if (isVersionAtLeast(4, 0)) { + logger.debug("Using PowerFlex 4.0+ overwriteVolumeContent request body"); + return String.format("{\"srcVolumeId\":\"%s\"}", srcVolumeId); + } else { + logger.debug("Using pre-4.0 overwriteVolumeContent request body"); + return String.format("{\"srcVolumeId\":\"%s\",\"allowOnExtManagedVol\":\"TRUE\"}", srcVolumeId); } + } + @Override public boolean mapVolumeToSdc(final String volumeId, final String sdcId) { Preconditions.checkArgument(StringUtils.isNotEmpty(volumeId), "Volume id cannot be null"); @@ -1168,4 +1182,49 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient { sb.append("\n"); return sb.toString(); } + + private String fetchGatewayVersion() { + try { + JsonNode node = get("/version", JsonNode.class); + if (node != null && node.isTextual()) { + return node.asText(); + } + if (node != null && node.has("version")) { + return node.get("version").asText(); + } + } catch (Exception e) { + logger.warn("Could not fetch PowerFlex gateway version: " + e.getMessage()); + } + return null; + } + + private int[] parseVersion(String version) { + if (StringUtils.isEmpty(version)) return new int[]{0, 0, 0}; + String[] parts = version.replaceAll("\"", "").split("\\."); + int[] parsed = new int[3]; + for (int i = 0; i < Math.min(parts.length, 3); i++) { + try { + parsed[i] = Integer.parseInt(parts[i].trim()); + } catch (NumberFormatException e) { + parsed[i] = 0; + } + } + return parsed; + } + + private synchronized int[] getGatewayVersion() { + if (parsedVersion == null) { + gatewayVersion = fetchGatewayVersion(); + parsedVersion = parseVersion(gatewayVersion); + logger.info("PowerFlex Gateway version detected: " + gatewayVersion + + " => parsed: " + Arrays.toString(parsedVersion)); + } + return parsedVersion; + } + + private boolean isVersionAtLeast(int major, int minor) { + int[] v = getGatewayVersion(); + if (v[0] != major) return v[0] > major; + return v[1] >= minor; + } } From 6ca6aa1c3f012ede9b8342ad07adbb9f14e37f8f Mon Sep 17 00:00:00 2001 From: James Peru Mmbono Date: Fri, 27 Mar 2026 19:02:13 +0300 Subject: [PATCH 02/26] Fix NPE in NASBackupProvider when no running KVM host is available (#12805) * Fix NPE in NASBackupProvider when no running KVM host is available ResourceManager.findOneRandomRunningHostByHypervisor() can return null when no KVM host in the zone has status=Up (e.g. during management server startup, brief agent disconnections, or host state transitions). NASBackupProvider.syncBackupStorageStats() and deleteBackup() call host.getId() without a null check, causing a NullPointerException that crashes the entire BackupSyncTask background job every sync interval. This adds null checks in both methods: - syncBackupStorageStats: log a warning and return early - deleteBackup: throw CloudRuntimeException with a descriptive message --- .../apache/cloudstack/backup/NASBackupProvider.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index f2ea8ac71c9..d4068d498d4 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -56,6 +56,7 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -471,6 +472,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } else { host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); } + if (host == null) { + throw new CloudRuntimeException(String.format("Unable to find a running KVM host in zone %d to delete backup %s", backup.getZoneId(), backup.getUuid())); + } DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), backupRepository.getAddress(), backupRepository.getMountOptions()); @@ -552,7 +556,14 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public void syncBackupStorageStats(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + if (CollectionUtils.isEmpty(repositories)) { + return; + } final Host host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + if (host == null) { + logger.warn("Unable to find a running KVM host in zone {} to sync backup storage stats", zoneId); + return; + } for (final BackupRepository repository : repositories) { GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions()); BackupStorageStatsAnswer answer; From 68030df10b1f3a1c203a1b8f65f56817ecb149fc Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Sat, 28 Mar 2026 00:05:08 +0530 Subject: [PATCH 03/26] VM start error handling improvements and config to expose error to users (#12894) * VM start error handling improvements, and config to expose error to user * refactor --- .../cloud/vm/VirtualMachineManagerImpl.java | 53 ++++++++++++++++--- .../ConfigurationManagerImpl.java | 6 ++- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 86f45630611..a6b6802e978 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.vm; +import static com.cloud.configuration.ConfigurationManagerImpl.EXPOSE_ERRORS_TO_USER; import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS; import java.lang.reflect.Field; @@ -931,10 +932,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public void start(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { try { advanceStart(vmUuid, params, planToDeploy, planner); - } catch (ConcurrentOperationException | InsufficientCapacityException e) { - throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } catch (ConcurrentOperationException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to concurrent operation.", vmUuid), e).add(VirtualMachine.class, vmUuid); + } catch (final InsufficientCapacityException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to insufficient capacity.", vmUuid), e).add(VirtualMachine.class, vmUuid); } catch (final ResourceUnavailableException e) { - if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)){ + if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)) { throw new CloudRuntimeException("Network is unavailable. Please contact administrator", e).add(VirtualMachine.class, vmUuid); } throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); @@ -1361,6 +1374,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); + Throwable lastKnownError = null; boolean canRetry = true; ExcludeList avoids = null; try { @@ -1384,7 +1398,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac int retry = StartRetry.value(); while (retry-- != 0) { - logger.debug("Instance start attempt #{}", (StartRetry.value() - retry)); + int attemptNumber = StartRetry.value() - retry; + logger.debug("Instance start attempt #{}", attemptNumber); if (reuseVolume) { final List vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); @@ -1450,8 +1465,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac reuseVolume = false; continue; } - throw new InsufficientServerCapacityException("Unable to create a deployment for " + vmProfile, DataCenter.class, plan.getDataCenterId(), - areAffinityGroupsAssociated(vmProfile)); + String message = String.format("Unable to create a deployment for %s after %s attempts", vmProfile, attemptNumber); + if (canExposeError(account) && lastKnownError != null) { + message += String.format(" Last known error: %s", lastKnownError.getMessage()); + throw new CloudRuntimeException(message, lastKnownError); + } else { + throw new InsufficientServerCapacityException(message, DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); + } } avoids.addHost(dest.getHost().getId()); @@ -1619,11 +1639,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new ExecutionException("Unable to start VM:" + vm.getUuid() + " due to error in finalizeStart, not retrying"); } } - logger.info("Unable to start VM on {} due to {}", dest.getHost(), (startAnswer == null ? " no start answer" : startAnswer.getDetails())); + String msg = String.format("Unable to start VM on %s due to %s", dest.getHost(), startAnswer == null ? "no start command answer" : startAnswer.getDetails()); + lastKnownError = new ExecutionException(msg); + if (startAnswer != null && startAnswer.getContextParam("stopRetry") != null) { + logger.error(msg, lastKnownError); break; } + logger.debug(msg, lastKnownError); } catch (OperationTimedoutException e) { logger.debug("Unable to send the start command to host {} failed to start VM: {}", dest.getHost(), vm); if (e.isActive()) { @@ -1633,6 +1657,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e); } catch (final ResourceUnavailableException e) { logger.warn("Unable to contact resource.", e); + lastKnownError = e; if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; @@ -1689,10 +1714,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } if (startedVm == null) { - throw new CloudRuntimeException("Unable to start Instance '" + vm.getHostName() + "' (" + vm.getUuid() + "), see management server log for details"); + String messageTmpl = "Unable to start Instance '%s' (%s)%s"; + String details; + if (canExposeError(account) && lastKnownError != null) { + details = ": " + lastKnownError.getMessage(); + } else { + details = ", see management server log for details"; + } + String message = String.format(messageTmpl, vm.getHostName(), vm.getUuid(), details); + throw new CloudRuntimeException(message, lastKnownError); } } + private boolean canExposeError(Account account) { + return (account != null && account.getType() == Account.Type.ADMIN) || Boolean.TRUE.equals(EXPOSE_ERRORS_TO_USER.value()); + } + protected void updateStartCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, StartCommand command) { if (!HypervisorType.External.equals(host.getHypervisorType())) { return; diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 7dbf3e1d2a2..e7306b3a8c5 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -536,6 +536,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati public static final ConfigKey ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS = new ConfigKey<>(Boolean.class, "allow.domain.admins.to.create.tagged.offerings", "Advanced", "false", "Allow domain admins to create offerings with tags.", true, ConfigKey.Scope.Account, null); + public static final ConfigKey EXPOSE_ERRORS_TO_USER = new ConfigKey<>(Boolean.class, "expose.errors.to.user", ConfigKey.CATEGORY_ADVANCED, + "false", "If set to true, detailed error messages will be returned to all user roles. If false, detailed errors are only shown to admin users", true, ConfigKey.Scope.Global, null); + public static final ConfigKey DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0", "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," + " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " + @@ -8494,11 +8497,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE, VM_SERVICE_OFFERING_MAX_CPU_CORES, VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS, ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN, ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN, - ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE, AllowNonRFC1918CompliantIPs, HostCapacityTypeCpuMemoryWeight + ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, EXPOSE_ERRORS_TO_USER, DELETE_QUERY_BATCH_SIZE, AllowNonRFC1918CompliantIPs, HostCapacityTypeCpuMemoryWeight }; } - /** * Returns a string representing the specified configuration's type. * @param configName name of the configuration. From 71bd26ff7cdabe07cc8c4344b26f156671d88fd4 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Sat, 28 Mar 2026 00:07:30 +0530 Subject: [PATCH 04/26] PowerFlex/ScaleIO storage - the MDMs validation improvements (#12893) --- .../kvm/storage/ScaleIOStorageAdaptor.java | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index e336e0e5a54..82bc35f009e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Predicate; import com.cloud.agent.api.PrepareStorageClientCommand; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; @@ -644,18 +645,30 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { // Assuming SDC service is started, add mdms String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS); String[] mdmAddresses = mdms.split(","); - if (mdmAddresses.length > 0) { - if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(true, getSDCDetails(details), "MDM added, no need to prepare the SDC client"); - } - - ScaleIOUtil.addMdms(mdmAddresses); - if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(false, null, "Failed to add MDMs"); + // remove MDMs already present in the config and added to the SDC + String[] mdmAddressesToAdd = Arrays.stream(mdmAddresses) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + // if all MDMs are already in the config and added to the SDC + if (mdmAddressesToAdd.length < 1 && mdmAddresses.length > 0) { + String msg = String.format("MDMs %s of the storage pool %s are already added", mdms, uuid); + logger.debug(msg); + return new Ternary<>(true, getSDCDetails(details), msg); + } else if (mdmAddressesToAdd.length > 0) { + ScaleIOUtil.addMdms(mdmAddressesToAdd); + String[] missingMdmAddresses = Arrays.stream(mdmAddressesToAdd) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + if (missingMdmAddresses.length > 0) { + String msg = String.format("Failed to add MDMs %s of the storage pool %s", String.join(", ", missingMdmAddresses), uuid); + logger.debug(msg); + return new Ternary<>(false, null, msg); } else { - logger.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid)); + logger.debug("MDMs {} of the storage pool {} are added", mdmAddressesToAdd, uuid); applyMdmsChangeWaitTime(details); } + } else { + return new Ternary<>(false, getSDCDetails(details), "No MDM addresses provided"); } } From 4ebe3349b77b0cfb11c21a1c25cc0fb331f4018b Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Mon, 30 Mar 2026 15:32:12 +0530 Subject: [PATCH 05/26] add user-agent header to template downloader request (#12791) --- .../template/HttpTemplateDownloader.java | 2 + .../template/MetalinkTemplateDownloader.java | 4 ++ .../SimpleHttpMultiFileDownloader.java | 3 ++ .../HttpDirectTemplateDownloader.java | 5 +++ .../main/java/com/cloud/utils/HttpUtils.java | 5 +++ .../main/java/com/cloud/utils/UriUtils.java | 3 ++ .../net/HttpClientCloudStackUserAgent.java | 39 +++++++++++++++++++ .../com/cloud/utils/storage/QCOW2Utils.java | 2 + 8 files changed, 63 insertions(+) create mode 100644 utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 6fe001de72c..71c329796d1 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -52,6 +52,7 @@ import com.cloud.storage.StorageLayer; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.net.Proxy; /** @@ -125,6 +126,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java index 95ed0d1e76d..0dad2564779 100644 --- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java @@ -18,8 +18,11 @@ // package com.cloud.storage.template; + import com.cloud.storage.StorageLayer; import com.cloud.utils.UriUtils; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; + import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethodRetryHandler; @@ -59,6 +62,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (!toFileSet) { String[] parts = downloadUrl.split("/"); String filename = parts[parts.length - 1]; diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index 8719947cb4f..6608754073a 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -44,6 +44,7 @@ import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.lang3.StringUtils; import com.cloud.storage.StorageLayer; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader { private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); @@ -95,6 +96,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } @@ -141,6 +143,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem continue; } HeadMethod headMethod = new HeadMethod(downloadUrl); + headMethod.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { if (client.executeMethod(headMethod) != HttpStatus.SC_OK) { continue; diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index c4a802ecdbc..99b84bb645c 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.direct.download; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -32,6 +33,7 @@ import java.util.Map; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; @@ -39,6 +41,7 @@ import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.io.IOUtils; public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -68,6 +71,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(this.isFollowRedirects()); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); @@ -111,6 +115,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { public boolean checkUrl(String url) { HeadMethod httpHead = new HeadMethod(url); httpHead.setFollowRedirects(this.isFollowRedirects()); + httpHead.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { int responseCode = client.executeMethod(httpHead); if (responseCode != HttpStatus.SC_OK) { diff --git a/utils/src/main/java/com/cloud/utils/HttpUtils.java b/utils/src/main/java/com/cloud/utils/HttpUtils.java index 9f998efe099..b7d95f8133b 100644 --- a/utils/src/main/java/com/cloud/utils/HttpUtils.java +++ b/utils/src/main/java/com/cloud/utils/HttpUtils.java @@ -19,6 +19,8 @@ package com.cloud.utils; +import static com.cloud.utils.UriUtils.USER_AGENT; + import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -33,6 +35,8 @@ import java.net.HttpURLConnection; import java.net.URL; import java.util.Map; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; + public class HttpUtils { protected static Logger LOGGER = LogManager.getLogger(HttpUtils.class); @@ -161,6 +165,7 @@ public class HttpUtils { try { URL url = new URL(fileURL); httpConn = (HttpURLConnection) url.openConnection(); + httpConn.setRequestProperty(USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); int responseCode = httpConn.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { int contentLength = httpConn.getContentLength(); diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index eba6f88201b..a4654091e0c 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -67,12 +67,14 @@ import org.w3c.dom.NodeList; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; public class UriUtils { protected static Logger LOGGER = LogManager.getLogger(UriUtils.class); + public static final String USER_AGENT = "User-Agent"; public static String formNfsUri(String host, String path) { try { @@ -227,6 +229,7 @@ public class UriUtils { URI uri = new URI(url); httpConn = (HttpURLConnection)uri.toURL().openConnection(); httpConn.setRequestMethod(method); + httpConn.setRequestProperty(USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); httpConn.setConnectTimeout(2000); httpConn.setReadTimeout(5000); httpConn.setInstanceFollowRedirects(Boolean.TRUE.equals(followRedirect)); diff --git a/utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java b/utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java new file mode 100644 index 00000000000..991aa296380 --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/net/HttpClientCloudStackUserAgent.java @@ -0,0 +1,39 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.utils.net; + +import org.apache.logging.log4j.util.Strings; + +public class HttpClientCloudStackUserAgent { + public static final String CLOUDSTACK_USER_AGENT = buildUserAgent(); + + private static String buildUserAgent() { + String version = HttpClientCloudStackUserAgent.class + .getPackage() + .getImplementationVersion(); + + if (Strings.isBlank(version)) { + version = "unknown"; + } + return "CloudStack-Agent/" + version + " (Apache CloudStack)"; + } + + private HttpClientCloudStackUserAgent() {} +} diff --git a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java index 34d748b0708..a4a49825d1f 100644 --- a/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java +++ b/utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.LogManager; import com.cloud.utils.NumbersUtil; import com.cloud.utils.UriUtils; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; public final class QCOW2Utils { protected static Logger LOGGER = LogManager.getLogger(QCOW2Utils.class); @@ -119,6 +120,7 @@ public final class QCOW2Utils { try { URI url = new URI(urlStr); httpConn = (HttpURLConnection)url.toURL().openConnection(); + httpConn.setRequestProperty(UriUtils.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); httpConn.setInstanceFollowRedirects(followRedirects); return getVirtualSize(httpConn.getInputStream(), UriUtils.isUrlForCompressedFile(urlStr)); } catch (URISyntaxException e) { From 59b6c32b60c6fb6bcb21ea5669310d23736bf5df Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 30 Mar 2026 15:49:35 +0530 Subject: [PATCH 06/26] [UI] Fix create backup notification (#12903) --- ui/src/views/compute/StartBackup.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/views/compute/StartBackup.vue b/ui/src/views/compute/StartBackup.vue index 812e9e49e05..de2680d1447 100644 --- a/ui/src/views/compute/StartBackup.vue +++ b/ui/src/views/compute/StartBackup.vue @@ -125,7 +125,7 @@ export default { postAPI('createBackup', data).then(response => { this.$pollJob({ jobId: response.createbackupresponse.jobid, - title: this.$t('label.create.bucket'), + title: this.$t('label.create.backup'), description: values.name, errorMessage: this.$t('message.create.backup.failed'), loadingMessage: `${this.$t('label.create.backup')}: ${this.resource.name || this.resource.id}`, From 4f93ba888c36d992c09af8d2486ea4ec75102f79 Mon Sep 17 00:00:00 2001 From: julien-vaz <54545601+julien-vaz@users.noreply.github.com> Date: Tue, 31 Mar 2026 20:29:30 -0300 Subject: [PATCH 07/26] Refactor Quota Summary API (#10505) * Refactor Quota Summary API * Fixes imports * Fix QuotaServiceImplTest * Update plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java Co-authored-by: Fabricio Duarte * Fix QuotaSummaryCmd * Remove unnecessary imports * Remove unused createQuotaSummaryResponse declarations * Remove unnecessary imports * Update plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java Co-authored-by: dahn * Fix QuotaSummaryCmd * Fix QuotaResponseBuilderImplTest * Refactor test * Fix QuotaSummaryCmd * Fix projectid behavior * Simplify QuotaSummary and deprecate listall * Fix createQuotaSummaryResponse * Remove unused import * Apply suggestions + some adjustments * Remove duplicated check * Fix checkstyle * Adjust entity owner * Remove unused method + fix tests * Add missing @ACL to some parameters * Adjust how the parameters behave * Allow domain admins and users to use keyword * Address reviews --------- Co-authored-by: Julien Hervot de Mattos Vaz Co-authored-by: Fabricio Duarte Co-authored-by: dahn --- .../java/com/cloud/user/AccountService.java | 2 + .../apache/cloudstack/api/ApiConstants.java | 1 + .../com/cloud/domain/dao/DomainDaoImpl.java | 2 +- .../views/cloud_usage.quota_summary_view.sql | 48 +++++ .../quota/QuotaAccountStateFilter.java | 35 ++++ .../cloudstack/quota/dao/QuotaSummaryDao.java | 32 ++++ .../quota/dao/QuotaSummaryDaoImpl.java | 80 ++++++++ .../cloudstack/quota/vo/QuotaSummaryVO.java | 154 +++++++++++++++ .../quota/spring-framework-quota-context.xml | 3 +- .../api/command/QuotaSummaryCmd.java | 98 ++++++---- .../api/response/QuotaResponseBuilder.java | 7 +- .../response/QuotaResponseBuilderImpl.java | 177 +++++++++++------- .../api/response/QuotaSummaryResponse.java | 109 ++++++----- .../cloudstack/quota/QuotaServiceImpl.java | 6 + .../QuotaResponseBuilderImplTest.java | 116 +++++++----- .../quota/QuotaServiceImplTest.java | 43 +++-- .../management/MockAccountManager.java | 6 + .../api/dispatch/ParamProcessWorker.java | 15 +- .../com/cloud/user/AccountManagerImpl.java | 44 +++++ 19 files changed, 747 insertions(+), 231 deletions(-) create mode 100644 engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 30919e5b782..4145e2b89eb 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -138,6 +138,8 @@ public interface AccountService { Long finalizeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly); + Long finalizeAccountId(Long accountId, String accountName, Long domainId, Long projectId); + /** * returns the user account object for a given user id * @param userId user id diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3d827641358..05c6098bc72 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -20,6 +20,7 @@ public class ApiConstants { public static final String ACCOUNT = "account"; public static final String ACCOUNTS = "accounts"; public static final String ACCOUNT_NAME = "accountname"; + public static final String ACCOUNT_STATE_TO_SHOW = "accountstatetoshow"; public static final String ACCOUNT_TYPE = "accounttype"; public static final String ACCOUNT_ID = "accountid"; public static final String ACCOUNT_IDS = "accountids"; diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java index 56d971bbe01..1afa0d22dcc 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java @@ -262,7 +262,7 @@ public class DomainDaoImpl extends GenericDaoBase implements Dom SearchCriteria sc = DomainPairSearch.create(); sc.setParameters("id", parentId, childId); - List domainPair = listBy(sc); + List domainPair = listIncludingRemovedBy(sc); if ((domainPair != null) && (domainPair.size() == 2)) { DomainVO d1 = domainPair.get(0); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql new file mode 100644 index 00000000000..0646c3b6f91 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql @@ -0,0 +1,48 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- cloud_usage.quota_summary_view source + +-- Create view for quota summary +DROP VIEW IF EXISTS `cloud_usage`.`quota_summary_view`; +CREATE VIEW `cloud_usage`.`quota_summary_view` AS +SELECT + cloud_usage.quota_account.account_id AS account_id, + cloud_usage.quota_account.quota_balance AS quota_balance, + cloud_usage.quota_account.quota_balance_date AS quota_balance_date, + cloud_usage.quota_account.quota_enforce AS quota_enforce, + cloud_usage.quota_account.quota_min_balance AS quota_min_balance, + cloud_usage.quota_account.quota_alert_date AS quota_alert_date, + cloud_usage.quota_account.quota_alert_type AS quota_alert_type, + cloud_usage.quota_account.last_statement_date AS last_statement_date, + cloud.account.uuid AS account_uuid, + cloud.account.account_name AS account_name, + cloud.account.state AS account_state, + cloud.account.removed AS account_removed, + cloud.domain.id AS domain_id, + cloud.domain.uuid AS domain_uuid, + cloud.domain.name AS domain_name, + cloud.domain.path AS domain_path, + cloud.domain.removed AS domain_removed, + cloud.projects.uuid AS project_uuid, + cloud.projects.name AS project_name, + cloud.projects.removed AS project_removed +FROM + cloud_usage.quota_account + INNER JOIN cloud.account ON (cloud.account.id = cloud_usage.quota_account.account_id) + INNER JOIN cloud.domain ON (cloud.domain.id = cloud.account.domain_id) + LEFT JOIN cloud.projects ON (cloud.account.type = 5 AND cloud.account.id = cloud.projects.project_account_id); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java new file mode 100644 index 00000000000..cc6ca203ef7 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota; + +import org.apache.commons.lang3.StringUtils; + +public enum QuotaAccountStateFilter { + ALL, ACTIVE, REMOVED; + + public static QuotaAccountStateFilter getValue(String value) { + if (StringUtils.isBlank(value)) { + return null; + } + for (QuotaAccountStateFilter state : values()) { + if (state.name().equalsIgnoreCase(value)) { + return state; + } + } + return null; + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java new file mode 100644 index 00000000000..d8ee2607501 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.dao; + +import java.util.List; + +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; + +import com.cloud.utils.Pair; +import com.cloud.utils.db.GenericDao; + +public interface QuotaSummaryDao extends GenericDao { + + Pair, Integer> listQuotaSummariesForAccountAndOrDomain(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter, Long startIndex, Long pageSize); +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java new file mode 100644 index 00000000000..d90d5a75859 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.dao; + +import java.util.List; + +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; + +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +public class QuotaSummaryDaoImpl extends GenericDaoBase implements QuotaSummaryDao { + + @Override + public Pair, Integer> listQuotaSummariesForAccountAndOrDomain(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter, Long startIndex, Long pageSize) { + SearchCriteria searchCriteria = createListQuotaSummariesSearchCriteria(accountId, accountName, domainId, domainPath, accountStateFilter); + Filter filter = new Filter(QuotaSummaryVO.class, "accountName", true, startIndex, pageSize); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback, Integer>>) status -> searchAndCount(searchCriteria, filter)); + } + + protected SearchCriteria createListQuotaSummariesSearchCriteria(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter) { + SearchCriteria searchCriteria = createListQuotaSummariesSearchBuilder(accountStateFilter).create(); + + searchCriteria.setParametersIfNotNull("accountId", accountId); + searchCriteria.setParametersIfNotNull("domainId", domainId); + + if (accountName != null) { + searchCriteria.setParameters("accountName", "%" + accountName + "%"); + } + + if (domainPath != null) { + searchCriteria.setParameters("domainPath", domainPath + "%"); + } + + return searchCriteria; + } + + protected SearchBuilder createListQuotaSummariesSearchBuilder(QuotaAccountStateFilter accountStateFilter) { + SearchBuilder searchBuilder = createSearchBuilder(); + + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("accountName", searchBuilder.entity().getAccountName(), SearchCriteria.Op.LIKE); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainPath", searchBuilder.entity().getDomainPath(), SearchCriteria.Op.LIKE); + + if (QuotaAccountStateFilter.REMOVED.equals(accountStateFilter)) { + searchBuilder.and("accountRemoved", searchBuilder.entity().getAccountRemoved(), SearchCriteria.Op.NNULL); + } else if (QuotaAccountStateFilter.ACTIVE.equals(accountStateFilter)) { + searchBuilder.and("accountRemoved", searchBuilder.entity().getAccountRemoved(), SearchCriteria.Op.NULL); + } + + return searchBuilder; + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java new file mode 100644 index 00000000000..f9796497d57 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import com.cloud.user.Account; + +@Entity +@Table(name = "quota_summary_view") +public class QuotaSummaryVO { + + @Id + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "quota_enforce") + private Integer quotaEnforce = 0; + + @Column(name = "quota_balance") + private BigDecimal quotaBalance; + + @Column(name = "quota_balance_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date quotaBalanceDate = null; + + @Column(name = "quota_min_balance") + private BigDecimal quotaMinBalance; + + @Column(name = "quota_alert_type") + private Integer quotaAlertType = null; + + @Column(name = "quota_alert_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date quotaAlertDate = null; + + @Column(name = "last_statement_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date lastStatementDate = null; + + @Column(name = "account_uuid") + private String accountUuid; + + @Column(name = "account_name") + private String accountName; + + @Column(name = "account_state") + @Enumerated(EnumType.STRING) + private Account.State accountState; + + @Column(name = "account_removed") + private Date accountRemoved; + + @Column(name = "domain_id") + private Long domainId; + + @Column(name = "domain_uuid") + private String domainUuid; + + @Column(name = "domain_name") + private String domainName; + + @Column(name = "domain_path") + private String domainPath; + + @Column(name = "domain_removed") + private Date domainRemoved; + + @Column(name = "project_uuid") + private String projectUuid; + + @Column(name = "project_name") + private String projectName; + + @Column(name = "project_removed") + private Date projectRemoved; + + public Long getAccountId() { + return accountId; + } + + public BigDecimal getQuotaBalance() { + return quotaBalance; + } + + public String getAccountUuid() { + return accountUuid; + } + + public String getAccountName() { + return accountName; + } + + public Date getAccountRemoved() { + return accountRemoved; + } + + public Account.State getAccountState() { + return accountState; + } + + public Long getDomainId() { + return domainId; + } + + public String getDomainUuid() { + return domainUuid; + } + + public String getDomainPath() { + return domainPath; + } + + public Date getDomainRemoved() { + return domainRemoved; + } + + public String getProjectUuid() { + return projectUuid; + } + + public String getProjectName() { + return projectName; + } + + public Date getProjectRemoved() { + return projectRemoved; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 453355c8522..304b23b7220 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -19,7 +19,8 @@ - + + , Integer> responses; - if (caller.getType() == Account.Type.ADMIN) { - if (getAccountName() != null && getDomainId() != null) - responses = _responseBuilder.createQuotaSummaryResponse(getAccountName(), getDomainId()); - else - responses = _responseBuilder.createQuotaSummaryResponse(isListAll(), getKeyword(), getStartIndex(), getPageSizeVal()); - } else { - responses = _responseBuilder.createQuotaSummaryResponse(caller.getAccountName(), caller.getDomainId()); - } - final ListResponse response = new ListResponse(); + Pair, Integer> responses = quotaResponseBuilder.createQuotaSummaryResponse(this); + ListResponse response = new ListResponse<>(); response.setResponses(responses.first(), responses.second()); response.setResponseName(getCommandName()); setResponseObject(response); } + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + public String getAccountName() { return accountName; } @@ -88,16 +107,31 @@ public class QuotaSummaryCmd extends BaseListCmd { } public Boolean isListAll() { - return listAll == null ? false: listAll; + // If a domain ID was specified, then allow listing all summaries of domain + return ObjectUtils.defaultIfNull(listAll, Boolean.FALSE) || domainId != null; } public void setListAll(Boolean listAll) { this.listAll = listAll; } - @Override - public long getEntityOwnerId() { - return Account.ACCOUNT_ID_SYSTEM; + public Long getProjectId() { + return projectId; } + public QuotaAccountStateFilter getAccountStateToShow() { + QuotaAccountStateFilter state = QuotaAccountStateFilter.getValue(accountStateToShow); + if (state != null) { + return state; + } + return QuotaAccountStateFilter.ACTIVE; + } + + @Override + public long getEntityOwnerId() { + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; + } + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 67f75ebf82f..177fb00d4b5 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; @@ -52,11 +53,7 @@ public interface QuotaResponseBuilder { QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); - Pair, Integer> createQuotaSummaryResponse(Boolean listAll); - - Pair, Integer> createQuotaSummaryResponse(Boolean listAll, String keyword, Long startIndex, Long pageSize); - - Pair, Integer> createQuotaSummaryResponse(String accountName, Long domainId); + Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd); QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index f9456587058..d1992d4c3ad 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -42,7 +42,9 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.domain.Domain; import com.cloud.exception.PermissionDeniedException; +import com.cloud.projects.dao.ProjectDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; @@ -56,6 +58,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; @@ -74,11 +77,13 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariable import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; + import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaCreditsDao; import org.apache.cloudstack.quota.dao.QuotaEmailConfigurationDao; import org.apache.cloudstack.quota.dao.QuotaEmailTemplatesDao; +import org.apache.cloudstack.quota.dao.QuotaSummaryDao; import org.apache.cloudstack.quota.dao.QuotaTariffDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; @@ -86,10 +91,13 @@ import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.compress.utils.Sets; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -108,7 +116,6 @@ import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.Pair; -import com.cloud.utils.db.Filter; @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @@ -121,7 +128,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private QuotaCreditsDao quotaCreditsDao; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageDao quotaUsageDao; @Inject private QuotaEmailTemplatesDao _quotaEmailTemplateDao; @@ -132,24 +139,30 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject + private ProjectDao projectDao; + @Inject private QuotaAccountDao quotaAccountDao; @Inject - private DomainDao _domainDao; + private DomainDao domainDao; @Inject private AccountManager _accountMgr; @Inject - private QuotaStatement _statement; + private QuotaStatement quotaStatement; @Inject private QuotaManager _quotaManager; @Inject private QuotaEmailConfigurationDao quotaEmailConfigurationDao; @Inject + private QuotaSummaryDao quotaSummaryDao; + @Inject private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; + private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); + protected void checkActivationRulesAllowed(String activationRule) { if (!_quotaService.isJsInterpretationEnabled() && StringUtils.isNotEmpty(activationRule)) { throw new PermissionDeniedException("Quota Tariff Activation Rule cannot be set, as Javascript interpretation is disabled in the configuration."); @@ -180,75 +193,113 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } @Override - public Pair, Integer> createQuotaSummaryResponse(final String accountName, final Long domainId) { - List result = new ArrayList(); + public Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd) { + Account caller = CallContext.current().getCallingAccount(); - if (accountName != null && domainId != null) { - Account account = _accountDao.findActiveAccount(accountName, domainId); - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); + if (!accountTypesThatCanListAllQuotaSummaries.contains(caller.getType()) || !cmd.isListAll()) { + return getQuotaSummaryResponse(cmd.getEntityOwnerId(), null, null, cmd); } - return new Pair<>(result, result.size()); + return getQuotaSummaryResponseWithListAll(cmd, caller); } - @Override - public Pair, Integer> createQuotaSummaryResponse(Boolean listAll) { - return createQuotaSummaryResponse(listAll, null, null, null); - } - - @Override - public Pair, Integer> createQuotaSummaryResponse(Boolean listAll, final String keyword, final Long startIndex, final Long pageSize) { - List result = new ArrayList(); - Integer count = 0; - if (listAll) { - Filter filter = new Filter(AccountVO.class, "accountName", true, startIndex, pageSize); - Pair, Integer> data = _accountDao.findAccountsLike(keyword, filter); - count = data.second(); - for (final AccountVO account : data.first()) { - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); - } - } else { - Pair, Integer> data = quotaAccountDao.listAllQuotaAccount(startIndex, pageSize); - count = data.second(); - for (final QuotaAccountVO quotaAccount : data.first()) { - AccountVO account = _accountDao.findById(quotaAccount.getId()); - if (account == null) { - continue; - } - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); + protected Pair, Integer> getQuotaSummaryResponseWithListAll(QuotaSummaryCmd cmd, Account caller) { + Long domainId = cmd.getDomainId(); + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + if (domain == null) { + throw new InvalidParameterValueException(String.format("Domain [%s] does not exist.", domainId)); } } - return new Pair<>(result, count); + + String domainPath = getDomainPathByDomainIdForDomainAdmin(caller); + + Long accountId = cmd.getEntityOwnerId(); + if (accountId == -1) { + accountId = cmd.isListAll() ? null : caller.getAccountId(); + } + + return getQuotaSummaryResponse(accountId, domainId, domainPath, cmd); } - protected QuotaSummaryResponse getQuotaSummaryResponse(final Account account) { - Calendar[] period = _statement.getCurrentStatementTime(); - - if (account != null) { - QuotaSummaryResponse qr = new QuotaSummaryResponse(); - DomainVO domain = _domainDao.findById(account.getDomainId()); - BigDecimal curBalance = _quotaBalanceDao.lastQuotaBalance(account.getAccountId(), account.getDomainId(), period[1].getTime()); - BigDecimal quotaUsage = _quotaUsageDao.findTotalQuotaUsage(account.getAccountId(), account.getDomainId(), null, period[0].getTime(), period[1].getTime()); - - qr.setAccountId(account.getUuid()); - qr.setAccountName(account.getAccountName()); - qr.setDomainId(domain.getUuid()); - qr.setDomainName(domain.getName()); - qr.setBalance(curBalance); - qr.setQuotaUsage(quotaUsage); - qr.setState(account.getState()); - qr.setStartDate(period[0].getTime()); - qr.setEndDate(period[1].getTime()); - qr.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - qr.setQuotaEnabled(QuotaConfig.QuotaAccountEnabled.valueIn(account.getId())); - qr.setObjectName("summary"); - return qr; - } else { - return new QuotaSummaryResponse(); + /** + * Retrieves the domain path of the caller's domain (if the caller is Domain Admin) for filtering in the quota summary query. + * @return null if the caller is an Admin or the domain path of the caller's domain if the caller is a Domain Admin. + * @throws InvalidParameterValueException if it cannot find the domain. + */ + protected String getDomainPathByDomainIdForDomainAdmin(Account caller) { + if (caller.getType() != Account.Type.DOMAIN_ADMIN) { + return null; } + + Long domainId = caller.getDomainId(); + Domain domain = domainDao.findById(domainId); + _accountMgr.checkAccess(caller, domain); + + if (domain == null) { + throw new InvalidParameterValueException(String.format("Domain ID [%s] is invalid.", domainId)); + } + + return domain.getPath(); + } + + /** + * Returns a List of QuotaSummaryResponse based on the provided parameters. + * @param accountId ID of the Account to return the summaries for. If -1, either because no specific + * Account was provided, or list all is disabled, then the summary is generated for the calling Account. + * @param domainId ID of the Domain to return the summaries for. + * @param domainPath path of the Domain to return the summaries for. + */ + protected Pair, Integer> getQuotaSummaryResponse(Long accountId, Long domainId, String domainPath, QuotaSummaryCmd cmd) { + if (accountId != null && accountId == -1) { + accountId = CallContext.current().getCallingAccountId(); + } + + Pair, Integer> pairSummaries = quotaSummaryDao.listQuotaSummariesForAccountAndOrDomain(accountId, cmd.getKeyword(), domainId, domainPath, + cmd.getAccountStateToShow(), cmd.getStartIndex(), cmd.getPageSizeVal()); + List summaries = pairSummaries.first(); + + if (CollectionUtils.isEmpty(summaries)) { + logger.info("There are no summaries to list for parameters [{}].", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "domainId", "listAll", "page", "pageSize")); + return new Pair<>(new ArrayList<>(), 0); + } + + List responses = summaries.stream().map(this::getQuotaSummaryResponse).collect(Collectors.toList()); + + return new Pair<>(responses, pairSummaries.second()); + } + + protected QuotaSummaryResponse getQuotaSummaryResponse(QuotaSummaryVO summary) { + QuotaSummaryResponse response = new QuotaSummaryResponse(); + Account account = _accountDao.findByUuidIncludingRemoved(summary.getAccountUuid()); + + Calendar[] period = quotaStatement.getCurrentStatementTime(); + Date startDate = period[0].getTime(); + Date endDate = period[1].getTime(); + BigDecimal quotaUsage = quotaUsageDao.findTotalQuotaUsage(account.getAccountId(), account.getDomainId(), null, startDate, endDate); + + response.setQuotaUsage(quotaUsage); + response.setStartDate(startDate); + response.setEndDate(endDate); + response.setAccountId(summary.getAccountUuid()); + response.setAccountName(summary.getAccountName()); + response.setDomainId(summary.getDomainUuid()); + response.setDomainPath(summary.getDomainPath()); + response.setBalance(summary.getQuotaBalance()); + response.setState(summary.getAccountState()); + response.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + response.setQuotaEnabled(QuotaConfig.QuotaAccountEnabled.valueIn(account.getId())); + response.setDomainRemoved(summary.getDomainRemoved() != null); + response.setAccountRemoved(summary.getAccountRemoved() != null); + response.setObjectName("summary"); + + if (summary.getProjectUuid() != null) { + response.setProjectId(summary.getProjectUuid()); + response.setProjectName(summary.getProjectName()); + response.setProjectRemoved(summary.getProjectRemoved() != null); + } + + return response; } public boolean isUserAllowedToSeeActivationRules(User user) { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java index f8f27b4813d..d76202bfd88 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import com.google.gson.annotations.SerializedName; @@ -30,40 +29,48 @@ import com.cloud.user.Account.State; public class QuotaSummaryResponse extends BaseResponse { @SerializedName("accountid") - @Param(description = "Account ID") + @Param(description = "Account's ID") private String accountId; @SerializedName("account") - @Param(description = "Account name") + @Param(description = "Account's name") private String accountName; @SerializedName("domainid") - @Param(description = "Domain ID") + @Param(description = "Domain's ID") private String domainId; @SerializedName("domain") - @Param(description = "Domain name") - private String domainName; + @Param(description = "Domain's path") + private String domainPath; @SerializedName("balance") - @Param(description = "Account balance") + @Param(description = "Account's balance") private BigDecimal balance; @SerializedName("state") - @Param(description = "Account state") + @Param(description = "Account's state") private State state; + @SerializedName("domainremoved") + @Param(description = "If the domain is removed or not", since = "4.23.0") + private boolean domainRemoved; + + @SerializedName("accountremoved") + @Param(description = "If the account is removed or not", since = "4.23.0") + private boolean accountRemoved; + @SerializedName("quota") - @Param(description = "Quota usage of this period") + @Param(description = "Quota consumed between the startdate and enddate") private BigDecimal quotaUsage; @SerializedName("startdate") - @Param(description = "Start date") - private Date startDate = null; + @Param(description = "Start date of the quota consumption") + private Date startDate; @SerializedName("enddate") - @Param(description = "End date") - private Date endDate = null; + @Param(description = "End date of the quota consumption") + private Date endDate; @SerializedName("currency") @Param(description = "Currency") @@ -73,9 +80,17 @@ public class QuotaSummaryResponse extends BaseResponse { @Param(description = "If the account has the quota config enabled") private boolean quotaEnabled; - public QuotaSummaryResponse() { - super(); - } + @SerializedName("projectname") + @Param(description = "Name of the project", since = "4.23.0") + private String projectName; + + @SerializedName("projectid") + @Param(description = "Project's id", since = "4.23.0") + private String projectId; + + @SerializedName("projectremoved") + @Param(description = "Whether the project is removed or not", since = "4.23.0") + private Boolean projectRemoved; public String getAccountId() { return accountId; @@ -101,28 +116,16 @@ public class QuotaSummaryResponse extends BaseResponse { this.domainId = domainId; } - public String getDomainName() { - return domainName; - } - - public void setDomainName(String domainName) { - this.domainName = domainName; - } - - public BigDecimal getQuotaUsage() { - return quotaUsage; - } - - public State getState() { - return state; + public void setDomainPath(String domainPath) { + this.domainPath = domainPath; } public void setState(State state) { this.state = state; } - public void setQuotaUsage(BigDecimal startQuota) { - this.quotaUsage = startQuota.setScale(2, RoundingMode.HALF_EVEN); + public void setQuotaUsage(BigDecimal quotaUsage) { + this.quotaUsage = quotaUsage; } public BigDecimal getBalance() { @@ -130,38 +133,42 @@ public class QuotaSummaryResponse extends BaseResponse { } public void setBalance(BigDecimal balance) { - this.balance = balance.setScale(2, RoundingMode.HALF_EVEN); - } - - public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + this.balance = balance; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); - } - - public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + this.startDate = startDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - public String getCurrency() { - return currency; + this.endDate = endDate; } public void setCurrency(String currency) { this.currency = currency; } - public boolean getQuotaEnabled() { - return quotaEnabled; - } - public void setQuotaEnabled(boolean quotaEnabled) { this.quotaEnabled = quotaEnabled; } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + public void setProjectRemoved(Boolean projectRemoved) { + this.projectRemoved = projectRemoved; + } + + public void setDomainRemoved(boolean domainRemoved) { + this.domainRemoved = domainRemoved; + } + + public void setAccountRemoved(boolean accountRemoved) { + this.accountRemoved = accountRemoved; + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index f455c3cba14..f39153ae0ac 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -26,6 +26,8 @@ import java.util.TimeZone; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.projects.ProjectManager; +import com.cloud.user.AccountService; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsCmd; @@ -75,6 +77,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private AccountDao _accountDao; @Inject + private AccountService accountService; + @Inject private QuotaAccountDao _quotaAcc; @Inject private QuotaUsageDao _quotaUsageDao; @@ -86,6 +90,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi private QuotaBalanceDao _quotaBalanceDao; @Inject private QuotaResponseBuilder _respBldr; + @Inject + private ProjectManager projectMgr; private TimeZone _usageTimezone; diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index 3629bf2e3fe..ea88a106b84 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -16,13 +16,11 @@ // under the License. package org.apache.cloudstack.api.response; -import java.lang.reflect.Field; import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -31,6 +29,7 @@ import java.util.Set; import java.util.HashSet; import java.util.function.Consumer; +import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.PermissionDeniedException; @@ -43,10 +42,10 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.discovery.ApiDiscoveryService; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.jsinterpreter.JsInterpreterHelper; import org.apache.cloudstack.quota.QuotaService; import org.apache.cloudstack.quota.QuotaStatement; @@ -79,7 +78,10 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.Account; @@ -89,8 +91,7 @@ import com.cloud.user.dao.UserDao; import com.cloud.user.User; import junit.framework.TestCase; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; + @RunWith(MockitoJUnitRunner.class) public class QuotaResponseBuilderImplTest extends TestCase { @@ -153,7 +154,7 @@ public class QuotaResponseBuilderImplTest extends TestCase { Account accountMock; @Mock - DomainVO domainVOMock; + DomainVO domainVoMock; @Mock QuotaConfigureEmailCmd quotaConfigureEmailCmdMock; @@ -161,6 +162,9 @@ public class QuotaResponseBuilderImplTest extends TestCase { @Mock QuotaAccountVO quotaAccountVOMock; + @Mock + CallContext callContextMock; + @Mock QuotaEmailTemplatesVO quotaEmailTemplatesVoMock; @@ -184,17 +188,8 @@ public class QuotaResponseBuilderImplTest extends TestCase { CallContext.register(callerUserMock, callerAccountMock); } - private void overrideDefaultQuotaEnabledConfigValue(final Object value) throws IllegalAccessException, NoSuchFieldException { - Field f = ConfigKey.class.getDeclaredField("_defaultValue"); - f.setAccessible(true); - f.set(QuotaConfig.QuotaAccountEnabled, value); - } - - private Calendar[] createPeriodForQuotaSummary() { - final Calendar calendar = Calendar.getInstance(); - calendar.set(Calendar.HOUR, 0); - return new Calendar[] {calendar, calendar}; - } + @Mock + Pair, Integer> quotaSummaryResponseMock1, quotaSummaryResponseMock2; @Mock QuotaValidateActivationRuleCmd quotaValidateActivationRuleCmdMock = Mockito.mock(QuotaValidateActivationRuleCmd.class); @@ -466,36 +461,6 @@ public class QuotaResponseBuilderImplTest extends TestCase { Mockito.verify(quotaTariffVoMock).setRemoved(Mockito.any(Date.class)); } - @Test - public void getQuotaSummaryResponseTestAccountIsNotNullQuotaIsDisabledShouldReturnFalse() throws NoSuchFieldException, IllegalAccessException { - Calendar[] period = createPeriodForQuotaSummary(); - overrideDefaultQuotaEnabledConfigValue("false"); - - Mockito.doReturn(period).when(quotaStatementMock).getCurrentStatementTime(); - Mockito.doReturn(domainVOMock).when(domainDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(BigDecimal.ZERO).when(quotaBalanceDaoMock).lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class)); - Mockito.doReturn(BigDecimal.ZERO).when(quotaUsageDaoMock).findTotalQuotaUsage(Mockito.anyLong(), Mockito.anyLong(), Mockito.isNull(), Mockito.any(Date.class), Mockito.any(Date.class)); - - QuotaSummaryResponse quotaSummaryResponse = quotaResponseBuilderSpy.getQuotaSummaryResponse(accountMock); - - assertFalse(quotaSummaryResponse.getQuotaEnabled()); - } - - @Test - public void getQuotaSummaryResponseTestAccountIsNotNullQuotaIsEnabledShouldReturnTrue() throws NoSuchFieldException, IllegalAccessException { - Calendar[] period = createPeriodForQuotaSummary(); - overrideDefaultQuotaEnabledConfigValue("true"); - - Mockito.doReturn(period).when(quotaStatementMock).getCurrentStatementTime(); - Mockito.doReturn(domainVOMock).when(domainDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(BigDecimal.ZERO).when(quotaBalanceDaoMock).lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class)); - Mockito.doReturn(BigDecimal.ZERO).when(quotaUsageDaoMock).findTotalQuotaUsage(Mockito.anyLong(), Mockito.anyLong(), Mockito.isNull(), Mockito.any(Date.class), Mockito.any(Date.class)); - - QuotaSummaryResponse quotaSummaryResponse = quotaResponseBuilderSpy.getQuotaSummaryResponse(accountMock); - - assertTrue(quotaSummaryResponse.getQuotaEnabled()); - } - @Test public void filterSupportedTypesTestReturnWhenQuotaTypeDoesNotMatch() throws NoSuchFieldException { List> variables = new ArrayList<>(); @@ -576,6 +541,63 @@ public class QuotaResponseBuilderImplTest extends TestCase { quotaResponseBuilderSpy.validateQuotaConfigureEmailCmdParameters(quotaConfigureEmailCmdMock); } + @Test + public void createQuotaSummaryResponseTestNotListAllAndAllAccountTypesReturnsSingleRecord() { + QuotaSummaryCmd cmd = new QuotaSummaryCmd(); + + try(MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + + Mockito.doReturn(quotaSummaryResponseMock1).when(quotaResponseBuilderSpy).getQuotaSummaryResponse(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + + for (Account.Type type : Account.Type.values()) { + Mockito.doReturn(type).when(accountMock).getType(); + + Pair, Integer> result = quotaResponseBuilderSpy.createQuotaSummaryResponse(cmd); + Assert.assertEquals(quotaSummaryResponseMock1, result); + } + + Mockito.verify(quotaResponseBuilderSpy, Mockito.times(Account.Type.values().length)).getQuotaSummaryResponse(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any()); + }; + } + + @Test + public void getDomainPathByDomainIdForDomainAdminTestAccountNotDomainAdminReturnsNull() { + for (Account.Type type : Account.Type.values()) { + if (Account.Type.DOMAIN_ADMIN.equals(type)) { + continue; + } + + Mockito.doReturn(type).when(accountMock).getType(); + Assert.assertNull(quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock)); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void getDomainPathByDomainIdForDomainAdminTestDomainFromCallerIsNullThrowsInvalidParameterValueException() { + Mockito.doReturn(Account.Type.DOMAIN_ADMIN).when(accountMock).getType(); + Mockito.doReturn(null).when(domainDaoMock).findById(Mockito.anyLong()); + Mockito.lenient().doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + + quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock); + } + + @Test + public void getDomainPathByDomainIdForDomainAdminTestDomainFromCallerIsNotNullReturnsPath() { + String expected = "/test/"; + + Mockito.doReturn(Account.Type.DOMAIN_ADMIN).when(accountMock).getType(); + Mockito.doReturn(domainVoMock).when(domainDaoMock).findById(Mockito.anyLong()); + Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + Mockito.doReturn(expected).when(domainVoMock).getPath(); + + String result = quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock); + Assert.assertEquals(expected, result); + } + @Test public void getQuotaEmailConfigurationVoTestTemplateNameIsNull() { Mockito.doReturn(null).when(quotaConfigureEmailCmdMock).getTemplateName(); diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 19e756d1d97..259264f3b0e 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.quota; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; +import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; import junit.framework.TestCase; @@ -33,8 +34,10 @@ import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import javax.naming.ConfigurationException; @@ -48,7 +51,7 @@ import java.util.List; public class QuotaServiceImplTest extends TestCase { @Mock - AccountDao accountDao; + AccountDao accountDaoMock; @Mock QuotaAccountDao quotaAcc; @Mock @@ -61,8 +64,13 @@ public class QuotaServiceImplTest extends TestCase { QuotaBalanceDao quotaBalanceDao; @Mock QuotaResponseBuilder respBldr; + @Mock + private AccountVO accountVoMock; + + @Spy + @InjectMocks + QuotaServiceImpl quotaServiceImplSpy; - QuotaServiceImpl quotaService = new QuotaServiceImpl(); @Before public void setup() throws IllegalAccessException, NoSuchFieldException, ConfigurationException { @@ -71,34 +79,34 @@ public class QuotaServiceImplTest extends TestCase { Field accountDaoField = QuotaServiceImpl.class.getDeclaredField("_accountDao"); accountDaoField.setAccessible(true); - accountDaoField.set(quotaService, accountDao); + accountDaoField.set(quotaServiceImplSpy, accountDaoMock); Field quotaAccountDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaAcc"); quotaAccountDaoField.setAccessible(true); - quotaAccountDaoField.set(quotaService, quotaAcc); + quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaService, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); - domainDaoField.set(quotaService, domainDao); + domainDaoField.set(quotaServiceImplSpy, domainDao); Field configDaoField = QuotaServiceImpl.class.getDeclaredField("_configDao"); configDaoField.setAccessible(true); - configDaoField.set(quotaService, configDao); + configDaoField.set(quotaServiceImplSpy, configDao); Field balanceDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaBalanceDao"); balanceDaoField.setAccessible(true); - balanceDaoField.set(quotaService, quotaBalanceDao); + balanceDaoField.set(quotaServiceImplSpy, quotaBalanceDao); Field QuotaResponseBuilderField = QuotaServiceImpl.class.getDeclaredField("_respBldr"); QuotaResponseBuilderField.setAccessible(true); - QuotaResponseBuilderField.set(quotaService, respBldr); + QuotaResponseBuilderField.set(quotaServiceImplSpy, respBldr); Mockito.when(configDao.getValue(Mockito.eq(Config.UsageAggregationTimezone.toString()))).thenReturn("IST"); - quotaService.configure("randomName", null); + quotaServiceImplSpy.configure("randomName", null); } @Test @@ -120,9 +128,9 @@ public class QuotaServiceImplTest extends TestCase { Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); // with enddate - assertTrue(quotaService.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); + assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); // without enddate - assertTrue(quotaService.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); + assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); } @Test @@ -133,7 +141,7 @@ public class QuotaServiceImplTest extends TestCase { final Date startDate = new DateTime().minusDays(2).toDate(); final Date endDate = new Date(); - quotaService.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); + quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); } @@ -142,13 +150,13 @@ public class QuotaServiceImplTest extends TestCase { // existing account QuotaAccountVO quotaAccountVO = new QuotaAccountVO(); Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(quotaAccountVO); - quotaService.setLockAccount(2L, true); + quotaServiceImplSpy.setLockAccount(2L, true); Mockito.verify(quotaAcc, Mockito.times(0)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); Mockito.verify(quotaAcc, Mockito.times(1)).updateQuotaAccount(Mockito.anyLong(), Mockito.any(QuotaAccountVO.class)); // new account Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(null); - quotaService.setLockAccount(2L, true); + quotaServiceImplSpy.setLockAccount(2L, true); Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } @@ -160,13 +168,14 @@ public class QuotaServiceImplTest extends TestCase { // existing account setting QuotaAccountVO quotaAccountVO = new QuotaAccountVO(); Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(quotaAccountVO); - quotaService.setMinBalance(accountId, balance); + quotaServiceImplSpy.setMinBalance(accountId, balance); Mockito.verify(quotaAcc, Mockito.times(0)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); Mockito.verify(quotaAcc, Mockito.times(1)).updateQuotaAccount(Mockito.anyLong(), Mockito.any(QuotaAccountVO.class)); // no account with limit set Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(null); - quotaService.setMinBalance(accountId, balance); + quotaServiceImplSpy.setMinBalance(accountId, balance); Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } + } diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java index 7953a6a27cd..d9f4963165e 100644 --- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java +++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java @@ -485,6 +485,12 @@ public class MockAccountManager extends ManagerBase implements AccountManager { return null; } + @Override + public Long finalizeAccountId(Long accountId, String accountName, Long domainId, Long projectId) { + // TODO Auto-generated method stub + return null; + } + @Override public void checkAccess(Account account, ServiceOffering so, DataCenter zone) throws PermissionDeniedException { // TODO Auto-generated method stub diff --git a/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java b/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java index 5a634cc0ca8..0205bfa9d8f 100644 --- a/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java +++ b/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java @@ -314,20 +314,7 @@ public class ParamProcessWorker implements DispatchWorker { protected void doAccessChecks(BaseCmd cmd, Map entitiesToAccess) { Account caller = CallContext.current().getCallingAccount(); - List entityOwners = cmd.getEntityOwnerIds(); - Account[] owners = null; - if (entityOwners != null) { - owners = entityOwners.stream().map(id -> _accountMgr.getAccount(id)).toArray(Account[]::new); - } else { - if (cmd.getEntityOwnerId() == Account.ACCOUNT_ID_SYSTEM && cmd instanceof BaseAsyncCmd && ((BaseAsyncCmd)cmd).getApiResourceType() == ApiCommandResourceType.Network) { - if (logger.isDebugEnabled()) { - logger.debug("Skipping access check on the network owner if the owner is ROOT/system."); - } - owners = new Account[]{}; - } else { - owners = new Account[]{_accountMgr.getAccount(cmd.getEntityOwnerId())}; - } - } + Account[] owners = getEntityOwners(cmd); if (cmd instanceof BaseAsyncCreateCmd) { // check that caller can access the owner account. diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index d9af7af6c33..2011d455646 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -73,7 +73,9 @@ import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.account.CreateAccountCmd; import org.apache.cloudstack.api.command.admin.account.UpdateAccountCmd; import org.apache.cloudstack.api.command.admin.user.DeleteUserCmd; @@ -3886,6 +3888,48 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M return null; } + @Override + public Long finalizeAccountId(Long accountId, String accountName, Long domainId, Long projectId) { + if (projectId != null) { + if (ObjectUtils.anyNotNull(accountId, accountName)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Project and account can not be specified together."); + } + return getActiveProjectAccountByProjectId(projectId); + } + if (accountId != null) { + if (getActiveAccountById(accountId) != null) { + return accountId; + } + throw new InvalidParameterValueException(String.format("Unable to find account with ID [%s].", accountId)); + } + + if (accountName == null && domainId == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Either %s or %s must be informed.", ApiConstants.ACCOUNT_ID, ApiConstants.PROJECT_ID)); + } + + try { + Account activeAccount = getActiveAccountByName(accountName, domainId); + if (activeAccount != null) { + return activeAccount.getId(); + } + } catch (InvalidParameterValueException exception) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Both %s and %s are needed if using either. Consider using %s instead.", + ApiConstants.ACCOUNT, ApiConstants.DOMAIN_ID, ApiConstants.ACCOUNT_ID)); + } + throw new InvalidParameterValueException(String.format("Unable to find account by name [%s] on domain [%s].", accountName, domainId)); + } + + protected long getActiveProjectAccountByProjectId(long projectId) { + Project project = _projectMgr.getProject(projectId); + if (project == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Unable to find project with ID [%s].", projectId)); + } + if (project.getState() != Project.State.Active) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Project with ID [%s] is not active.", projectId)); + } + return project.getProjectAccountId(); + } + @Override public UserAccount getUserAccountById(Long userId) { UserAccount userAccount = userAccountDao.findById(userId); From b805766f4baf322988b8628c2ac5dc655383e2b3 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Wed, 1 Apr 2026 07:25:19 -0400 Subject: [PATCH 08/26] Fix Host setup when persistent networks exist (#12751) --- .../src/main/java/com/cloud/network/dao/NetworkDaoImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 8066b89b4b9..5887498e094 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -193,6 +193,7 @@ public class NetworkDaoImpl extends GenericDaoBaseimplements Ne PersistentNetworkSearch.and("id", PersistentNetworkSearch.entity().getId(), Op.NEQ); PersistentNetworkSearch.and("guestType", PersistentNetworkSearch.entity().getGuestType(), Op.IN); PersistentNetworkSearch.and("broadcastUri", PersistentNetworkSearch.entity().getBroadcastUri(), Op.EQ); + PersistentNetworkSearch.and("dc", PersistentNetworkSearch.entity().getDataCenterId(), Op.EQ); PersistentNetworkSearch.and("removed", PersistentNetworkSearch.entity().getRemoved(), Op.NULL); final SearchBuilder persistentNtwkOffJoin = _ntwkOffDao.createSearchBuilder(); persistentNtwkOffJoin.and("persistent", persistentNtwkOffJoin.entity().isPersistent(), Op.EQ); From e10c066cc14306193f4cebc63a3ec1ee07e41084 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 1 Apr 2026 19:59:44 +0530 Subject: [PATCH 09/26] Fix NPE during VM setup for pvlan (#12781) * Fix NPE during VM setup for pvlan * review comments --- .../java/com/cloud/vm/UserVmManagerImpl.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4cb721666bd..65bd285ca90 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5430,7 +5430,19 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Override public boolean setupVmForPvlan(boolean add, Long hostId, NicProfile nic) { - if (!nic.getBroadCastUri().getScheme().equals("pvlan")) { + if (nic == null) { + logger.warn("Skipping PVLAN setup on host {} because NIC profile is null", hostId); + return false; + } + + if (nic.getBroadCastUri() == null) { + logger.debug("Skipping PVLAN setup on host {} for NIC {} because broadcast URI is null", hostId, nic); + return false; + } + + String scheme = nic.getBroadCastUri().getScheme(); + if (!"pvlan".equalsIgnoreCase(scheme)) { + logger.debug("Skipping PVLAN setup on host {} for NIC {} because broadcast URI scheme is {}", hostId, nic, scheme); return false; } String op = "add"; @@ -5438,11 +5450,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // "delete" would remove all the rules(if using ovs) related to this vm op = "delete"; } - Network network = _networkDao.findById(nic.getNetworkId()); + Host host = _hostDao.findById(hostId); + if (host == null) { + logger.warn("Host with id {} does not exist", hostId); + return false; + } + + Network network = _networkDao.findById(nic.getNetworkId()); String networkTag = _networkModel.getNetworkTag(host.getHypervisorType(), network); PvlanSetupCommand cmd = PvlanSetupCommand.createVmSetup(op, nic.getBroadCastUri(), networkTag, nic.getMacAddress()); - Answer answer = null; + Answer answer; try { answer = _agentMgr.send(hostId, cmd); } catch (OperationTimedoutException e) { From 470812100ea6e9b5790bbc3a9f747d7d733ee36d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 2 Apr 2026 06:04:28 +0200 Subject: [PATCH 10/26] server: set template type to ROUTING or USER if template type is not specified when upload a template (#12768) --- .../src/main/resources/META-INF/db/schema-42200to42210.sql | 3 +++ .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index 2326a855f32..369159e4836 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -35,3 +35,6 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template'; + +-- Update existing vm_template records with NULL type to "USER" +UPDATE `cloud`.`vm_template` SET `type` = 'USER' WHERE `type` IS NULL; diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index ff810ef9231..a2cb82a0a6b 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -2416,7 +2416,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } else if ((cmd instanceof RegisterVnfTemplateCmd || cmd instanceof UpdateVnfTemplateCmd) && !TemplateType.VNF.equals(templateType)) { throw new InvalidParameterValueException("The template type must be VNF for VNF templates, but the actual type is " + templateType); } - } else if (cmd instanceof RegisterTemplateCmd) { + } else if (cmd instanceof RegisterTemplateCmd || cmd instanceof GetUploadParamsForTemplateCmd) { boolean isRouting = Boolean.TRUE.equals(isRoutingType); templateType = (cmd instanceof RegisterVnfTemplateCmd) ? TemplateType.VNF : (isRouting ? TemplateType.ROUTING : TemplateType.USER); } @@ -2426,6 +2426,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException(String.format("Users can not register Template with template type %s.", templateType)); } else if (cmd instanceof UpdateTemplateCmd) { throw new InvalidParameterValueException(String.format("Users can not update Template to template type %s.", templateType)); + } else if (cmd instanceof GetUploadParamsForTemplateCmd) { + throw new InvalidParameterValueException(String.format("Users can not request upload parameters for Template with template type %s.", templateType)); } } return templateType; From 30dd234b000845706c7f79f44ed498fd4eab01f2 Mon Sep 17 00:00:00 2001 From: Daniil Zhyliaiev Date: Mon, 6 Apr 2026 21:50:17 +0300 Subject: [PATCH 11/26] fix: NsxResource.executeRequest DeleteNsxNatRuleCommand comparison bug (#12833) Fixes an issue in NsxResource.executeRequest where Network.Service comparison failed when DeleteNsxNatRuleCommand was executed in a different process. Due to serialization/deserialization, the deserialized Network.Service instance was not equal to the static instances Network.Service.StaticNat and Network.Service.PortForwarding, causing the comparison to always return false. Co-authored-by: Andrey Volchkov --- .../cloudstack/agent/api/DeleteNsxNatRuleCommand.java | 7 +++++++ .../java/org/apache/cloudstack/resource/NsxResource.java | 6 +++--- .../org/apache/cloudstack/resource/NsxResourceTest.java | 6 ++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java index c5231b19ac4..b642df85618 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxNatRuleCommand.java @@ -54,6 +54,13 @@ public class DeleteNsxNatRuleCommand extends NsxNetworkCommand { return protocol; } + public String getNetworkServiceName() { + if (service != null) { + return service.getName(); + } + return null; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java index 76815b0deeb..78a9363a5e4 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java @@ -415,10 +415,10 @@ public class NsxResource implements ServerResource { private NsxAnswer executeRequest(DeleteNsxNatRuleCommand cmd) { String ruleName = null; - if (cmd.getService() == Network.Service.StaticNat) { + if (Network.Service.StaticNat.getName().equals(cmd.getNetworkServiceName())) { ruleName = NsxControllerUtils.getStaticNatRuleName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), cmd.getNetworkResourceId(), cmd.isResourceVpc()); - } else if (cmd.getService() == Network.Service.PortForwarding) { + } else if (Network.Service.PortForwarding.getName().equals(cmd.getNetworkServiceName())) { ruleName = NsxControllerUtils.getPortForwardRuleName(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), cmd.getNetworkResourceId(), cmd.getRuleId(), cmd.isResourceVpc()); } @@ -456,7 +456,7 @@ public class NsxResource implements ServerResource { try { nsxApiClient.deleteNsxLbResources(tier1GatewayName, cmd.getLbId()); } catch (Exception e) { - logger.error(String.format("Failed to add NSX load balancer rule %s for network: %s", ruleName, cmd.getNetworkResourceName())); + logger.error(String.format("Failed to delete NSX load balancer rule %s for network: %s", ruleName, cmd.getNetworkResourceName())); return new NsxAnswer(cmd, new CloudRuntimeException(e.getMessage())); } return new NsxAnswer(cmd, true, null); diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java index ee4f4fb64c2..0d74bb8a3b3 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/resource/NsxResourceTest.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.resource; +import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; import com.cloud.utils.exception.CloudRuntimeException; import com.vmware.nsx.model.TransportZone; @@ -61,6 +62,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -247,8 +249,12 @@ public class NsxResourceTest { @Test public void testDeleteNsxNatRule() { DeleteNsxNatRuleCommand cmd = new DeleteNsxNatRuleCommand(domainId, accountId, zoneId, 3L, "VPC01", true, 2L, 5L, "22", "tcp"); + Network.Service service = mock(Network.Service.class); + when(service.getName()).thenReturn("PortForwarding"); + cmd.setService(service); NsxAnswer answer = (NsxAnswer) nsxResource.executeRequest(cmd); assertTrue(answer.getResult()); + verify(nsxApi).deleteNatRule(service, "22", "tcp", "VPC01", "D1-A2-Z1-V3", "D1-A2-Z1-V3-PF5"); } @Test From abdf926219af0faa90c6515bb6ad51f0e8881890 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 8 Apr 2026 09:43:44 +0530 Subject: [PATCH 12/26] Revert "Use lateral join (introduced in MySQL 8.0.14) with subquery on user_statistics table in account_view for netstats (#12631)" (#12965) This reverts commit 58916eb608036669c3fabe0239b339745b8475cf. --- .../db/schema-42200to42210-cleanup.sql | 2 -- .../db/views/cloud.account_netstats_view.sql | 31 +++++++++++++++++++ .../META-INF/db/views/cloud.account_view.sql | 15 +++------ 3 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 505c8ef5715..54baf226ac4 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,5 +18,3 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; - -DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql new file mode 100644 index 00000000000..11193c465fd --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- cloud.account_netstats_view source + + +DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; + +CREATE VIEW `cloud`.`account_netstats_view` AS +select + `user_statistics`.`account_id` AS `account_id`, + (sum(`user_statistics`.`net_bytes_received`) + sum(`user_statistics`.`current_bytes_received`)) AS `bytesReceived`, + (sum(`user_statistics`.`net_bytes_sent`) + sum(`user_statistics`.`current_bytes_sent`)) AS `bytesSent` +from + `user_statistics` +group by + `user_statistics`.`account_id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql index 327c6c627e2..edc164c40cb 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -39,8 +39,8 @@ select `data_center`.`id` AS `data_center_id`, `data_center`.`uuid` AS `data_center_uuid`, `data_center`.`name` AS `data_center_name`, - `account_netstats`.`bytesReceived` AS `bytesReceived`, - `account_netstats`.`bytesSent` AS `bytesSent`, + `account_netstats_view`.`bytesReceived` AS `bytesReceived`, + `account_netstats_view`.`bytesSent` AS `bytesSent`, `vmlimit`.`max` AS `vmLimit`, `vmcount`.`count` AS `vmTotal`, `runningvm`.`vmcount` AS `runningVms`, @@ -89,15 +89,8 @@ from `cloud`.`domain` ON account.domain_id = domain.id left join `cloud`.`data_center` ON account.default_zone_id = data_center.id - left join lateral ( - select - coalesce(sum(`user_statistics`.`net_bytes_received` + `user_statistics`.`current_bytes_received`), 0) AS `bytesReceived`, - coalesce(sum(`user_statistics`.`net_bytes_sent` + `user_statistics`.`current_bytes_sent`), 0) AS `bytesSent` - from - `cloud`.`user_statistics` - where - `user_statistics`.`account_id` = `account`.`id` - ) AS `account_netstats` ON TRUE + left join + `cloud`.`account_netstats_view` ON account.id = account_netstats_view.account_id left join `cloud`.`resource_limit` vmlimit ON account.id = vmlimit.account_id and vmlimit.type = 'user_vm' and vmlimit.tag IS NULL From 03de62bf3890d686e58d90c1cc5972a75b65ee24 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 8 Apr 2026 15:14:20 +0530 Subject: [PATCH 13/26] Support Linstor Primary Storage for NAS BnR (#12796) --- .../backup/RestoreBackupCommand.java | 9 ++ .../cloudstack/backup/NASBackupProvider.java | 21 +++- .../LibvirtRestoreBackupCommandWrapper.java | 95 +++++++++++++------ ...ibvirtRestoreBackupCommandWrapperTest.java | 29 ++++++ scripts/vm/hypervisor/kvm/nasbackup.sh | 59 ++++++++++-- 5 files changed, 174 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index f5ad5fbea2c..972c2eaf7bb 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -34,6 +34,7 @@ public class RestoreBackupCommand extends Command { private List backupVolumesUUIDs; private List restoreVolumePools; private List restoreVolumePaths; + private List restoreVolumeSizes; private List backupFiles; private String diskType; private Boolean vmExists; @@ -92,6 +93,14 @@ public class RestoreBackupCommand extends Command { this.restoreVolumePaths = restoreVolumePaths; } + public List getRestoreVolumeSizes() { + return restoreVolumeSizes; + } + + public void setRestoreVolumeSizes(List restoreVolumeSizes) { + this.restoreVolumeSizes = restoreVolumeSizes; + } + public List getBackupFiles() { return backupFiles; } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index d4068d498d4..fb1b78eb963 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -351,7 +351,8 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co volumePools.add(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null); String volumePathPrefix = getVolumePathPrefix(storagePool); - volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath())); + String volumePathSuffix = getVolumePathSuffix(storagePool); + volumePaths.add(String.format("%s%s%s", volumePathPrefix, volume.getPath(), volumePathSuffix)); } return new Pair<>(volumePools, volumePaths); } @@ -361,14 +362,24 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co if (ScopeType.HOST.equals(storagePool.getScope()) || Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType()) || Storage.StoragePoolType.RBD.equals(storagePool.getPoolType())) { - volumePathPrefix = storagePool.getPath(); + volumePathPrefix = storagePool.getPath() + "/"; + } else if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + volumePathPrefix = "/dev/drbd/by-res/cs-"; } else { // Should be Storage.StoragePoolType.NetworkFilesystem - volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); + volumePathPrefix = String.format("/mnt/%s/", storagePool.getUuid()); } return volumePathPrefix; } + private String getVolumePathSuffix(StoragePoolVO storagePool) { + if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + return "/0"; + } else { + return ""; + } + } + @Override public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); @@ -413,7 +424,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoreCommand.setBackupRepoType(backupRepository.getType()); restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setVmName(vmNameAndState.first()); - restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", getVolumePathPrefix(pool), volumeUUID))); + String restoreVolumePath = String.format("%s%s%s", getVolumePathPrefix(pool), volumeUUID, getVolumePathSuffix(pool)); + restoreCommand.setRestoreVolumePaths(Collections.singletonList(restoreVolumePath)); + restoreCommand.setRestoreVolumeSizes(Collections.singletonList(backedUpVolumeSize)); DataStore dataStore = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); restoreCommand.setRestoreVolumePools(Collections.singletonList(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null)); restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT)); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 714e3844b34..46561a9bddf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -41,9 +41,9 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.Locale; @@ -56,10 +56,25 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(vmName, command.getVmState()), mountDirectory, timeout); } else if (Boolean.TRUE.equals(vmExists)) { restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); @@ -143,7 +158,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper vmNameAndState, String mountDirectory, int timeout) { + Long size, Pair vmNameAndState, String mountDirectory, int timeout) { String bkpPath; String volumeUuid; try { bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType); - volumeUuid = volumePath.substring(volumePath.lastIndexOf(File.separator) + 1); + volumeUuid = getVolumeUuidFromPath(volumePath, volumePool); verifyBackupFile(bkpPath, volumeUuid); - if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true)) { + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true, size)) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", volumeUuid)); } @@ -247,42 +262,66 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper rbd:/:mon_host=... + # sample: rbd:cloudstack/53d5c355-d726-4d3e-9422-046a503a0b12:mon_host=10.0.1.2... + local beforeUuid="${fullpath#*/}" # Remove up to first slash after rbd: + local volUuid="${beforeUuid%%:*}" # Remove everything after colon to get the uuid + echo ""$volUuid"" +} + +get_linstor_uuid_from_path() { + local fullpath="$1" + # disk for linstor => /dev/drbd/by-res/cs-/0 + # sample: /dev/drbd/by-res/cs-53d5c355-d726-4d3e-9422-046a503a0b12/0 + local beforeUuid="${fullpath#/dev/drbd/by-res/}" + local volUuid="${beforeUuid%%/*}" + volUuid="${volUuid#cs-}" + echo "$volUuid" +} + backup_running_vm() { mount_operation mkdir -p "$dest" || { echo "Failed to create backup directory $dest"; exit 1; } name="root" echo "" > $dest/backup.xml - for disk in $(virsh -c qemu:///system domblklist $VM --details 2>/dev/null | awk '/disk/{print$3}'); do - volpath=$(virsh -c qemu:///system domblklist $VM --details | awk "/$disk/{print $4}" | sed 's/.*\///') - echo "" >> $dest/backup.xml + while read -r disk fullpath; do + if [[ "$fullpath" == /dev/drbd/by-res/* ]]; then + volUuid=$(get_linstor_uuid_from_path "$fullpath") + else + volUuid="${fullpath##*/}" + fi + echo "" >> $dest/backup.xml name="datadisk" - done + done < <( + virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}' + ) echo "" >> $dest/backup.xml local thaw=0 @@ -146,6 +171,25 @@ backup_running_vm() { sleep 5 done + # Use qemu-img convert to sparsify linstor backups which get bloated due to virsh backup-begin. + name="root" + while read -r disk fullpath; do + if [[ "$fullpath" != /dev/drbd/by-res/* ]]; then + continue + fi + volUuid=$(get_linstor_uuid_from_path "$fullpath") + if ! qemu-img convert -O qcow2 "$dest/$name.$volUuid.qcow2" "$dest/$name.$volUuid.qcow2.tmp" >> "$logFile" 2> >(cat >&2); then + echo "qemu-img convert failed for $dest/$name.$volUuid.qcow2" + cleanup + exit 1 + fi + + mv "$dest/$name.$volUuid.qcow2.tmp" "$dest/$name.$volUuid.qcow2" + name="datadisk" + done < <( + virsh -c qemu:///system domblklist "$VM" --details 2>/dev/null | awk '$2=="disk"{print $3, $4}' + ) + rm -f $dest/backup.xml sync @@ -166,10 +210,9 @@ backup_stopped_vm() { name="root" for disk in $DISK_PATHS; do if [[ "$disk" == rbd:* ]]; then - # disk for rbd => rbd:/:mon_host=... - # sample: rbd:cloudstack/53d5c355-d726-4d3e-9422-046a503a0b12:mon_host=10.0.1.2... - beforeUuid="${disk#*/}" # Remove up to first slash after rbd: - volUuid="${beforeUuid%%:*}" # Remove everything after colon to get the uuid + volUuid=$(get_ceph_uuid_from_path "$disk") + elif [[ "$disk" == /dev/drbd/by-res/* ]]; then + volUuid=$(get_linstor_uuid_from_path "$disk") else volUuid="${disk##*/}" fi From 7ba5240b311f4b66ff4d2c2ced0a14292d7cd62f Mon Sep 17 00:00:00 2001 From: Daman Arora <61474540+Damans227@users.noreply.github.com> Date: Wed, 8 Apr 2026 22:39:01 -0700 Subject: [PATCH 14/26] Block backup deletion while create-VM-from-backup or restore jobs are in progress (#12792) * Block backup deletion while create-VM-from-backup or restore jobs are in progress * Add tests * Fix exception message * Update test Co-authored-by: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> --- .../cloudstack/backup/BackupManagerImpl.java | 14 +++++++++ .../cloudstack/backup/BackupManagerTest.java | 30 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index d5663ed3272..e9ed8e42eaa 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -1531,6 +1531,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { validateBackupForZone(backup.getZoneId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm == null ? backup : vm); + + checkForPendingBackupJobs(backup); final BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(backup.getBackupOfferingId()); if (offering == null) { throw new CloudRuntimeException(String.format("Backup offering with ID [%s] does not exist.", backup.getBackupOfferingId())); @@ -1551,6 +1553,18 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { throw new CloudRuntimeException("Failed to delete the backup"); } + private void checkForPendingBackupJobs(final BackupVO backup) { + String backupUuid = backup.getUuid(); + long pendingJobs = asyncJobManager.countPendingJobs(backupUuid, + CreateVMFromBackupCmd.class.getName(), + CreateVMFromBackupCmdByAdmin.class.getName(), + RestoreBackupCmd.class.getName(), + RestoreVolumeFromBackupAndAttachToVMCmd.class.getName()); + if (pendingJobs > 0) { + throw new CloudRuntimeException("Cannot delete Backup while a create Instance from Backup or restore Backup operation is in progress, please try again later."); + } + } + /** * Get the pair: hostIp, datastoreUuid in which to restore the volume, based on the VM to be attached information */ diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 8b13fd47494..2c9d11612c4 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -91,6 +91,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.junit.After; import org.junit.Assert; @@ -241,6 +242,9 @@ public class BackupManagerTest { @Mock private GuestOSDao _guestOSDao; + @Mock + AsyncJobManager asyncJobManager; + private Gson gson; private String[] hostPossibleValues = {"127.0.0.1", "hostname"}; @@ -1489,6 +1493,7 @@ public class BackupManagerTest { when(backup.getAccountId()).thenReturn(accountId); when(backup.getBackupOfferingId()).thenReturn(backupOfferingId); when(backup.getSize()).thenReturn(100L); + when(backup.getUuid()).thenReturn("backup-uuid"); overrideBackupFrameworkConfigValue(); @@ -1523,6 +1528,31 @@ public class BackupManagerTest { } } + @Test(expected = CloudRuntimeException.class) + public void testDeleteBackupBlockedByPendingJobs() { + Long backupId = 1L; + Long vmId = 2L; + + BackupVO backup = mock(BackupVO.class); + when(backup.getVmId()).thenReturn(vmId); + when(backup.getUuid()).thenReturn("backup-uuid"); + when(backup.getZoneId()).thenReturn(1L); + when(backupDao.findByIdIncludingRemoved(backupId)).thenReturn(backup); + + VMInstanceVO vm = mock(VMInstanceVO.class); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + + overrideBackupFrameworkConfigValue(); + + when(asyncJobManager.countPendingJobs("backup-uuid", + "org.apache.cloudstack.api.command.user.vm.CreateVMFromBackupCmd", + "org.apache.cloudstack.api.command.admin.vm.CreateVMFromBackupCmdByAdmin", + "org.apache.cloudstack.api.command.user.backup.RestoreBackupCmd", + "org.apache.cloudstack.api.command.user.backup.RestoreVolumeFromBackupAndAttachToVMCmd")).thenReturn(1L); + + backupManager.deleteBackup(backupId, false); + } + @Test public void testNewBackupResponse() { Long vmId = 1L; From 1ff9eec9977fc68f59e11644b9543177df403ab5 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Thu, 9 Apr 2026 13:19:49 +0530 Subject: [PATCH 15/26] Load arch data for backup from template during create instance from backup (#12801) --- ui/src/components/view/DeployVMFromBackup.vue | 46 +------------------ ui/src/views/storage/CreateVMFromBackup.vue | 25 ++++++++-- 2 files changed, 24 insertions(+), 47 deletions(-) diff --git a/ui/src/components/view/DeployVMFromBackup.vue b/ui/src/components/view/DeployVMFromBackup.vue index 55808db291c..888d29509e1 100644 --- a/ui/src/components/view/DeployVMFromBackup.vue +++ b/ui/src/components/view/DeployVMFromBackup.vue @@ -1451,7 +1451,7 @@ export default { this.initForm() this.dataPreFill = this.preFillContent && Object.keys(this.preFillContent).length > 0 ? this.preFillContent : {} this.showOverrideDiskOfferingOption = this.dataPreFill.overridediskoffering - + this.selectedArchitecture = this.dataPreFill.backupArch ? this.dataPreFill.backupArch : this.architectureTypes.opts[0].id if (this.dataPreFill.isIso) { this.tabKey = 'isoid' } else { @@ -1540,46 +1540,6 @@ export default { fillValue (field) { this.form[field] = this.dataPreFill[field] }, - fetchZoneByQuery () { - return new Promise(resolve => { - let zones = [] - let apiName = '' - const params = {} - if (this.templateId) { - apiName = 'listTemplates' - params.listall = true - params.templatefilter = this.isNormalAndDomainUser ? 'executable' : 'all' - params.id = this.templateId - } else if (this.isoId) { - apiName = 'listIsos' - params.listall = true - params.isofilter = this.isNormalAndDomainUser ? 'executable' : 'all' - params.id = this.isoId - } else if (this.networkId) { - params.listall = true - params.id = this.networkId - apiName = 'listNetworks' - } - if (!apiName) return resolve(zones) - - getAPI(apiName, params).then(json => { - let objectName - const responseName = [apiName.toLowerCase(), 'response'].join('') - for (const key in json[responseName]) { - if (key === 'count') { - continue - } - objectName = key - break - } - const data = json?.[responseName]?.[objectName] || [] - zones = data.map(item => item.zoneid) - return resolve(zones) - }).catch(() => { - return resolve(zones) - }) - }) - }, async fetchData () { this.fetchZones(null, null) _.each(this.params, (param, name) => { @@ -1718,6 +1678,7 @@ export default { if (template.details['vmware-to-kvm-mac-addresses']) { this.dataPreFill.macAddressArray = JSON.parse(template.details['vmware-to-kvm-mac-addresses']) } + this.selectedArchitecture = template?.arch || 'x86_64' } } else if (name === 'isoid') { this.templateConfigurations = [] @@ -2344,9 +2305,6 @@ export default { this.clusterId = null this.zone = _.find(this.options.zones, (option) => option.id === value) this.isZoneSelectedMultiArch = this.zone.ismultiarch - if (this.isZoneSelectedMultiArch) { - this.selectedArchitecture = this.architectureTypes.opts[0].id - } this.zoneSelected = true this.form.startvm = true this.selectedZone = this.zoneId diff --git a/ui/src/views/storage/CreateVMFromBackup.vue b/ui/src/views/storage/CreateVMFromBackup.vue index 8d29397e45a..891e8fe9642 100644 --- a/ui/src/views/storage/CreateVMFromBackup.vue +++ b/ui/src/views/storage/CreateVMFromBackup.vue @@ -92,10 +92,11 @@ export default { } }, async created () { - await Promise.all[( + await Promise.all([ this.fetchServiceOffering(), - this.fetchBackupOffering() - )] + this.fetchBackupOffering(), + this.fetchBackupArch() + ]) this.loading = false }, methods: { @@ -118,6 +119,23 @@ export default { this.backupOffering = backupOfferings[0] }) }, + fetchBackupArch () { + const isIso = this.resource.vmdetails.isiso === 'true' + const api = isIso ? 'listIsos' : 'listTemplates' + const responseKey = isIso ? 'listisosresponse' : 'listtemplatesresponse' + const itemKey = isIso ? 'iso' : 'template' + + return getAPI(api, { + id: this.resource.vmdetails.templateid, + listall: true, + ...(isIso ? {} : { templatefilter: 'all' }) + }).then(response => { + const items = response?.[responseKey]?.[itemKey] || [] + this.backupArch = items[0]?.arch || 'x86_64' + }).catch(() => { + this.backupArch = 'x86_64' + }) + }, populatePreFillData () { this.vmdetails = this.resource.vmdetails this.dataPreFill.zoneid = this.resource.zoneid @@ -128,6 +146,7 @@ export default { this.dataPreFill.backupid = this.resource.id this.dataPreFill.computeofferingid = this.vmdetails.serviceofferingid this.dataPreFill.templateid = this.vmdetails.templateid + this.dataPreFill.backupArch = this.backupArch this.dataPreFill.allowtemplateisoselection = true this.dataPreFill.isoid = this.vmdetails.templateid this.dataPreFill.allowIpAddressesFetch = this.resource.isbackupvmexpunged From b5858029bb516329f5688662b2232c9595f67748 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 9 Apr 2026 05:55:47 -0300 Subject: [PATCH 16/26] Fix listing service offerings with different host tags (#12919) --- .../java/com/cloud/host/dao/HostTagsDao.java | 5 +++ .../com/cloud/host/dao/HostTagsDaoImpl.java | 20 ++++++++++ .../com/cloud/api/query/QueryManagerImpl.java | 30 +++++++++++++- .../main/java/com/cloud/vm/UserVmManager.java | 3 ++ .../java/com/cloud/vm/UserVmManagerImpl.java | 2 +- .../cloud/api/query/QueryManagerImplTest.java | 40 +++++++++++++++++++ .../wizard/ComputeOfferingSelection.vue | 23 ++++++++++- 7 files changed, 120 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 7a00829fd44..0d86ca0e48c 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -45,4 +45,9 @@ public interface HostTagsDao extends GenericDao { HostTagResponse newHostTagResponse(HostTagVO hostTag); List searchByIds(Long... hostTagIds); + + /** + * List all host tags defined on hosts within a cluster + */ + List listByClusterId(Long clusterId); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index 4aa14a31cfc..d3fee6a2676 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -43,9 +44,12 @@ public class HostTagsDaoImpl extends GenericDaoBase implements private final SearchBuilder stSearch; private final SearchBuilder tagIdsearch; private final SearchBuilder ImplicitTagsSearch; + private final GenericSearchBuilder tagSearch; @Inject private ConfigurationDao _configDao; + @Inject + private HostDao hostDao; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); @@ -72,6 +76,11 @@ public class HostTagsDaoImpl extends GenericDaoBase implements ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ); ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ); ImplicitTagsSearch.done(); + + tagSearch = createSearchBuilder(String.class); + tagSearch.selectFields(tagSearch.entity().getTag()); + tagSearch.and("hostIdIN", tagSearch.entity().getHostId(), SearchCriteria.Op.IN); + tagSearch.done(); } @Override @@ -235,4 +244,15 @@ public class HostTagsDaoImpl extends GenericDaoBase implements return tagList; } + + @Override + public List listByClusterId(Long clusterId) { + List hostIds = hostDao.listIdsByClusterId(clusterId); + if (CollectionUtils.isEmpty(hostIds)) { + return new ArrayList<>(); + } + SearchCriteria sc = tagSearch.create(); + sc.setParameters("hostIdIN", hostIds.toArray()); + return customSearch(sc, null); + } } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index ac9f8ee1433..fea87b66fed 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -45,6 +45,7 @@ import com.cloud.server.ManagementService; import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao; import com.cloud.cluster.ManagementServerHostPeerJoinVO; +import com.cloud.vm.UserVmManager; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker; @@ -4330,6 +4331,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q List hostTags = new ArrayList<>(); if (currentVmOffering != null) { hostTags.addAll(com.cloud.utils.StringUtils.csvTagsToList(currentVmOffering.getHostTag())); + if (UserVmManager.AllowDifferentHostTagsOfferingsForVmScale.value()) { + addVmCurrentClusterHostTags(vmInstance, hostTags); + } } if (!hostTags.isEmpty()) { @@ -4341,7 +4345,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q flag = false; serviceOfferingSearch.op("hostTag" + tag, serviceOfferingSearch.entity().getHostTag(), Op.FIND_IN_SET); } else { - serviceOfferingSearch.and("hostTag" + tag, serviceOfferingSearch.entity().getHostTag(), Op.FIND_IN_SET); + serviceOfferingSearch.or("hostTag" + tag, serviceOfferingSearch.entity().getHostTag(), Op.FIND_IN_SET); } } serviceOfferingSearch.cp().cp(); @@ -4486,6 +4490,30 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return new Pair<>(offeringIds, count); } + protected void addVmCurrentClusterHostTags(VMInstanceVO vmInstance, List hostTags) { + if (vmInstance == null) { + return; + } + Long hostId = vmInstance.getHostId() == null ? vmInstance.getLastHostId() : vmInstance.getHostId(); + if (hostId == null) { + return; + } + HostVO host = hostDao.findById(hostId); + if (host == null) { + logger.warn("Unable to find host with id " + hostId); + return; + } + List clusterTags = _hostTagDao.listByClusterId(host.getClusterId()); + if (CollectionUtils.isEmpty(clusterTags)) { + logger.debug("No host tags defined for hosts in the cluster " + host.getClusterId()); + return; + } + Set existingTagsSet = new HashSet<>(hostTags); + clusterTags.stream() + .filter(tag -> !existingTagsSet.contains(tag)) + .forEach(hostTags::add); + } + @Override public ListResponse listDataCenters(ListZonesCmd cmd) { Pair, Integer> result = listDataCentersInternal(cmd); diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index 0a744709644..38cb6d2db46 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -108,6 +108,9 @@ public interface UserVmManager extends UserVmService { "Comma separated list of allowed additional VM settings if VM instance settings are read from OVA.", true, ConfigKey.Scope.Zone, null, null, null, null, null, ConfigKey.Kind.CSV, null); + ConfigKey AllowDifferentHostTagsOfferingsForVmScale = new ConfigKey<>("Advanced", Boolean.class, "allow.different.host.tags.offerings.for.vm.scale", "false", + "Enables/Disable allowing to change a VM offering to offerings with different host tags", true); + static final int MAX_USER_DATA_LENGTH_BYTES = 2048; public static final String CKS_NODE = "cksnode"; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 65bd285ca90..9cec033d07c 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -9412,7 +9412,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir VmIpFetchThreadPoolMax, VmIpFetchTaskWorkers, AllowDeployVmIfGivenHostFails, EnableAdditionalVmConfig, DisplayVMOVFProperties, KvmAdditionalConfigAllowList, XenServerAdditionalConfigAllowList, VmwareAdditionalConfigAllowList, DestroyRootVolumeOnVmDestruction, EnforceStrictResourceLimitHostTagCheck, StrictHostTags, AllowUserForceStopVm, VmDistinctHostNameScope, - VmwareAdditionalDetailsFromOvaEnabled, VmwareAllowedAdditionalDetailsFromOva}; + VmwareAdditionalDetailsFromOvaEnabled, VmwareAllowedAdditionalDetailsFromOva, AllowDifferentHostTagsOfferingsForVmScale}; } @Override diff --git a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java index 892cd1e7def..750f4d8655b 100644 --- a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java +++ b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java @@ -18,6 +18,7 @@ package com.cloud.api.query; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -33,6 +34,7 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import com.cloud.host.dao.HostTagsDao; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ResponseObject; @@ -156,6 +158,9 @@ public class QueryManagerImplTest { @Mock HostDao hostDao; + @Mock + HostTagsDao hostTagsDao; + @Mock ClusterDao clusterDao; @@ -622,4 +627,39 @@ public class QueryManagerImplTest { verify(host1).setExtensionId("a"); verify(host2).setExtensionId("b"); } + + @Test + public void testAddVmCurrentClusterHostTags() { + String tag1 = "tag1"; + String tag2 = "tag2"; + VMInstanceVO vmInstance = mock(VMInstanceVO.class); + HostVO host = mock(HostVO.class); + when(vmInstance.getHostId()).thenReturn(null); + when(vmInstance.getLastHostId()).thenReturn(1L); + when(hostDao.findById(1L)).thenReturn(host); + when(host.getClusterId()).thenReturn(1L); + when(hostTagsDao.listByClusterId(1L)).thenReturn(Arrays.asList(tag1, tag2)); + + List hostTags = new ArrayList<>(Collections.singleton(tag1)); + queryManagerImplSpy.addVmCurrentClusterHostTags(vmInstance, hostTags); + assertEquals(2, hostTags.size()); + assertTrue(hostTags.contains(tag2)); + } + + @Test + public void testAddVmCurrentClusterHostTagsEmptyHostTagsInCluster() { + String tag1 = "tag1"; + VMInstanceVO vmInstance = mock(VMInstanceVO.class); + HostVO host = mock(HostVO.class); + when(vmInstance.getHostId()).thenReturn(null); + when(vmInstance.getLastHostId()).thenReturn(1L); + when(hostDao.findById(1L)).thenReturn(host); + when(host.getClusterId()).thenReturn(1L); + when(hostTagsDao.listByClusterId(1L)).thenReturn(null); + + List hostTags = new ArrayList<>(Collections.singleton(tag1)); + queryManagerImplSpy.addVmCurrentClusterHostTags(vmInstance, hostTags); + assertEquals(1, hostTags.size()); + assertTrue(hostTags.contains(tag1)); + } } diff --git a/ui/src/views/compute/wizard/ComputeOfferingSelection.vue b/ui/src/views/compute/wizard/ComputeOfferingSelection.vue index eb6e228a93f..0a3e5fd2e82 100644 --- a/ui/src/views/compute/wizard/ComputeOfferingSelection.vue +++ b/ui/src/views/compute/wizard/ComputeOfferingSelection.vue @@ -41,6 +41,8 @@