diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a0602871ef1..689275cff11 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ /plugins/storage/volume/linstor @rp- /plugins/storage/volume/storpool @slavkap +/plugins/storage/volume/ontap @rajiv1 @sandeeplocharla @piyush5 @suryag .pre-commit-config.yaml @jbampton /.github/linters/ @jbampton diff --git a/api/src/main/java/com/cloud/server/ManagementService.java b/api/src/main/java/com/cloud/server/ManagementService.java index 51737546ffa..bcca229c06b 100644 --- a/api/src/main/java/com/cloud/server/ManagementService.java +++ b/api/src/main/java/com/cloud/server/ManagementService.java @@ -71,7 +71,6 @@ import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.config.ConfigurationGroup; -import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.alert.Alert; import com.cloud.capacity.Capacity; @@ -108,14 +107,6 @@ import com.cloud.vm.VirtualMachineProfile; public interface ManagementService { static final String Name = "management-server"; - ConfigKey JsInterpretationEnabled = new ConfigKey<>("Hidden" - , Boolean.class - , "js.interpretation.enabled" - , "false" - , "Enable/Disable all JavaScript interpretation related functionalities to create or update Javascript rules." - , false - , ConfigKey.Scope.Global); - /** * returns the a map of the names/values in the configuration table * @@ -534,6 +525,4 @@ public interface ManagementService { boolean removeManagementServer(RemoveManagementServerCmd cmd); - void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue); - } diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 30919e5b782..4145e2b89eb 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -138,6 +138,8 @@ public interface AccountService { Long finalizeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly); + Long finalizeAccountId(Long accountId, String accountName, Long domainId, Long projectId); + /** * returns the user account object for a given user id * @param userId user id diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3d827641358..05c6098bc72 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -20,6 +20,7 @@ public class ApiConstants { public static final String ACCOUNT = "account"; public static final String ACCOUNTS = "accounts"; public static final String ACCOUNT_NAME = "accountname"; + public static final String ACCOUNT_STATE_TO_SHOW = "accountstatetoshow"; public static final String ACCOUNT_TYPE = "accounttype"; public static final String ACCOUNT_ID = "accountid"; public static final String ACCOUNT_IDS = "accountids"; diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java index 78c598272e0..13f01b4944a 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java @@ -37,7 +37,7 @@ public interface VolumeImportUnmanageService extends PluggableService, Configura Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware); List SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem, - Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD); + Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint); ConfigKey AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class, "allow.import.volume.with.backing.file", diff --git a/client/pom.xml b/client/pom.xml index b8dffe65d4f..7118f455ab5 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -121,6 +121,11 @@ cloud-plugin-storage-volume-adaptive ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-volume-ontap + ${project.version} + org.apache.cloudstack cloud-plugin-storage-volume-solidfire diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 6fe001de72c..71c329796d1 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -52,6 +52,7 @@ import com.cloud.storage.StorageLayer; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.net.Proxy; /** @@ -125,6 +126,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java index 95ed0d1e76d..0dad2564779 100644 --- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java @@ -18,8 +18,11 @@ // package com.cloud.storage.template; + import com.cloud.storage.StorageLayer; import com.cloud.utils.UriUtils; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; + import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethodRetryHandler; @@ -59,6 +62,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (!toFileSet) { String[] parts = downloadUrl.split("/"); String filename = parts[parts.length - 1]; diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index 8719947cb4f..6608754073a 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -44,6 +44,7 @@ import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.lang3.StringUtils; import com.cloud.storage.StorageLayer; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader { private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); @@ -95,6 +96,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } @@ -141,6 +143,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem continue; } HeadMethod headMethod = new HeadMethod(downloadUrl); + headMethod.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { if (client.executeMethod(headMethod) != HttpStatus.SC_OK) { continue; diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index f5ad5fbea2c..972c2eaf7bb 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -34,6 +34,7 @@ public class RestoreBackupCommand extends Command { private List backupVolumesUUIDs; private List restoreVolumePools; private List restoreVolumePaths; + private List restoreVolumeSizes; private List backupFiles; private String diskType; private Boolean vmExists; @@ -92,6 +93,14 @@ public class RestoreBackupCommand extends Command { this.restoreVolumePaths = restoreVolumePaths; } + public List getRestoreVolumeSizes() { + return restoreVolumeSizes; + } + + public void setRestoreVolumeSizes(List restoreVolumeSizes) { + this.restoreVolumeSizes = restoreVolumeSizes; + } + public List getBackupFiles() { return backupFiles; } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index c4a802ecdbc..99b84bb645c 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.direct.download; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -32,6 +33,7 @@ import java.util.Map; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; @@ -39,6 +41,7 @@ import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.io.IOUtils; public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -68,6 +71,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(this.isFollowRedirects()); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); @@ -111,6 +115,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { public boolean checkUrl(String url) { HeadMethod httpHead = new HeadMethod(url); httpHead.setFollowRedirects(this.isFollowRedirects()); + httpHead.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { int responseCode = client.executeMethod(httpHead); if (responseCode != HttpStatus.SC_OK) { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index befddf78954..7dbd2fedf80 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.vm; +import static com.cloud.configuration.ConfigurationManagerImpl.EXPOSE_ERRORS_TO_USER; import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS; import java.lang.reflect.Field; @@ -941,10 +942,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public void start(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { try { advanceStart(vmUuid, params, planToDeploy, planner); - } catch (ConcurrentOperationException | InsufficientCapacityException e) { - throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } catch (ConcurrentOperationException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to concurrent operation.", vmUuid), e).add(VirtualMachine.class, vmUuid); + } catch (final InsufficientCapacityException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to insufficient capacity.", vmUuid), e).add(VirtualMachine.class, vmUuid); } catch (final ResourceUnavailableException e) { - if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)){ + if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)) { Account callingAccount = CallContext.current().getCallingAccount(); String errorSuffix = (callingAccount != null && callingAccount.getType() == Account.Type.ADMIN) ? String.format("Failure: %s", e.getMessage()) : @@ -1375,6 +1388,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); + Throwable lastKnownError = null; boolean canRetry = true; ExcludeList avoids = null; try { @@ -1398,7 +1412,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac int retry = StartRetry.value(); while (retry-- != 0) { - logger.debug("Instance start attempt #{}", (StartRetry.value() - retry)); + int attemptNumber = StartRetry.value() - retry; + logger.debug("Instance start attempt #{}", attemptNumber); if (reuseVolume) { final List vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); @@ -1464,8 +1479,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac reuseVolume = false; continue; } - throw new InsufficientServerCapacityException("Unable to create a deployment for " + vmProfile, DataCenter.class, plan.getDataCenterId(), - areAffinityGroupsAssociated(vmProfile)); + String message = String.format("Unable to create a deployment for %s after %s attempts", vmProfile, attemptNumber); + if (canExposeError(account) && lastKnownError != null) { + message += String.format(" Last known error: %s", lastKnownError.getMessage()); + throw new CloudRuntimeException(message, lastKnownError); + } else { + throw new InsufficientServerCapacityException(message, DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); + } } avoids.addHost(dest.getHost().getId()); @@ -1633,11 +1653,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new ExecutionException("Unable to start VM:" + vm.getUuid() + " due to error in finalizeStart, not retrying"); } } - logger.info("Unable to start VM on {} due to {}", dest.getHost(), (startAnswer == null ? " no start answer" : startAnswer.getDetails())); + String msg = String.format("Unable to start VM on %s due to %s", dest.getHost(), startAnswer == null ? "no start command answer" : startAnswer.getDetails()); + lastKnownError = new ExecutionException(msg); + if (startAnswer != null && startAnswer.getContextParam("stopRetry") != null) { + logger.error(msg, lastKnownError); break; } + logger.debug(msg, lastKnownError); } catch (OperationTimedoutException e) { logger.debug("Unable to send the start command to host {} failed to start VM: {}", dest.getHost(), vm); if (e.isActive()) { @@ -1647,6 +1671,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e); } catch (final ResourceUnavailableException e) { logger.warn("Unable to contact resource.", e); + lastKnownError = e; if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; @@ -1703,10 +1728,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } if (startedVm == null) { - throw new CloudRuntimeException("Unable to start Instance '" + vm.getHostName() + "' (" + vm.getUuid() + "), see management server log for details"); + String messageTmpl = "Unable to start Instance '%s' (%s)%s"; + String details; + if (canExposeError(account) && lastKnownError != null) { + details = ": " + lastKnownError.getMessage(); + } else { + details = ", see management server log for details"; + } + String message = String.format(messageTmpl, vm.getHostName(), vm.getUuid(), details); + throw new CloudRuntimeException(message, lastKnownError); } } + private boolean canExposeError(Account account) { + return (account != null && account.getType() == Account.Type.ADMIN) || Boolean.TRUE.equals(EXPOSE_ERRORS_TO_USER.value()); + } + protected void updateStartCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, StartCommand command) { if (!HypervisorType.External.equals(host.getHypervisorType())) { return; diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java index 56d971bbe01..1afa0d22dcc 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java @@ -262,7 +262,7 @@ public class DomainDaoImpl extends GenericDaoBase implements Dom SearchCriteria sc = DomainPairSearch.create(); sc.setParameters("id", parentId, childId); - List domainPair = listBy(sc); + List domainPair = listIncludingRemovedBy(sc); if ((domainPair != null) && (domainPair.size() == 2)) { DomainVO d1 = domainPair.get(0); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 7a00829fd44..0d86ca0e48c 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -45,4 +45,9 @@ public interface HostTagsDao extends GenericDao { HostTagResponse newHostTagResponse(HostTagVO hostTag); List searchByIds(Long... hostTagIds); + + /** + * List all host tags defined on hosts within a cluster + */ + List listByClusterId(Long clusterId); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index 4aa14a31cfc..d3fee6a2676 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -43,9 +44,12 @@ public class HostTagsDaoImpl extends GenericDaoBase implements private final SearchBuilder stSearch; private final SearchBuilder tagIdsearch; private final SearchBuilder ImplicitTagsSearch; + private final GenericSearchBuilder tagSearch; @Inject private ConfigurationDao _configDao; + @Inject + private HostDao hostDao; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); @@ -72,6 +76,11 @@ public class HostTagsDaoImpl extends GenericDaoBase implements ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ); ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ); ImplicitTagsSearch.done(); + + tagSearch = createSearchBuilder(String.class); + tagSearch.selectFields(tagSearch.entity().getTag()); + tagSearch.and("hostIdIN", tagSearch.entity().getHostId(), SearchCriteria.Op.IN); + tagSearch.done(); } @Override @@ -235,4 +244,15 @@ public class HostTagsDaoImpl extends GenericDaoBase implements return tagList; } + + @Override + public List listByClusterId(Long clusterId) { + List hostIds = hostDao.listIdsByClusterId(clusterId); + if (CollectionUtils.isEmpty(hostIds)) { + return new ArrayList<>(); + } + SearchCriteria sc = tagSearch.create(); + sc.setParameters("hostIdIN", hostIds.toArray()); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 4e8b6204f72..9f7ffabac93 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -193,6 +193,7 @@ public class NetworkDaoImpl extends GenericDaoBaseimplements Ne PersistentNetworkSearch.and("id", PersistentNetworkSearch.entity().getId(), Op.NEQ); PersistentNetworkSearch.and("guestType", PersistentNetworkSearch.entity().getGuestType(), Op.IN); PersistentNetworkSearch.and("broadcastUri", PersistentNetworkSearch.entity().getBroadcastUri(), Op.EQ); + PersistentNetworkSearch.and("dc", PersistentNetworkSearch.entity().getDataCenterId(), Op.EQ); PersistentNetworkSearch.and("removed", PersistentNetworkSearch.entity().getRemoved(), Op.NULL); final SearchBuilder persistentNtwkOffJoin = _ntwkOffDao.createSearchBuilder(); persistentNtwkOffJoin.and("persistent", persistentNtwkOffJoin.entity().isPersistent(), Op.EQ); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java index 80e1b7d4d4b..f167b573187 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java @@ -170,6 +170,7 @@ public class SnapshotDaoImpl extends GenericDaoBase implements CountSnapshotsByAccount.select(null, Func.COUNT, null); CountSnapshotsByAccount.and("account", CountSnapshotsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); CountSnapshotsByAccount.and("status", CountSnapshotsByAccount.entity().getState(), SearchCriteria.Op.NIN); + CountSnapshotsByAccount.and("snapshotTypeNEQ", CountSnapshotsByAccount.entity().getSnapshotType(), SearchCriteria.Op.NIN); CountSnapshotsByAccount.and("removed", CountSnapshotsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); CountSnapshotsByAccount.done(); @@ -220,6 +221,7 @@ public class SnapshotDaoImpl extends GenericDaoBase implements SearchCriteria sc = CountSnapshotsByAccount.create(); sc.setParameters("account", accountId); sc.setParameters("status", State.Error, State.Destroyed); + sc.setParameters("snapshotTypeNEQ", Snapshot.Type.GROUP.ordinal()); return customSearch(sc, null).get(0); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java index df4743894c9..393f2039950 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java @@ -17,7 +17,12 @@ package com.cloud.upgrade.dao; import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; public class Upgrade42210to42300 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { @@ -42,4 +47,46 @@ public class Upgrade42210to42300 extends DbUpgradeAbstractImpl implements DbUpgr return new InputStream[] {script}; } + + @Override + public void performDataMigration(Connection conn) { + unhideJsInterpretationEnabled(conn); + } + + protected void unhideJsInterpretationEnabled(Connection conn) { + String value = getJsInterpretationEnabled(conn); + if (value != null) { + updateJsInterpretationEnabledFields(conn, value); + } + } + + protected String getJsInterpretationEnabled(Connection conn) { + String query = "SELECT value FROM cloud.configuration WHERE name = 'js.interpretation.enabled' AND category = 'Hidden';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getString("value"); + } + logger.debug("Unable to retrieve value of hidden configuration 'js.interpretation.enabled'. The configuration may already be unhidden."); + return null; + } catch (SQLException e) { + throw new CloudRuntimeException("Error while retrieving value of hidden configuration 'js.interpretation.enabled'.", e); + } + } + + protected void updateJsInterpretationEnabledFields(Connection conn, String encryptedValue) { + String query = "UPDATE cloud.configuration SET value = ?, category = 'System', component = 'JsInterpreter', is_dynamic = 1 WHERE name = 'js.interpretation.enabled';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + String decryptedValue = DBEncryptionUtil.decrypt(encryptedValue); + logger.info("Updating setting 'js.interpretation.enabled' to decrypted value [{}], category 'System', component 'JsInterpreter', and is_dynamic '1'.", decryptedValue); + pstmt.setString(1, decryptedValue); + pstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Error while unhiding configuration 'js.interpretation.enabled'.", e); + } catch (CloudRuntimeException e) { + logger.warn("Error while decrypting configuration 'js.interpretation.enabled'. The configuration may already be decrypted."); + } + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 6aeee1ad1cc..3329983d711 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -51,6 +51,8 @@ StateDao listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, ObjectInDataStoreStateMachine.State... state); + List listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId); SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index f5daf673d3c..8b7a2b78de7 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -68,6 +68,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; private SearchBuilder idStateNinSearch; + private SearchBuilder idEqRoleEqStateInSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; private SearchBuilder dataStoreAndInstallPathSearch; @@ -151,6 +152,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, State... state) { + SearchCriteria sc = idEqRoleEqStateInSearch.create(); + sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STORE_ROLE, role); + sc.setParameters(STATE, (Object[])state); + return listBy(sc); + } + @Override public SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId) { try (TransactionLegacy transactionLegacy = TransactionLegacy.currentTxn()) { diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4c65f37e0fe..000b54b7207 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -687,22 +687,23 @@ CREATE TABLE IF NOT EXISTS `cloud`.`backup_details` ( UPDATE `cloud`.`backups` b INNER JOIN `cloud`.`vm_instance` vm ON b.vm_id = vm.id SET b.backed_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id @@ -711,22 +712,23 @@ SET b.backed_volumes = ( -- Add diskOfferingId, deviceId, minIops and maxIops to backup_volumes in vm_instance table UPDATE `cloud`.`vm_instance` vm SET vm.backup_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 505c8ef5715..54baf226ac4 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,5 +18,3 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; - -DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index 2326a855f32..369159e4836 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -35,3 +35,6 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template'; + +-- Update existing vm_template records with NULL type to "USER" +UPDATE `cloud`.`vm_template` SET `type` = 'USER' WHERE `type` IS NULL; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql new file mode 100644 index 00000000000..11193c465fd --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- cloud.account_netstats_view source + + +DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; + +CREATE VIEW `cloud`.`account_netstats_view` AS +select + `user_statistics`.`account_id` AS `account_id`, + (sum(`user_statistics`.`net_bytes_received`) + sum(`user_statistics`.`current_bytes_received`)) AS `bytesReceived`, + (sum(`user_statistics`.`net_bytes_sent`) + sum(`user_statistics`.`current_bytes_sent`)) AS `bytesSent` +from + `user_statistics` +group by + `user_statistics`.`account_id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql index 327c6c627e2..edc164c40cb 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -39,8 +39,8 @@ select `data_center`.`id` AS `data_center_id`, `data_center`.`uuid` AS `data_center_uuid`, `data_center`.`name` AS `data_center_name`, - `account_netstats`.`bytesReceived` AS `bytesReceived`, - `account_netstats`.`bytesSent` AS `bytesSent`, + `account_netstats_view`.`bytesReceived` AS `bytesReceived`, + `account_netstats_view`.`bytesSent` AS `bytesSent`, `vmlimit`.`max` AS `vmLimit`, `vmcount`.`count` AS `vmTotal`, `runningvm`.`vmcount` AS `runningVms`, @@ -89,15 +89,8 @@ from `cloud`.`domain` ON account.domain_id = domain.id left join `cloud`.`data_center` ON account.default_zone_id = data_center.id - left join lateral ( - select - coalesce(sum(`user_statistics`.`net_bytes_received` + `user_statistics`.`current_bytes_received`), 0) AS `bytesReceived`, - coalesce(sum(`user_statistics`.`net_bytes_sent` + `user_statistics`.`current_bytes_sent`), 0) AS `bytesSent` - from - `cloud`.`user_statistics` - where - `user_statistics`.`account_id` = `account`.`id` - ) AS `account_netstats` ON TRUE + left join + `cloud`.`account_netstats_view` ON account.id = account_netstats_view.account_id left join `cloud`.`resource_limit` vmlimit ON account.id = vmlimit.account_id and vmlimit.type = 'user_vm' and vmlimit.tag IS NULL diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql new file mode 100644 index 00000000000..0646c3b6f91 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql @@ -0,0 +1,48 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- cloud_usage.quota_summary_view source + +-- Create view for quota summary +DROP VIEW IF EXISTS `cloud_usage`.`quota_summary_view`; +CREATE VIEW `cloud_usage`.`quota_summary_view` AS +SELECT + cloud_usage.quota_account.account_id AS account_id, + cloud_usage.quota_account.quota_balance AS quota_balance, + cloud_usage.quota_account.quota_balance_date AS quota_balance_date, + cloud_usage.quota_account.quota_enforce AS quota_enforce, + cloud_usage.quota_account.quota_min_balance AS quota_min_balance, + cloud_usage.quota_account.quota_alert_date AS quota_alert_date, + cloud_usage.quota_account.quota_alert_type AS quota_alert_type, + cloud_usage.quota_account.last_statement_date AS last_statement_date, + cloud.account.uuid AS account_uuid, + cloud.account.account_name AS account_name, + cloud.account.state AS account_state, + cloud.account.removed AS account_removed, + cloud.domain.id AS domain_id, + cloud.domain.uuid AS domain_uuid, + cloud.domain.name AS domain_name, + cloud.domain.path AS domain_path, + cloud.domain.removed AS domain_removed, + cloud.projects.uuid AS project_uuid, + cloud.projects.name AS project_name, + cloud.projects.removed AS project_removed +FROM + cloud_usage.quota_account + INNER JOIN cloud.account ON (cloud.account.id = cloud_usage.quota_account.account_id) + INNER JOIN cloud.domain ON (cloud.domain.id = cloud.account.domain_id) + LEFT JOIN cloud.projects ON (cloud.account.type = 5 AND cloud.account.id = cloud.projects.project_account_id); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java index e02d17fe19b..030fc31e6b9 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java @@ -121,7 +121,7 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase { private final List snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error, Snapshot.State.Hidden); public SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { - List snaps = snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + List snaps = snapshotStoreDao.listBySnapshotIdAndDataStoreRoleAndStateIn(snapshotId, DataStoreRole.Image, State.Ready, State.Hidden); for (SnapshotDataStoreVO ref : snaps) { if (zoneId == dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())) { return ref; diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java index 7199fce1d34..aced750bd32 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.datastore.api.Volume; import com.cloud.agent.api.VMSnapshotTO; import com.cloud.alert.AlertManager; @@ -200,11 +201,35 @@ public class ScaleIOVMSnapshotStrategy extends ManagerBase implements VMSnapshot if (volumeIds != null && !volumeIds.isEmpty()) { List vmSnapshotDetails = new ArrayList(); vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "SnapshotGroupId", snapshotGroupId, false)); + Map snapshotNameToSrcPathMap = new HashMap<>(); + for (Map.Entry entry : srcVolumeDestSnapshotMap.entrySet()) { + snapshotNameToSrcPathMap.put(entry.getValue(), entry.getKey()); + } - for (int index = 0; index < volumeIds.size(); index++) { - String volumeSnapshotName = srcVolumeDestSnapshotMap.get(ScaleIOUtil.getVolumePath(volumeTOs.get(index).getPath())); - String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(volumeIds.get(index), volumeSnapshotName); - vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "Vol_" + volumeTOs.get(index).getId() + "_Snapshot", pathWithScaleIOVolumeName, false)); + for (String snapshotVolumeId : volumeIds) { + // Use getVolume() to fetch snapshot volume details and get its name + Volume snapshotVolume = client.getVolume(snapshotVolumeId); + if (snapshotVolume == null) { + throw new CloudRuntimeException("Cannot find snapshot volume with id: " + snapshotVolumeId); + } + String snapshotName = snapshotVolume.getName(); + + // Match back to source volume path + String srcVolumePath = snapshotNameToSrcPathMap.get(snapshotName); + if (srcVolumePath == null) { + throw new CloudRuntimeException("Cannot match snapshot " + snapshotName + " to a source volume"); + } + + // Find the matching VolumeObjectTO by path + VolumeObjectTO matchedVolume = volumeTOs.stream() + .filter(v -> ScaleIOUtil.getVolumePath(v.getPath()).equals(srcVolumePath)) + .findFirst() + .orElseThrow(() -> new CloudRuntimeException("Cannot find source volume for path: " + srcVolumePath)); + + String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, snapshotName); + vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), + "Vol_" + matchedVolume.getId() + "_Snapshot", + pathWithScaleIOVolumeName, false)); } vmSnapshotDetailsDao.saveDetails(vmSnapshotDetails); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java index d08e62dd92b..c27d4e13fa4 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java @@ -258,11 +258,6 @@ public class DefaultSnapshotStrategyTest { @Test public void testGetSnapshotImageStoreRefNull() { - SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); - Mockito.when(ref1.getDataStoreId()).thenReturn(1L); - Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); - Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); - Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(2L); Assert.assertNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); } @@ -271,7 +266,7 @@ public class DefaultSnapshotStrategyTest { SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); Mockito.when(ref1.getDataStoreId()).thenReturn(1L); Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); - Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); + Mockito.when(snapshotDataStoreDao.listBySnapshotIdAndDataStoreRoleAndStateIn(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.any(), Mockito.any())).thenReturn(List.of(ref1)); Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(1L); Assert.assertNotNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java new file mode 100644 index 00000000000..cc6ca203ef7 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota; + +import org.apache.commons.lang3.StringUtils; + +public enum QuotaAccountStateFilter { + ALL, ACTIVE, REMOVED; + + public static QuotaAccountStateFilter getValue(String value) { + if (StringUtils.isBlank(value)) { + return null; + } + for (QuotaAccountStateFilter state : values()) { + if (state.name().equalsIgnoreCase(value)) { + return state; + } + } + return null; + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java new file mode 100644 index 00000000000..d8ee2607501 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.dao; + +import java.util.List; + +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; + +import com.cloud.utils.Pair; +import com.cloud.utils.db.GenericDao; + +public interface QuotaSummaryDao extends GenericDao { + + Pair, Integer> listQuotaSummariesForAccountAndOrDomain(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter, Long startIndex, Long pageSize); +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java new file mode 100644 index 00000000000..d90d5a75859 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.dao; + +import java.util.List; + +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; + +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +public class QuotaSummaryDaoImpl extends GenericDaoBase implements QuotaSummaryDao { + + @Override + public Pair, Integer> listQuotaSummariesForAccountAndOrDomain(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter, Long startIndex, Long pageSize) { + SearchCriteria searchCriteria = createListQuotaSummariesSearchCriteria(accountId, accountName, domainId, domainPath, accountStateFilter); + Filter filter = new Filter(QuotaSummaryVO.class, "accountName", true, startIndex, pageSize); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback, Integer>>) status -> searchAndCount(searchCriteria, filter)); + } + + protected SearchCriteria createListQuotaSummariesSearchCriteria(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter) { + SearchCriteria searchCriteria = createListQuotaSummariesSearchBuilder(accountStateFilter).create(); + + searchCriteria.setParametersIfNotNull("accountId", accountId); + searchCriteria.setParametersIfNotNull("domainId", domainId); + + if (accountName != null) { + searchCriteria.setParameters("accountName", "%" + accountName + "%"); + } + + if (domainPath != null) { + searchCriteria.setParameters("domainPath", domainPath + "%"); + } + + return searchCriteria; + } + + protected SearchBuilder createListQuotaSummariesSearchBuilder(QuotaAccountStateFilter accountStateFilter) { + SearchBuilder searchBuilder = createSearchBuilder(); + + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("accountName", searchBuilder.entity().getAccountName(), SearchCriteria.Op.LIKE); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainPath", searchBuilder.entity().getDomainPath(), SearchCriteria.Op.LIKE); + + if (QuotaAccountStateFilter.REMOVED.equals(accountStateFilter)) { + searchBuilder.and("accountRemoved", searchBuilder.entity().getAccountRemoved(), SearchCriteria.Op.NNULL); + } else if (QuotaAccountStateFilter.ACTIVE.equals(accountStateFilter)) { + searchBuilder.and("accountRemoved", searchBuilder.entity().getAccountRemoved(), SearchCriteria.Op.NULL); + } + + return searchBuilder; + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java new file mode 100644 index 00000000000..f9796497d57 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import com.cloud.user.Account; + +@Entity +@Table(name = "quota_summary_view") +public class QuotaSummaryVO { + + @Id + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "quota_enforce") + private Integer quotaEnforce = 0; + + @Column(name = "quota_balance") + private BigDecimal quotaBalance; + + @Column(name = "quota_balance_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date quotaBalanceDate = null; + + @Column(name = "quota_min_balance") + private BigDecimal quotaMinBalance; + + @Column(name = "quota_alert_type") + private Integer quotaAlertType = null; + + @Column(name = "quota_alert_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date quotaAlertDate = null; + + @Column(name = "last_statement_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date lastStatementDate = null; + + @Column(name = "account_uuid") + private String accountUuid; + + @Column(name = "account_name") + private String accountName; + + @Column(name = "account_state") + @Enumerated(EnumType.STRING) + private Account.State accountState; + + @Column(name = "account_removed") + private Date accountRemoved; + + @Column(name = "domain_id") + private Long domainId; + + @Column(name = "domain_uuid") + private String domainUuid; + + @Column(name = "domain_name") + private String domainName; + + @Column(name = "domain_path") + private String domainPath; + + @Column(name = "domain_removed") + private Date domainRemoved; + + @Column(name = "project_uuid") + private String projectUuid; + + @Column(name = "project_name") + private String projectName; + + @Column(name = "project_removed") + private Date projectRemoved; + + public Long getAccountId() { + return accountId; + } + + public BigDecimal getQuotaBalance() { + return quotaBalance; + } + + public String getAccountUuid() { + return accountUuid; + } + + public String getAccountName() { + return accountName; + } + + public Date getAccountRemoved() { + return accountRemoved; + } + + public Account.State getAccountState() { + return accountState; + } + + public Long getDomainId() { + return domainId; + } + + public String getDomainUuid() { + return domainUuid; + } + + public String getDomainPath() { + return domainPath; + } + + public Date getDomainRemoved() { + return domainRemoved; + } + + public String getProjectUuid() { + return projectUuid; + } + + public String getProjectName() { + return projectName; + } + + public Date getProjectRemoved() { + return projectRemoved; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 453355c8522..304b23b7220 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -19,7 +19,8 @@ - + + (volumePools, volumePaths); } @@ -367,14 +368,24 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co if (ScopeType.HOST.equals(storagePool.getScope()) || Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType()) || Storage.StoragePoolType.RBD.equals(storagePool.getPoolType())) { - volumePathPrefix = storagePool.getPath(); + volumePathPrefix = storagePool.getPath() + "/"; + } else if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + volumePathPrefix = "/dev/drbd/by-res/cs-"; } else { // Should be Storage.StoragePoolType.NetworkFilesystem - volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); + volumePathPrefix = String.format("/mnt/%s/", storagePool.getUuid()); } return volumePathPrefix; } + private String getVolumePathSuffix(StoragePoolVO storagePool) { + if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + return "/0"; + } else { + return ""; + } + } + @Override public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); @@ -419,7 +430,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoreCommand.setBackupRepoType(backupRepository.getType()); restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setVmName(vmNameAndState.first()); - restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", getVolumePathPrefix(pool), volumeUUID))); + String restoreVolumePath = String.format("%s%s%s", getVolumePathPrefix(pool), volumeUUID, getVolumePathSuffix(pool)); + restoreCommand.setRestoreVolumePaths(Collections.singletonList(restoreVolumePath)); + restoreCommand.setRestoreVolumeSizes(Collections.singletonList(backedUpVolumeSize)); DataStore dataStore = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); restoreCommand.setRestoreVolumePools(Collections.singletonList(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null)); restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT)); @@ -478,6 +491,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } else { host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); } + if (host == null) { + throw new CloudRuntimeException(String.format("Unable to find a running KVM host in zone %d to delete backup %s", backup.getZoneId(), backup.getUuid())); + } DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), backupRepository.getAddress(), backupRepository.getMountOptions()); @@ -564,7 +580,14 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public void syncBackupStorageStats(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + if (CollectionUtils.isEmpty(repositories)) { + return; + } final Host host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + if (host == null) { + logger.warn("Unable to find a running KVM host in zone {} to sync backup storage stats", zoneId); + return; + } for (final BackupRepository repository : repositories) { GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions()); BackupStorageStatsAnswer answer; diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java index 87322b01f4d..870b9b6df6e 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java @@ -16,61 +16,80 @@ //under the License. package org.apache.cloudstack.api.command; -import com.cloud.user.Account; import com.cloud.utils.Pair; + +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaSummaryResponse; -import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.QuotaService; +import org.apache.commons.lang3.ObjectUtils; import java.util.List; import javax.inject.Inject; -@APICommand(name = "quotaSummary", responseObject = QuotaSummaryResponse.class, description = "Lists balance and quota usage for all Accounts", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - httpMethod = "GET") +@APICommand(name = "quotaSummary", responseObject = QuotaSummaryResponse.class, description = "Lists Quota balance summary of Accounts and Projects.", since = "4.7.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaSummaryCmd extends BaseListCmd { - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = false, description = "Optional, Account Id for which statement needs to be generated") - private String accountName; - - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = false, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") - private Long domainId; - - @Parameter(name = ApiConstants.LIST_ALL, type = CommandType.BOOLEAN, required = false, description = "Optional, to list all Accounts irrespective of the quota activity") - private Boolean listAll; + @Inject + QuotaResponseBuilder quotaResponseBuilder; @Inject - QuotaResponseBuilder _responseBuilder; + QuotaService quotaService; - public QuotaSummaryCmd() { - super(); - } + @ACL + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "ID of the Account for which balance will be listed. Can not be specified with projectid.", since = "4.23.0") + private Long accountId; + + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = false, description = "Name of the Account for which balance will be listed.") + private String accountName; + + @ACL + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = false, entityType = DomainResponse.class, description = "ID of the Domain for which balance will be listed. May be used individually or with accountname.") + private Long domainId; + + @Parameter(name = ApiConstants.LIST_ALL, type = CommandType.BOOLEAN, description = "False (default) lists the Quota balance summary for calling Account. True lists balance summary for " + + "Accounts which the caller has access. If domain ID is informed, this parameter is considered as true.") + private Boolean listAll; + + @Parameter(name = ApiConstants.ACCOUNT_STATE_TO_SHOW, type = CommandType.STRING, description = "Possible values are [ALL, ACTIVE, REMOVED]. ALL will list summaries for " + + "active and removed accounts; ACTIVE will list summaries only for active accounts; REMOVED will list summaries only for removed accounts. The default value is ACTIVE.", + since = "4.23.0") + private String accountStateToShow; + + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "ID of the Project for which balance will be listed. Can not be specified with accountId.", since = "4.23.0") + private Long projectId; @Override public void execute() { - Account caller = CallContext.current().getCallingAccount(); - Pair, Integer> responses; - if (caller.getType() == Account.Type.ADMIN) { - if (getAccountName() != null && getDomainId() != null) - responses = _responseBuilder.createQuotaSummaryResponse(getAccountName(), getDomainId()); - else - responses = _responseBuilder.createQuotaSummaryResponse(isListAll(), getKeyword(), getStartIndex(), getPageSizeVal()); - } else { - responses = _responseBuilder.createQuotaSummaryResponse(caller.getAccountName(), caller.getDomainId()); - } - final ListResponse response = new ListResponse(); + Pair, Integer> responses = quotaResponseBuilder.createQuotaSummaryResponse(this); + ListResponse response = new ListResponse<>(); response.setResponses(responses.first(), responses.second()); response.setResponseName(getCommandName()); setResponseObject(response); } + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + public String getAccountName() { return accountName; } @@ -88,16 +107,31 @@ public class QuotaSummaryCmd extends BaseListCmd { } public Boolean isListAll() { - return listAll == null ? false: listAll; + // If a domain ID was specified, then allow listing all summaries of domain + return ObjectUtils.defaultIfNull(listAll, Boolean.FALSE) || domainId != null; } public void setListAll(Boolean listAll) { this.listAll = listAll; } - @Override - public long getEntityOwnerId() { - return Account.ACCOUNT_ID_SYSTEM; + public Long getProjectId() { + return projectId; } + public QuotaAccountStateFilter getAccountStateToShow() { + QuotaAccountStateFilter state = QuotaAccountStateFilter.getValue(accountStateToShow); + if (state != null) { + return state; + } + return QuotaAccountStateFilter.ACTIVE; + } + + @Override + public long getEntityOwnerId() { + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; + } + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 67f75ebf82f..177fb00d4b5 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; @@ -52,11 +53,7 @@ public interface QuotaResponseBuilder { QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); - Pair, Integer> createQuotaSummaryResponse(Boolean listAll); - - Pair, Integer> createQuotaSummaryResponse(Boolean listAll, String keyword, Long startIndex, Long pageSize); - - Pair, Integer> createQuotaSummaryResponse(String accountName, Long domainId); + Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd); QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index f9456587058..2d6ec3255f4 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -42,11 +42,14 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.domain.Domain; import com.cloud.exception.PermissionDeniedException; +import com.cloud.projects.dao.ProjectDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaBalanceCmd; @@ -56,6 +59,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; @@ -74,11 +78,13 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariable import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; + import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaCreditsDao; import org.apache.cloudstack.quota.dao.QuotaEmailConfigurationDao; import org.apache.cloudstack.quota.dao.QuotaEmailTemplatesDao; +import org.apache.cloudstack.quota.dao.QuotaSummaryDao; import org.apache.cloudstack.quota.dao.QuotaTariffDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; @@ -86,10 +92,13 @@ import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.compress.utils.Sets; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -108,7 +117,6 @@ import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.Pair; -import com.cloud.utils.db.Filter; @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @@ -121,7 +129,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private QuotaCreditsDao quotaCreditsDao; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageDao quotaUsageDao; @Inject private QuotaEmailTemplatesDao _quotaEmailTemplateDao; @@ -132,29 +140,29 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject + private ProjectDao projectDao; + @Inject private QuotaAccountDao quotaAccountDao; @Inject - private DomainDao _domainDao; + private DomainDao domainDao; @Inject private AccountManager _accountMgr; @Inject - private QuotaStatement _statement; + private QuotaStatement quotaStatement; @Inject private QuotaManager _quotaManager; @Inject private QuotaEmailConfigurationDao quotaEmailConfigurationDao; @Inject + private QuotaSummaryDao quotaSummaryDao; + @Inject private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; - protected void checkActivationRulesAllowed(String activationRule) { - if (!_quotaService.isJsInterpretationEnabled() && StringUtils.isNotEmpty(activationRule)) { - throw new PermissionDeniedException("Quota Tariff Activation Rule cannot be set, as Javascript interpretation is disabled in the configuration."); - } - } + private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); @Override public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff, boolean returnActivationRule) { @@ -180,75 +188,113 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } @Override - public Pair, Integer> createQuotaSummaryResponse(final String accountName, final Long domainId) { - List result = new ArrayList(); + public Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd) { + Account caller = CallContext.current().getCallingAccount(); - if (accountName != null && domainId != null) { - Account account = _accountDao.findActiveAccount(accountName, domainId); - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); + if (!accountTypesThatCanListAllQuotaSummaries.contains(caller.getType()) || !cmd.isListAll()) { + return getQuotaSummaryResponse(cmd.getEntityOwnerId(), null, null, cmd); } - return new Pair<>(result, result.size()); + return getQuotaSummaryResponseWithListAll(cmd, caller); } - @Override - public Pair, Integer> createQuotaSummaryResponse(Boolean listAll) { - return createQuotaSummaryResponse(listAll, null, null, null); - } - - @Override - public Pair, Integer> createQuotaSummaryResponse(Boolean listAll, final String keyword, final Long startIndex, final Long pageSize) { - List result = new ArrayList(); - Integer count = 0; - if (listAll) { - Filter filter = new Filter(AccountVO.class, "accountName", true, startIndex, pageSize); - Pair, Integer> data = _accountDao.findAccountsLike(keyword, filter); - count = data.second(); - for (final AccountVO account : data.first()) { - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); - } - } else { - Pair, Integer> data = quotaAccountDao.listAllQuotaAccount(startIndex, pageSize); - count = data.second(); - for (final QuotaAccountVO quotaAccount : data.first()) { - AccountVO account = _accountDao.findById(quotaAccount.getId()); - if (account == null) { - continue; - } - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); + protected Pair, Integer> getQuotaSummaryResponseWithListAll(QuotaSummaryCmd cmd, Account caller) { + Long domainId = cmd.getDomainId(); + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + if (domain == null) { + throw new InvalidParameterValueException(String.format("Domain [%s] does not exist.", domainId)); } } - return new Pair<>(result, count); + + String domainPath = getDomainPathByDomainIdForDomainAdmin(caller); + + Long accountId = cmd.getEntityOwnerId(); + if (accountId == -1) { + accountId = cmd.isListAll() ? null : caller.getAccountId(); + } + + return getQuotaSummaryResponse(accountId, domainId, domainPath, cmd); } - protected QuotaSummaryResponse getQuotaSummaryResponse(final Account account) { - Calendar[] period = _statement.getCurrentStatementTime(); - - if (account != null) { - QuotaSummaryResponse qr = new QuotaSummaryResponse(); - DomainVO domain = _domainDao.findById(account.getDomainId()); - BigDecimal curBalance = _quotaBalanceDao.lastQuotaBalance(account.getAccountId(), account.getDomainId(), period[1].getTime()); - BigDecimal quotaUsage = _quotaUsageDao.findTotalQuotaUsage(account.getAccountId(), account.getDomainId(), null, period[0].getTime(), period[1].getTime()); - - qr.setAccountId(account.getUuid()); - qr.setAccountName(account.getAccountName()); - qr.setDomainId(domain.getUuid()); - qr.setDomainName(domain.getName()); - qr.setBalance(curBalance); - qr.setQuotaUsage(quotaUsage); - qr.setState(account.getState()); - qr.setStartDate(period[0].getTime()); - qr.setEndDate(period[1].getTime()); - qr.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - qr.setQuotaEnabled(QuotaConfig.QuotaAccountEnabled.valueIn(account.getId())); - qr.setObjectName("summary"); - return qr; - } else { - return new QuotaSummaryResponse(); + /** + * Retrieves the domain path of the caller's domain (if the caller is Domain Admin) for filtering in the quota summary query. + * @return null if the caller is an Admin or the domain path of the caller's domain if the caller is a Domain Admin. + * @throws InvalidParameterValueException if it cannot find the domain. + */ + protected String getDomainPathByDomainIdForDomainAdmin(Account caller) { + if (caller.getType() != Account.Type.DOMAIN_ADMIN) { + return null; } + + Long domainId = caller.getDomainId(); + Domain domain = domainDao.findById(domainId); + _accountMgr.checkAccess(caller, domain); + + if (domain == null) { + throw new InvalidParameterValueException(String.format("Domain ID [%s] is invalid.", domainId)); + } + + return domain.getPath(); + } + + /** + * Returns a List of QuotaSummaryResponse based on the provided parameters. + * @param accountId ID of the Account to return the summaries for. If -1, either because no specific + * Account was provided, or list all is disabled, then the summary is generated for the calling Account. + * @param domainId ID of the Domain to return the summaries for. + * @param domainPath path of the Domain to return the summaries for. + */ + protected Pair, Integer> getQuotaSummaryResponse(Long accountId, Long domainId, String domainPath, QuotaSummaryCmd cmd) { + if (accountId != null && accountId == -1) { + accountId = CallContext.current().getCallingAccountId(); + } + + Pair, Integer> pairSummaries = quotaSummaryDao.listQuotaSummariesForAccountAndOrDomain(accountId, cmd.getKeyword(), domainId, domainPath, + cmd.getAccountStateToShow(), cmd.getStartIndex(), cmd.getPageSizeVal()); + List summaries = pairSummaries.first(); + + if (CollectionUtils.isEmpty(summaries)) { + logger.info("There are no summaries to list for parameters [{}].", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "domainId", "listAll", "page", "pageSize")); + return new Pair<>(new ArrayList<>(), 0); + } + + List responses = summaries.stream().map(this::getQuotaSummaryResponse).collect(Collectors.toList()); + + return new Pair<>(responses, pairSummaries.second()); + } + + protected QuotaSummaryResponse getQuotaSummaryResponse(QuotaSummaryVO summary) { + QuotaSummaryResponse response = new QuotaSummaryResponse(); + Account account = _accountDao.findByUuidIncludingRemoved(summary.getAccountUuid()); + + Calendar[] period = quotaStatement.getCurrentStatementTime(); + Date startDate = period[0].getTime(); + Date endDate = period[1].getTime(); + BigDecimal quotaUsage = quotaUsageDao.findTotalQuotaUsage(account.getAccountId(), account.getDomainId(), null, startDate, endDate); + + response.setQuotaUsage(quotaUsage); + response.setStartDate(startDate); + response.setEndDate(endDate); + response.setAccountId(summary.getAccountUuid()); + response.setAccountName(summary.getAccountName()); + response.setDomainId(summary.getDomainUuid()); + response.setDomainPath(summary.getDomainPath()); + response.setBalance(summary.getQuotaBalance()); + response.setState(summary.getAccountState()); + response.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + response.setQuotaEnabled(QuotaConfig.QuotaAccountEnabled.valueIn(account.getId())); + response.setDomainRemoved(summary.getDomainRemoved() != null); + response.setAccountRemoved(summary.getAccountRemoved() != null); + response.setObjectName("summary"); + + if (summary.getProjectUuid() != null) { + response.setProjectId(summary.getProjectUuid()); + response.setProjectName(summary.getProjectName()); + response.setProjectRemoved(summary.getProjectRemoved() != null); + } + + return response; } public boolean isUserAllowedToSeeActivationRules(User user) { @@ -450,6 +496,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { Integer position = cmd.getPosition(); warnQuotaTariffUpdateDeprecatedFields(cmd); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); @@ -457,8 +504,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { throw new InvalidParameterValueException(String.format("There is no quota tariffs with name [%s].", name)); } - checkActivationRulesAllowed(activationRule); - Date currentQuotaTariffStartDate = currentQuotaTariff.getEffectiveOn(); currentQuotaTariff.setRemoved(now); @@ -707,14 +752,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { String activationRule = cmd.getActivationRule(); Integer position = ObjectUtils.defaultIfNull(cmd.getPosition(), 1); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); + QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); if (currentQuotaTariff != null) { throw new InvalidParameterValueException(String.format("A quota tariff with name [%s] already exist.", name)); } - checkActivationRulesAllowed(activationRule); - if (startDate.compareTo(now) < 0) { throw new InvalidParameterValueException(String.format("The value passed as Quota tariff's start date is in the past: [%s]. " + "Please, inform a date in the future or do not pass the parameter to use the current date and time.", startDate)); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java index f8f27b4813d..d76202bfd88 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import com.google.gson.annotations.SerializedName; @@ -30,40 +29,48 @@ import com.cloud.user.Account.State; public class QuotaSummaryResponse extends BaseResponse { @SerializedName("accountid") - @Param(description = "Account ID") + @Param(description = "Account's ID") private String accountId; @SerializedName("account") - @Param(description = "Account name") + @Param(description = "Account's name") private String accountName; @SerializedName("domainid") - @Param(description = "Domain ID") + @Param(description = "Domain's ID") private String domainId; @SerializedName("domain") - @Param(description = "Domain name") - private String domainName; + @Param(description = "Domain's path") + private String domainPath; @SerializedName("balance") - @Param(description = "Account balance") + @Param(description = "Account's balance") private BigDecimal balance; @SerializedName("state") - @Param(description = "Account state") + @Param(description = "Account's state") private State state; + @SerializedName("domainremoved") + @Param(description = "If the domain is removed or not", since = "4.23.0") + private boolean domainRemoved; + + @SerializedName("accountremoved") + @Param(description = "If the account is removed or not", since = "4.23.0") + private boolean accountRemoved; + @SerializedName("quota") - @Param(description = "Quota usage of this period") + @Param(description = "Quota consumed between the startdate and enddate") private BigDecimal quotaUsage; @SerializedName("startdate") - @Param(description = "Start date") - private Date startDate = null; + @Param(description = "Start date of the quota consumption") + private Date startDate; @SerializedName("enddate") - @Param(description = "End date") - private Date endDate = null; + @Param(description = "End date of the quota consumption") + private Date endDate; @SerializedName("currency") @Param(description = "Currency") @@ -73,9 +80,17 @@ public class QuotaSummaryResponse extends BaseResponse { @Param(description = "If the account has the quota config enabled") private boolean quotaEnabled; - public QuotaSummaryResponse() { - super(); - } + @SerializedName("projectname") + @Param(description = "Name of the project", since = "4.23.0") + private String projectName; + + @SerializedName("projectid") + @Param(description = "Project's id", since = "4.23.0") + private String projectId; + + @SerializedName("projectremoved") + @Param(description = "Whether the project is removed or not", since = "4.23.0") + private Boolean projectRemoved; public String getAccountId() { return accountId; @@ -101,28 +116,16 @@ public class QuotaSummaryResponse extends BaseResponse { this.domainId = domainId; } - public String getDomainName() { - return domainName; - } - - public void setDomainName(String domainName) { - this.domainName = domainName; - } - - public BigDecimal getQuotaUsage() { - return quotaUsage; - } - - public State getState() { - return state; + public void setDomainPath(String domainPath) { + this.domainPath = domainPath; } public void setState(State state) { this.state = state; } - public void setQuotaUsage(BigDecimal startQuota) { - this.quotaUsage = startQuota.setScale(2, RoundingMode.HALF_EVEN); + public void setQuotaUsage(BigDecimal quotaUsage) { + this.quotaUsage = quotaUsage; } public BigDecimal getBalance() { @@ -130,38 +133,42 @@ public class QuotaSummaryResponse extends BaseResponse { } public void setBalance(BigDecimal balance) { - this.balance = balance.setScale(2, RoundingMode.HALF_EVEN); - } - - public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + this.balance = balance; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); - } - - public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + this.startDate = startDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - public String getCurrency() { - return currency; + this.endDate = endDate; } public void setCurrency(String currency) { this.currency = currency; } - public boolean getQuotaEnabled() { - return quotaEnabled; - } - public void setQuotaEnabled(boolean quotaEnabled) { this.quotaEnabled = quotaEnabled; } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + public void setProjectRemoved(Boolean projectRemoved) { + this.projectRemoved = projectRemoved; + } + + public void setDomainRemoved(boolean domainRemoved) { + this.domainRemoved = domainRemoved; + } + + public void setAccountRemoved(boolean accountRemoved) { + this.accountRemoved = accountRemoved; + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index f6a34e01be8..a421d0f066a 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -40,6 +40,4 @@ public interface QuotaService extends PluggableService { boolean saveQuotaAccount(AccountVO account, BigDecimal aggrUsage, Date endDate); - boolean isJsInterpretationEnabled(); - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index f455c3cba14..2d7d623d1d9 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -26,6 +26,8 @@ import java.util.TimeZone; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.projects.ProjectManager; +import com.cloud.user.AccountService; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsCmd; @@ -62,7 +64,6 @@ import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.server.ManagementService; import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; @@ -75,6 +76,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private AccountDao _accountDao; @Inject + private AccountService accountService; + @Inject private QuotaAccountDao _quotaAcc; @Inject private QuotaUsageDao _quotaUsageDao; @@ -86,11 +89,11 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi private QuotaBalanceDao _quotaBalanceDao; @Inject private QuotaResponseBuilder _respBldr; + @Inject + private ProjectManager projectMgr; private TimeZone _usageTimezone; - private boolean jsInterpretationEnabled = false; - public QuotaServiceImpl() { super(); } @@ -102,8 +105,6 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT"); _usageTimezone = TimeZone.getTimeZone(timeZoneStr); - jsInterpretationEnabled = ManagementService.JsInterpretationEnabled.value(); - return true; } @@ -292,9 +293,4 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi _quotaAcc.updateQuotaAccount(accountId, acc); } } - - @Override - public boolean isJsInterpretationEnabled() { - return jsInterpretationEnabled; - } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index 3629bf2e3fe..ea88a106b84 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -16,13 +16,11 @@ // under the License. package org.apache.cloudstack.api.response; -import java.lang.reflect.Field; import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -31,6 +29,7 @@ import java.util.Set; import java.util.HashSet; import java.util.function.Consumer; +import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.PermissionDeniedException; @@ -43,10 +42,10 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.discovery.ApiDiscoveryService; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.jsinterpreter.JsInterpreterHelper; import org.apache.cloudstack.quota.QuotaService; import org.apache.cloudstack.quota.QuotaStatement; @@ -79,7 +78,10 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.Account; @@ -89,8 +91,7 @@ import com.cloud.user.dao.UserDao; import com.cloud.user.User; import junit.framework.TestCase; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; + @RunWith(MockitoJUnitRunner.class) public class QuotaResponseBuilderImplTest extends TestCase { @@ -153,7 +154,7 @@ public class QuotaResponseBuilderImplTest extends TestCase { Account accountMock; @Mock - DomainVO domainVOMock; + DomainVO domainVoMock; @Mock QuotaConfigureEmailCmd quotaConfigureEmailCmdMock; @@ -161,6 +162,9 @@ public class QuotaResponseBuilderImplTest extends TestCase { @Mock QuotaAccountVO quotaAccountVOMock; + @Mock + CallContext callContextMock; + @Mock QuotaEmailTemplatesVO quotaEmailTemplatesVoMock; @@ -184,17 +188,8 @@ public class QuotaResponseBuilderImplTest extends TestCase { CallContext.register(callerUserMock, callerAccountMock); } - private void overrideDefaultQuotaEnabledConfigValue(final Object value) throws IllegalAccessException, NoSuchFieldException { - Field f = ConfigKey.class.getDeclaredField("_defaultValue"); - f.setAccessible(true); - f.set(QuotaConfig.QuotaAccountEnabled, value); - } - - private Calendar[] createPeriodForQuotaSummary() { - final Calendar calendar = Calendar.getInstance(); - calendar.set(Calendar.HOUR, 0); - return new Calendar[] {calendar, calendar}; - } + @Mock + Pair, Integer> quotaSummaryResponseMock1, quotaSummaryResponseMock2; @Mock QuotaValidateActivationRuleCmd quotaValidateActivationRuleCmdMock = Mockito.mock(QuotaValidateActivationRuleCmd.class); @@ -466,36 +461,6 @@ public class QuotaResponseBuilderImplTest extends TestCase { Mockito.verify(quotaTariffVoMock).setRemoved(Mockito.any(Date.class)); } - @Test - public void getQuotaSummaryResponseTestAccountIsNotNullQuotaIsDisabledShouldReturnFalse() throws NoSuchFieldException, IllegalAccessException { - Calendar[] period = createPeriodForQuotaSummary(); - overrideDefaultQuotaEnabledConfigValue("false"); - - Mockito.doReturn(period).when(quotaStatementMock).getCurrentStatementTime(); - Mockito.doReturn(domainVOMock).when(domainDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(BigDecimal.ZERO).when(quotaBalanceDaoMock).lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class)); - Mockito.doReturn(BigDecimal.ZERO).when(quotaUsageDaoMock).findTotalQuotaUsage(Mockito.anyLong(), Mockito.anyLong(), Mockito.isNull(), Mockito.any(Date.class), Mockito.any(Date.class)); - - QuotaSummaryResponse quotaSummaryResponse = quotaResponseBuilderSpy.getQuotaSummaryResponse(accountMock); - - assertFalse(quotaSummaryResponse.getQuotaEnabled()); - } - - @Test - public void getQuotaSummaryResponseTestAccountIsNotNullQuotaIsEnabledShouldReturnTrue() throws NoSuchFieldException, IllegalAccessException { - Calendar[] period = createPeriodForQuotaSummary(); - overrideDefaultQuotaEnabledConfigValue("true"); - - Mockito.doReturn(period).when(quotaStatementMock).getCurrentStatementTime(); - Mockito.doReturn(domainVOMock).when(domainDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(BigDecimal.ZERO).when(quotaBalanceDaoMock).lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class)); - Mockito.doReturn(BigDecimal.ZERO).when(quotaUsageDaoMock).findTotalQuotaUsage(Mockito.anyLong(), Mockito.anyLong(), Mockito.isNull(), Mockito.any(Date.class), Mockito.any(Date.class)); - - QuotaSummaryResponse quotaSummaryResponse = quotaResponseBuilderSpy.getQuotaSummaryResponse(accountMock); - - assertTrue(quotaSummaryResponse.getQuotaEnabled()); - } - @Test public void filterSupportedTypesTestReturnWhenQuotaTypeDoesNotMatch() throws NoSuchFieldException { List> variables = new ArrayList<>(); @@ -576,6 +541,63 @@ public class QuotaResponseBuilderImplTest extends TestCase { quotaResponseBuilderSpy.validateQuotaConfigureEmailCmdParameters(quotaConfigureEmailCmdMock); } + @Test + public void createQuotaSummaryResponseTestNotListAllAndAllAccountTypesReturnsSingleRecord() { + QuotaSummaryCmd cmd = new QuotaSummaryCmd(); + + try(MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + + Mockito.doReturn(quotaSummaryResponseMock1).when(quotaResponseBuilderSpy).getQuotaSummaryResponse(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + + for (Account.Type type : Account.Type.values()) { + Mockito.doReturn(type).when(accountMock).getType(); + + Pair, Integer> result = quotaResponseBuilderSpy.createQuotaSummaryResponse(cmd); + Assert.assertEquals(quotaSummaryResponseMock1, result); + } + + Mockito.verify(quotaResponseBuilderSpy, Mockito.times(Account.Type.values().length)).getQuotaSummaryResponse(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any()); + }; + } + + @Test + public void getDomainPathByDomainIdForDomainAdminTestAccountNotDomainAdminReturnsNull() { + for (Account.Type type : Account.Type.values()) { + if (Account.Type.DOMAIN_ADMIN.equals(type)) { + continue; + } + + Mockito.doReturn(type).when(accountMock).getType(); + Assert.assertNull(quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock)); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void getDomainPathByDomainIdForDomainAdminTestDomainFromCallerIsNullThrowsInvalidParameterValueException() { + Mockito.doReturn(Account.Type.DOMAIN_ADMIN).when(accountMock).getType(); + Mockito.doReturn(null).when(domainDaoMock).findById(Mockito.anyLong()); + Mockito.lenient().doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + + quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock); + } + + @Test + public void getDomainPathByDomainIdForDomainAdminTestDomainFromCallerIsNotNullReturnsPath() { + String expected = "/test/"; + + Mockito.doReturn(Account.Type.DOMAIN_ADMIN).when(accountMock).getType(); + Mockito.doReturn(domainVoMock).when(domainDaoMock).findById(Mockito.anyLong()); + Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + Mockito.doReturn(expected).when(domainVoMock).getPath(); + + String result = quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock); + Assert.assertEquals(expected, result); + } + @Test public void getQuotaEmailConfigurationVoTestTemplateNameIsNull() { Mockito.doReturn(null).when(quotaConfigureEmailCmdMock).getTemplateName(); diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 19e756d1d97..259264f3b0e 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.quota; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; +import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; import junit.framework.TestCase; @@ -33,8 +34,10 @@ import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import javax.naming.ConfigurationException; @@ -48,7 +51,7 @@ import java.util.List; public class QuotaServiceImplTest extends TestCase { @Mock - AccountDao accountDao; + AccountDao accountDaoMock; @Mock QuotaAccountDao quotaAcc; @Mock @@ -61,8 +64,13 @@ public class QuotaServiceImplTest extends TestCase { QuotaBalanceDao quotaBalanceDao; @Mock QuotaResponseBuilder respBldr; + @Mock + private AccountVO accountVoMock; + + @Spy + @InjectMocks + QuotaServiceImpl quotaServiceImplSpy; - QuotaServiceImpl quotaService = new QuotaServiceImpl(); @Before public void setup() throws IllegalAccessException, NoSuchFieldException, ConfigurationException { @@ -71,34 +79,34 @@ public class QuotaServiceImplTest extends TestCase { Field accountDaoField = QuotaServiceImpl.class.getDeclaredField("_accountDao"); accountDaoField.setAccessible(true); - accountDaoField.set(quotaService, accountDao); + accountDaoField.set(quotaServiceImplSpy, accountDaoMock); Field quotaAccountDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaAcc"); quotaAccountDaoField.setAccessible(true); - quotaAccountDaoField.set(quotaService, quotaAcc); + quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaService, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); - domainDaoField.set(quotaService, domainDao); + domainDaoField.set(quotaServiceImplSpy, domainDao); Field configDaoField = QuotaServiceImpl.class.getDeclaredField("_configDao"); configDaoField.setAccessible(true); - configDaoField.set(quotaService, configDao); + configDaoField.set(quotaServiceImplSpy, configDao); Field balanceDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaBalanceDao"); balanceDaoField.setAccessible(true); - balanceDaoField.set(quotaService, quotaBalanceDao); + balanceDaoField.set(quotaServiceImplSpy, quotaBalanceDao); Field QuotaResponseBuilderField = QuotaServiceImpl.class.getDeclaredField("_respBldr"); QuotaResponseBuilderField.setAccessible(true); - QuotaResponseBuilderField.set(quotaService, respBldr); + QuotaResponseBuilderField.set(quotaServiceImplSpy, respBldr); Mockito.when(configDao.getValue(Mockito.eq(Config.UsageAggregationTimezone.toString()))).thenReturn("IST"); - quotaService.configure("randomName", null); + quotaServiceImplSpy.configure("randomName", null); } @Test @@ -120,9 +128,9 @@ public class QuotaServiceImplTest extends TestCase { Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); // with enddate - assertTrue(quotaService.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); + assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); // without enddate - assertTrue(quotaService.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); + assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); } @Test @@ -133,7 +141,7 @@ public class QuotaServiceImplTest extends TestCase { final Date startDate = new DateTime().minusDays(2).toDate(); final Date endDate = new Date(); - quotaService.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); + quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); } @@ -142,13 +150,13 @@ public class QuotaServiceImplTest extends TestCase { // existing account QuotaAccountVO quotaAccountVO = new QuotaAccountVO(); Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(quotaAccountVO); - quotaService.setLockAccount(2L, true); + quotaServiceImplSpy.setLockAccount(2L, true); Mockito.verify(quotaAcc, Mockito.times(0)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); Mockito.verify(quotaAcc, Mockito.times(1)).updateQuotaAccount(Mockito.anyLong(), Mockito.any(QuotaAccountVO.class)); // new account Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(null); - quotaService.setLockAccount(2L, true); + quotaServiceImplSpy.setLockAccount(2L, true); Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } @@ -160,13 +168,14 @@ public class QuotaServiceImplTest extends TestCase { // existing account setting QuotaAccountVO quotaAccountVO = new QuotaAccountVO(); Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(quotaAccountVO); - quotaService.setMinBalance(accountId, balance); + quotaServiceImplSpy.setMinBalance(accountId, balance); Mockito.verify(quotaAcc, Mockito.times(0)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); Mockito.verify(quotaAcc, Mockito.times(1)).updateQuotaAccount(Mockito.anyLong(), Mockito.any(QuotaAccountVO.class)); // no account with limit set Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(null); - quotaService.setMinBalance(accountId, balance); + quotaServiceImplSpy.setMinBalance(accountId, balance); Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java index 08086859fb7..d3765c01ccb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java @@ -79,28 +79,47 @@ public class LibvirtGpuDef { gpuBuilder.append(" \n"); gpuBuilder.append(" \n"); - // Parse the bus address (e.g., 00:02.0) into domain, bus, slot, function - String domain = "0x0000"; - String bus = "0x00"; - String slot = "0x00"; - String function = "0x0"; + // Parse the bus address into domain, bus, slot, function. Two input formats are accepted: + // - "dddd:bb:ss.f" full PCI address with domain (e.g. 0000:00:02.0) + // - "bb:ss.f" legacy short BDF; domain defaults to 0000 + // Each segment is parsed as a hex integer and formatted with fixed widths + // (domain: 4 hex digits, bus/slot: 2 hex digits, function: 1 hex digit) to + // produce canonical libvirt XML values regardless of input casing or padding. + int domainVal = 0, busVal = 0, slotVal = 0, funcVal = 0; if (busAddress != null && !busAddress.isEmpty()) { String[] parts = busAddress.split(":"); - if (parts.length > 1) { - bus = "0x" + parts[0]; - String[] slotFunctionParts = parts[1].split("\\."); - if (slotFunctionParts.length > 0) { - slot = "0x" + slotFunctionParts[0]; - if (slotFunctionParts.length > 1) { - function = "0x" + slotFunctionParts[1].trim(); - } + try { + String slotFunction; + if (parts.length == 3) { + domainVal = Integer.parseInt(parts[0], 16); + busVal = Integer.parseInt(parts[1], 16); + slotFunction = parts[2]; + } else if (parts.length == 2) { + busVal = Integer.parseInt(parts[0], 16); + slotFunction = parts[1]; + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); } + String[] sf = slotFunction.split("\\."); + if (sf.length == 2) { + slotVal = Integer.parseInt(sf[0], 16); + funcVal = Integer.parseInt(sf[1].trim(), 16); + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'", e); } } + String domain = String.format("0x%04x", domainVal); + String bus = String.format("0x%02x", busVal); + String slot = String.format("0x%02x", slotVal); + String function = String.format("0x%x", funcVal); + gpuBuilder.append("
\n"); + .append(slot).append("' function='").append(function).append("'/>\n"); gpuBuilder.append(" \n"); gpuBuilder.append("\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java index 55225ba086c..6788516df74 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -47,7 +47,10 @@ import java.util.Map; @ResourceWrapper(handles = CheckVolumeCommand.class) public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper { - private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem); + private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList( + Storage.StoragePoolType.Filesystem, + Storage.StoragePoolType.NetworkFilesystem, + Storage.StoragePoolType.SharedMountPoint); @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 6d918435277..ab6bdd6135d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -52,7 +52,7 @@ import java.util.stream.Collectors; public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapper { static final List STORAGE_POOL_TYPES_SUPPORTED_BY_QEMU_IMG = Arrays.asList(StoragePoolType.NetworkFilesystem, - StoragePoolType.Filesystem, StoragePoolType.RBD); + StoragePoolType.Filesystem, StoragePoolType.RBD, StoragePoolType.SharedMountPoint); @Override public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 714e3844b34..22dbfbdd67a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -41,9 +41,9 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.Locale; @@ -56,10 +56,25 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper restoreVolumePools = command.getRestoreVolumePools(); List restoreVolumePaths = command.getRestoreVolumePaths(); Integer mountTimeout = command.getMountTimeout() * 1000; - int timeout = command.getWait(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : serverResource.getCmdsTimeout(); KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); List backupFiles = command.getBackupFiles(); @@ -84,9 +99,9 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(vmName, command.getVmState()), mountDirectory, timeout); } else if (Boolean.TRUE.equals(vmExists)) { restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); @@ -143,7 +158,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper vmNameAndState, String mountDirectory, int timeout) { + Long size, Pair vmNameAndState, String mountDirectory, int timeout) { String bkpPath; String volumeUuid; try { bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType); - volumeUuid = volumePath.substring(volumePath.lastIndexOf(File.separator) + 1); + volumeUuid = getVolumeUuidFromPath(volumePath, volumePool); verifyBackupFile(bkpPath, volumeUuid); - if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true)) { + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true, size)) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", volumeUuid)); } @@ -247,42 +262,66 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper volumePools = command.getVolumePools(); final List volumePaths = command.getVolumePaths(); KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : libvirtComputingResource.getCmdsTimeout(); List diskPaths = new ArrayList<>(); if (Objects.nonNull(volumePaths)) { @@ -81,7 +82,7 @@ public class LibvirtTakeBackupCommandWrapper extends CommandWrapper result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + Pair result = Script.executePipedCommands(commands, timeout); if (result.first() != 0) { logger.debug("Failed to take VM backup: " + result.second()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index e336e0e5a54..82bc35f009e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Predicate; import com.cloud.agent.api.PrepareStorageClientCommand; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; @@ -644,18 +645,30 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { // Assuming SDC service is started, add mdms String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS); String[] mdmAddresses = mdms.split(","); - if (mdmAddresses.length > 0) { - if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(true, getSDCDetails(details), "MDM added, no need to prepare the SDC client"); - } - - ScaleIOUtil.addMdms(mdmAddresses); - if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(false, null, "Failed to add MDMs"); + // remove MDMs already present in the config and added to the SDC + String[] mdmAddressesToAdd = Arrays.stream(mdmAddresses) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + // if all MDMs are already in the config and added to the SDC + if (mdmAddressesToAdd.length < 1 && mdmAddresses.length > 0) { + String msg = String.format("MDMs %s of the storage pool %s are already added", mdms, uuid); + logger.debug(msg); + return new Ternary<>(true, getSDCDetails(details), msg); + } else if (mdmAddressesToAdd.length > 0) { + ScaleIOUtil.addMdms(mdmAddressesToAdd); + String[] missingMdmAddresses = Arrays.stream(mdmAddressesToAdd) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + if (missingMdmAddresses.length > 0) { + String msg = String.format("Failed to add MDMs %s of the storage pool %s", String.join(", ", missingMdmAddresses), uuid); + logger.debug(msg); + return new Ternary<>(false, null, msg); } else { - logger.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid)); + logger.debug("MDMs {} of the storage pool {} are added", mdmAddressesToAdd, uuid); applyMdmsChangeWaitTime(details); } + } else { + return new Ternary<>(false, getSDCDetails(details), "No MDM addresses provided"); } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java index 0060e1d7ed4..e6f63e852f8 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java @@ -115,6 +115,145 @@ public class LibvirtGpuDefTest extends TestCase { assertTrue(gpuXml.contains("")); } + @Test + public void testGpuDef_withFullPciAddressDomainZero() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0000:00:02.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0001:65:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withNvidiaStyleEightDigitDomain() { + // nvidia-smi reports PCI addresses with an 8-digit domain (e.g. "00000001:af:00.1"). + // generatePciXml must normalize it to the canonical 4-digit form "0x0001". + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "00000001:af:00.1", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressVfNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo vfGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "VF-Profile", + "VF-Profile", + "0002:81:00.3", + "10de", + "NVIDIA Corporation", + "1eb8", + "Tesla T4" + ); + gpuDef.defGpu(vfGpuInfo); + + String gpuXml = gpuDef.toString(); + + // Non-passthrough NVIDIA VFs should be unmanaged + assertTrue(gpuXml.contains("")); + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withLegacyShortBdfDefaultsDomainToZero() { + // Backward compatibility: short BDF with no domain segment must still + // produce a valid libvirt address with domain 0x0000. + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "af:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withInvalidBusAddressThrows() { + String[] invalidAddresses = { + "notahex:00.0", // non-hex bus + "gg:00:02.0", // non-hex domain + "00:02:03:04", // too many colon-separated parts + "00", // missing slot/function + "00:02", // missing function (no dot) + "00:02.0.1", // extra dot in ss.f + }; + for (String addr : invalidAddresses) { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo info = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + addr, + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(info); + try { + String ignored = gpuDef.toString(); + fail("Expected IllegalArgumentException for address: " + addr + " but got: " + ignored); + } catch (IllegalArgumentException e) { + assertTrue("Exception message should contain the bad address", + e.getMessage().contains(addr)); + } + } + } + @Test public void testGpuDef_withNullDeviceType() { LibvirtGpuDef gpuDef = new LibvirtGpuDef(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java index d72dc0d8ac3..ef6b5c08189 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.storage.Storage; import com.cloud.utils.script.Script; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.backup.BackupAnswer; @@ -66,7 +67,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -109,6 +113,7 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.isVmExists()).thenReturn(true); when(command.getDiskType()).thenReturn("root"); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupVolumesUUIDs()).thenReturn(Arrays.asList("volume-123")); @@ -148,6 +153,7 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.isVmExists()).thenReturn(false); when(command.getDiskType()).thenReturn("root"); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -185,7 +191,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("username=user,password=pass"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -226,7 +235,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { lenient().when(command.getMountOptions()).thenReturn("rw"); lenient().when(command.isVmExists()).thenReturn(null); lenient().when(command.getDiskType()).thenReturn("root"); + lenient().when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + lenient().when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + lenient().when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); lenient().when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -262,7 +274,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -308,7 +323,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -356,7 +374,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -370,7 +391,15 @@ public class LibvirtRestoreBackupCommandWrapperTest { try (MockedStatic