From 314c4591ecbf27b7d1b08d3f9f50270ed16e2585 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 8 Oct 2025 10:25:36 +0200 Subject: [PATCH 001/127] systemvmtemplate: Bump Debian version to 12.12.0 (#11778) --- .../template-base_aarch64-target_aarch64.json | 4 ++-- .../systemvmtemplate/template-base_x86_64-target_aarch64.json | 4 ++-- .../systemvmtemplate/template-base_x86_64-target_x86_64.json | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json index e87c37be61e..f15552ca5cc 100644 --- a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json @@ -32,8 +32,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:892cf1185a214d16ff62a18c6b89cdcd58719647c99916f6214bfca6f9915275d727b666c0b8fbf022c425ef18647e9759974abf7fc440431c39b50c296a98d3", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.11.0/arm64/iso-cd/debian-12.11.0-arm64-netinst.iso", + "iso_checksum": "sha512:55ab206cd8b0da2898767c3eb6ab5ebef101e3925ec91b3b5f0a286136195b7072588f6ac2d059c545c6938978704ae78cd18d7d9d2a86a7380e46ce27ee4e7b", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.12.0/arm64/iso-cd/debian-12.12.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json index 4d803b25890..c16bf1fb2a2 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json @@ -31,8 +31,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:892cf1185a214d16ff62a18c6b89cdcd58719647c99916f6214bfca6f9915275d727b666c0b8fbf022c425ef18647e9759974abf7fc440431c39b50c296a98d3", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.11.0/arm64/iso-cd/debian-12.11.0-arm64-netinst.iso", + "iso_checksum": "sha512:55ab206cd8b0da2898767c3eb6ab5ebef101e3925ec91b3b5f0a286136195b7072588f6ac2d059c545c6938978704ae78cd18d7d9d2a86a7380e46ce27ee4e7b", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.12.0/arm64/iso-cd/debian-12.12.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json index e825e98e5f1..941eea9dcd7 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:0921d8b297c63ac458d8a06f87cd4c353f751eb5fe30fd0d839ca09c0833d1d9934b02ee14bbd0c0ec4f8917dde793957801ae1af3c8122cdf28dde8f3c3e0da", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.11.0/amd64/iso-cd/debian-12.11.0-amd64-netinst.iso", + "iso_checksum": "sha512:c93055182057dd19a334260671c7e10880541b7721ad9c8df87be47e0a11d5bbf85018350ff224ff6a5f6a68320b07e95d539cef9dc020c93966bfaa86d4b2ce", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.12.0/amd64/iso-cd/debian-12.12.0-amd64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [ From 270d3f9a2da79319637738a32fb0d04f5df8c96c Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 8 Oct 2025 10:42:00 +0200 Subject: [PATCH 002/127] UI: Deal with crosssite api call after login (#10533) --- ui/public/config.json | 3 ++- ui/src/store/modules/user.js | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ui/public/config.json b/ui/public/config.json index 38d1fd9bbe7..64d10284186 100644 --- a/ui/public/config.json +++ b/ui/public/config.json @@ -97,5 +97,6 @@ "basicZoneEnabled": true, "multipleServer": false, "allowSettingTheme": true, - "docHelpMappings": {} + "docHelpMappings": {}, + "notifyLatestCSVersion": true } diff --git a/ui/src/store/modules/user.js b/ui/src/store/modules/user.js index 5c9dff69f96..fc1b0dc25a9 100644 --- a/ui/src/store/modules/user.js +++ b/ui/src/store/modules/user.js @@ -317,7 +317,6 @@ const user = { const result = response.listusersresponse.user[0] commit('SET_INFO', result) commit('SET_NAME', result.firstname + ' ' + result.lastname) - store.dispatch('SetCsLatestVersion', result.rolename) resolve(cachedApis) }).catch(error => { reject(error) @@ -564,6 +563,9 @@ const user = { commit('SET_DOMAIN_STORE', domainStore) }, SetCsLatestVersion ({ commit }, rolename) { + if (!vueProps.$config.notifyLatestCSVersion) { + return + } const lastFetchTs = store.getters.latestVersion?.fetchedTs ? store.getters.latestVersion.fetchedTs : 0 if (rolename === 'Root Admin' && (+new Date() - lastFetchTs) > 24 * 60 * 60 * 1000) { axios.get( From 9f20979bcee61509788f0f26520ed170a2398040 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Wed, 8 Oct 2025 17:08:03 +0530 Subject: [PATCH 003/127] UI: Fix primary storage for datastore cluster and retain traffic labels during zone deployment (#11760) --- ui/src/views/infra/zone/ZoneWizardAddResources.vue | 2 +- ui/src/views/infra/zone/ZoneWizardLaunchZone.vue | 4 ++-- .../views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui/src/views/infra/zone/ZoneWizardAddResources.vue b/ui/src/views/infra/zone/ZoneWizardAddResources.vue index c37b70d9444..4bd602f0aca 100644 --- a/ui/src/views/infra/zone/ZoneWizardAddResources.vue +++ b/ui/src/views/infra/zone/ZoneWizardAddResources.vue @@ -396,7 +396,7 @@ export default { placeHolder: 'message.error.server', required: true, display: { - primaryStorageProtocol: ['nfs', 'iscsi', 'gluster', 'SMB', 'Linstor'] + primaryStorageProtocol: ['nfs', 'iscsi', 'gluster', 'SMB', 'Linstor', 'datastorecluster', 'vmfs'] } }, { diff --git a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue index ea0b9c8c909..a787ad839cd 100644 --- a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue +++ b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue @@ -1510,10 +1510,10 @@ export default { } path += '/' + this.prefillContent.primaryStorageVmfsDatastore if (protocol === 'vmfs') { - url = this.vmfsURL('dummy', path) + url = this.vmfsURL(server, path) } if (protocol === 'datastorecluster') { - url = this.datastoreclusterURL('dummy', path) + url = this.datastoreclusterURL(server, path) } } else if (protocol === 'iscsi') { let iqn = this.prefillContent?.primaryStorageTargetIQN || '' diff --git a/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue b/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue index 1117cb6ec01..5ef0ff107be 100644 --- a/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue +++ b/ui/src/views/infra/zone/ZoneWizardPhysicalNetworkSetupStep.vue @@ -413,7 +413,7 @@ export default { for (const index in net.traffics) { if (this.hypervisor === 'VMware') { delete this.physicalNetworks[idx].traffics[index].label - } else { + } else if (!net.traffics[index].label) { this.physicalNetworks[idx].traffics[index].label = '' } const traffic = net.traffics[index] From cc3170577c12aed73e9d59d440337ce3cc50216d Mon Sep 17 00:00:00 2001 From: Henrique Sato Date: Wed, 8 Oct 2025 08:39:28 -0300 Subject: [PATCH 004/127] Add `Hypervisor default` as cache mode for disk offerings (#10282) Co-authored-by: Henrique Sato --- api/src/main/java/com/cloud/offering/DiskOffering.java | 2 +- .../api/command/admin/offering/CreateDiskOfferingCmd.java | 2 +- .../command/admin/offering/CreateServiceOfferingCmd.java | 2 +- .../cloudstack/api/response/ServiceOfferingResponse.java | 2 +- .../org/apache/cloudstack/storage/to/VolumeObjectTO.java | 6 +++++- .../main/resources/META-INF/db/schema-42100to42200.sql | 3 +++ .../com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java | 2 +- .../datastore/adapter/ProviderAdapterDiskOffering.java | 8 +++++--- .../com/cloud/configuration/ConfigurationManagerImpl.java | 2 +- ui/public/locales/en.json | 3 ++- ui/public/locales/pt_BR.json | 3 ++- ui/src/views/offering/AddComputeOffering.vue | 3 +++ ui/src/views/offering/AddDiskOffering.vue | 5 ++++- 13 files changed, 30 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/cloud/offering/DiskOffering.java b/api/src/main/java/com/cloud/offering/DiskOffering.java index e1c41f77cbf..9407acfdec1 100644 --- a/api/src/main/java/com/cloud/offering/DiskOffering.java +++ b/api/src/main/java/com/cloud/offering/DiskOffering.java @@ -37,7 +37,7 @@ public interface DiskOffering extends InfrastructureEntity, Identity, InternalId State getState(); enum DiskCacheMode { - NONE("none"), WRITEBACK("writeback"), WRITETHROUGH("writethrough"); + NONE("none"), WRITEBACK("writeback"), WRITETHROUGH("writethrough"), HYPERVISOR_DEFAULT("hypervisor_default"); private final String _diskCacheMode; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java index c46e4cd6b44..557c76d63c3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java @@ -151,7 +151,7 @@ public class CreateDiskOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.CACHE_MODE, type = CommandType.STRING, required = false, - description = "the cache mode to use for this disk offering. none, writeback or writethrough", + description = "the cache mode to use for this disk offering. none, writeback, writethrough or hypervisor default. If the hypervisor default cache mode is used on other hypervisors than KVM, it will fall back to none cache mode", since = "4.14") private String cacheMode; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java index 3d20ed50a5d..ec109a2a4f3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java @@ -190,7 +190,7 @@ public class CreateServiceOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.CACHE_MODE, type = CommandType.STRING, required = false, - description = "the cache mode to use for this disk offering. none, writeback or writethrough", + description = "the cache mode to use for this disk offering. none, writeback, writethrough or hypervisor default. If the hypervisor default cache mode is used on other hypervisors than KVM, it will fall back to none cache mode", since = "4.14") private String cacheMode; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java index 4565a878b34..69f80b54010 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java @@ -197,7 +197,7 @@ public class ServiceOfferingResponse extends BaseResponseWithAnnotations { private Boolean isCustomized; @SerializedName("cacheMode") - @Param(description = "the cache mode to use for this disk offering. none, writeback or writethrough", since = "4.14") + @Param(description = "the cache mode to use for this disk offering. none, writeback, writethrough or hypervisor default", since = "4.14") private String cacheMode; @SerializedName("vspherestoragepolicy") diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 92f9ee497a4..827403ac5ef 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -116,8 +116,8 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { iopsWriteRate = volume.getIopsWriteRate(); iopsWriteRateMax = volume.getIopsWriteRateMax(); iopsWriteRateMaxLength = volume.getIopsWriteRateMaxLength(); - cacheMode = volume.getCacheMode(); hypervisorType = volume.getHypervisorType(); + setCacheMode(volume.getCacheMode()); setDeviceId(volume.getDeviceId()); this.migrationOptions = volume.getMigrationOptions(); this.directDownload = volume.isDirectDownload(); @@ -343,6 +343,10 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { } public void setCacheMode(DiskCacheMode cacheMode) { + if (DiskCacheMode.HYPERVISOR_DEFAULT.equals(cacheMode) && !Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { + this.cacheMode = DiskCacheMode.NONE; + return; + } this.cacheMode = cacheMode; } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index d6087ed9a5f..405f2af9564 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -26,6 +26,9 @@ CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', ' -- Increase length of scripts_version column to 128 due to md5sum to sha512sum change CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.domain_router', 'scripts_version', 'scripts_version', 'VARCHAR(128)'); +-- Increase the cache_mode column size from cloud.disk_offering table +CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.disk_offering', 'cache_mode', 'cache_mode', 'varchar(18) DEFAULT "none" COMMENT "The disk cache mode to use for disks created with this offering"'); + -- Add uuid column to ldap_configuration table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.ldap_configuration', 'uuid', 'VARCHAR(40) NOT NULL'); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 46a5024bf15..bf002b37f35 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -707,7 +707,7 @@ public class LibvirtVMDef { } public enum DiskCacheMode { - NONE("none"), WRITEBACK("writeback"), WRITETHROUGH("writethrough"); + NONE("none"), WRITEBACK("writeback"), WRITETHROUGH("writethrough"), HYPERVISOR_DEFAULT("default"); String _diskCacheMode; DiskCacheMode(String cacheMode) { diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderAdapterDiskOffering.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderAdapterDiskOffering.java index 1db5efbb8ec..293f5f3db2a 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderAdapterDiskOffering.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/adapter/ProviderAdapterDiskOffering.java @@ -34,7 +34,7 @@ public class ProviderAdapterDiskOffering { this.type = ProvisioningType.getProvisioningType(hiddenDiskOffering.getProvisioningType().toString()); } if (hiddenDiskOffering.getCacheMode() != null) { - this.diskCacheMode = DiskCacheMode.getDiskCasehMode(hiddenDiskOffering.getCacheMode().toString()); + this.diskCacheMode = DiskCacheMode.getDiskCacheMode(hiddenDiskOffering.getCacheMode().toString()); } if (hiddenDiskOffering.getState() != null) { this.state = State.valueOf(hiddenDiskOffering.getState().toString()); @@ -166,7 +166,7 @@ public class ProviderAdapterDiskOffering { } enum DiskCacheMode { - NONE("none"), WRITEBACK("writeback"), WRITETHROUGH("writethrough"); + NONE("none"), WRITEBACK("writeback"), WRITETHROUGH("writethrough"), HYPERVISOR_DEFAULT("hypervisor_default"); private final String _diskCacheMode; @@ -179,13 +179,15 @@ public class ProviderAdapterDiskOffering { return _diskCacheMode; } - public static DiskCacheMode getDiskCasehMode(String cacheMode) { + public static DiskCacheMode getDiskCacheMode(String cacheMode) { if (cacheMode.equals(NONE._diskCacheMode)) { return NONE; } else if (cacheMode.equals(WRITEBACK._diskCacheMode)) { return WRITEBACK; } else if (cacheMode.equals(WRITETHROUGH._diskCacheMode)) { return WRITETHROUGH; + } else if (cacheMode.equals(HYPERVISOR_DEFAULT._diskCacheMode)) { + return HYPERVISOR_DEFAULT; } else { throw new NotImplementedException("Invalid cache mode specified: " + cacheMode); } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index b96de3c3501..6f9db1dcdbd 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -8440,7 +8440,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati !Enums.getIfPresent(DiskOffering.DiskCacheMode.class, cacheMode.toUpperCase()).isPresent()) { throw new InvalidParameterValueException(String.format("Invalid cache mode (%s). Please specify one of the following " + - "valid cache mode parameters: none, writeback or writethrough", cacheMode)); + "valid cache mode parameters: none, writeback, writethrough or hypervisor_default.", cacheMode)); } } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 3cd8e8335ba..c372076e0f3 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1211,6 +1211,7 @@ "label.hourly": "Hourly", "label.hypervisor": "Hypervisor", "label.hypervisor.capabilities": "Hypervisor Capabilities", +"label.hypervisor.default": "Hypervisor default", "label.hypervisor.type": "Hypervisor type", "label.hypervisors": "Hypervisors", "label.hypervisorsnapshotreserve": "Hypervisor Snapshot reserve", @@ -2813,7 +2814,7 @@ "label.windows": "Windows", "label.with.snapshotid": "with Snapshot ID", "label.write": "Write", -"label.writeback": "Write-back disk caching", +"label.writeback": "Write-back", "label.writecachetype": "Write-cache Type", "label.writeio": "Write (IO)", "label.writethrough": "Write-through", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 711dd36deff..c4b5add82fc 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -773,6 +773,7 @@ "label.hourly": "A cada hora", "label.hypervisor": "Virtualizador", "label.hypervisor.capabilities": "Recursos do virtualizador", +"label.hypervisor.default": "Padr\u00e3o do virtualizador", "label.hypervisor.type": "Tipo do virtualizador", "label.hypervisors": "Virtualizadores", "label.hypervisorsnapshotreserve": "Reserva de snapshot do virtualizador", @@ -1813,7 +1814,7 @@ "label.windows": "Windows", "label.with.snapshotid": "com o ID da snapshot", "label.write": "Escreva", -"label.writeback": "Cache de disco write-back", +"label.writeback": "Write-back", "label.writecachetype": "Tipo do cache de escrita", "label.writeio": "Escrita (IO)", "label.writethrough": "Write-through", diff --git a/ui/src/views/offering/AddComputeOffering.vue b/ui/src/views/offering/AddComputeOffering.vue index 0455551723c..dc4d5b18818 100644 --- a/ui/src/views/offering/AddComputeOffering.vue +++ b/ui/src/views/offering/AddComputeOffering.vue @@ -467,6 +467,9 @@ {{ $t('label.writethrough') }} + + {{ $t('label.hypervisor.default') }} + diff --git a/ui/src/views/offering/AddDiskOffering.vue b/ui/src/views/offering/AddDiskOffering.vue index 886465cfcf7..bfdd778d449 100644 --- a/ui/src/views/offering/AddDiskOffering.vue +++ b/ui/src/views/offering/AddDiskOffering.vue @@ -211,6 +211,9 @@ {{ $t('label.writethrough') }} + + {{ $t('label.hypervisor.default') }} + @@ -604,7 +607,7 @@ export default { width: 80vw; @media (min-width: 800px) { - width: 430px; + width: 480px; } } From b143ddc4058cbcba797aed30c40e2b9f51af6afc Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 8 Oct 2025 17:25:08 +0530 Subject: [PATCH 005/127] Sanitize the rbd file cmd parameter logs during qemu-img convert (through Script) (#11801) --- .../java/com/cloud/utils/script/Script.java | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/utils/src/main/java/com/cloud/utils/script/Script.java b/utils/src/main/java/com/cloud/utils/script/Script.java index fce4cf8e467..6c62c910648 100644 --- a/utils/src/main/java/com/cloud/utils/script/Script.java +++ b/utils/src/main/java/com/cloud/utils/script/Script.java @@ -29,6 +29,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Properties; import java.util.concurrent.Callable; @@ -157,13 +158,7 @@ public class Script implements Callable { boolean obscureParam = false; for (int i = 0; i < command.length; i++) { String cmd = command[i]; - if (StringUtils.isNotEmpty(cmd) && cmd.startsWith("vi://")) { - String[] tokens = cmd.split("@"); - if (tokens.length >= 2) { - builder.append("vi://").append("******@").append(tokens[1]).append(" "); - } else { - builder.append("vi://").append("******").append(" "); - } + if (sanitizeViCmdParameter(cmd, builder) || sanitizeRbdFileFormatCmdParameter(cmd, builder)) { continue; } if (obscureParam) { @@ -181,6 +176,41 @@ public class Script implements Callable { return builder.toString(); } + private boolean sanitizeViCmdParameter(String cmd, StringBuilder builder) { + if (StringUtils.isEmpty(cmd) || !cmd.startsWith("vi://")) { + return false; + } + + String[] tokens = cmd.split("@"); + if (tokens.length >= 2) { + builder.append("vi://").append("******@").append(tokens[1]).append(" "); + } else { + builder.append("vi://").append("******").append(" "); + } + return true; + } + + private boolean sanitizeRbdFileFormatCmdParameter(String cmd, StringBuilder builder) { + if (StringUtils.isEmpty(cmd) || !cmd.startsWith("rbd:") || !cmd.contains("key=")) { + return false; + } + + String[] tokens = cmd.split("key="); + if (tokens.length != 2) { + return false; + } + + String tokenWithKey = tokens[1]; + String[] options = tokenWithKey.split(":"); + if (options.length > 1) { + String optionsAfterKey = String.join(":", Arrays.copyOfRange(options, 1, options.length)); + builder.append(tokens[0]).append("key=").append("******").append(":").append(optionsAfterKey).append(" "); + } else { + builder.append(tokens[0]).append("key=").append("******").append(" "); + } + return true; + } + public long getTimeout() { return _timeout; } From 89d2b17461f72c09d28301d2995aa5ff1ba29489 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 8 Oct 2025 15:34:59 +0200 Subject: [PATCH 006/127] storage: change storage pool to Up state when cancel storage migration (#11773) * storage: change storage pool to Up state when cancel storage migration * Update 11773: connect host to shared pool after cancelling storage migration * Update 11773: update db only * Update 11773: skip capacity update for storpool --- .../src/main/java/com/cloud/storage/StorageManager.java | 2 ++ .../CloudStackPrimaryDataStoreLifeCycleImpl.java | 2 +- .../main/java/com/cloud/storage/StorageManagerImpl.java | 9 ++++++--- .../com/cloud/storage/StoragePoolAutomationImpl.java | 1 + 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 529e506e8a0..95e44bbb7b3 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -294,6 +294,8 @@ public interface StorageManager extends StorageService { Answer sendToPool(StoragePool pool, long[] hostIdsToTryFirst, Command cmd) throws StorageUnavailableException; + void updateStoragePoolHostVOAndBytes(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer); + CapacityVO getSecondaryStorageUsedStats(Long hostId, Long zoneId); CapacityVO getStoragePoolUsedStats(Long poolId, Long clusterId, Long podId, Long zoneId); diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index e5ff0ab5873..30996b0020a 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -448,8 +448,8 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl extends BasePrimaryDataStor @Override public boolean cancelMaintain(DataStore store) { - storagePoolAutmation.cancelMaintain(store); dataStoreHelper.cancelMaintain(store); + storagePoolAutmation.cancelMaintain(store); return true; } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 90113f66aaf..323c0eb3ee4 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -2688,7 +2688,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } - private void updateStoragePoolHostVOAndBytes(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) { + @Override + public void updateStoragePoolHostVOAndBytes(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) { StoragePoolHostVO poolHost = _storagePoolHostDao.findByPoolHost(pool.getId(), hostId); if (poolHost == null) { poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); @@ -2698,8 +2699,10 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId()); - poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes()); - poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); + if (!Storage.StoragePoolType.StorPool.equals(poolVO.getPoolType())) { + poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes()); + poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); + } _storagePoolDao.update(pool.getId(), poolVO); } diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 60494dcb05c..3ce23a0bd3b 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -346,6 +346,7 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { if (logger.isDebugEnabled()) { logger.debug("ModifyStoragePool add succeeded"); } + storageManager.updateStoragePoolHostVOAndBytes(pool, host.getId(), (ModifyStoragePoolAnswer) answer); if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { logger.debug("Started synchronising datastore cluster storage pool {} with vCenter", pool); storageManager.syncDatastoreClusterStoragePool(pool.getId(), ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren(), host.getId()); From 309b444205a0323f9608d85105f8b30c66613884 Mon Sep 17 00:00:00 2001 From: dahn Date: Thu, 9 Oct 2025 08:39:45 +0200 Subject: [PATCH 007/127] pom.xml: update jetty version (#11793) * update jetty * Rollback jetty-maven-plugin version in pom.xml Co-authored-by: Rohit Yadav --------- Co-authored-by: Daan Hoogland Co-authored-by: Wei Zhou Co-authored-by: Rohit Yadav --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 17385ab5a8c..b620a648e36 100644 --- a/pom.xml +++ b/pom.xml @@ -158,7 +158,7 @@ 2.3.3 2.3.7 2.26 - 9.4.51.v20230217 + 9.4.58.v20250814 9.4.27.v20200227 5.5.0 2.12.5 From a6ef24d1679812c223322b16767baf9faf269bd3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Oct 2025 13:06:28 +0530 Subject: [PATCH 008/127] server: consistent domainpath in api responses (#11589) * server: consistent domainpath in api responses Currently, some APIs return domainpath as 'ROOT/domain1/domain2' while other return it as '/domain1/domain2'. This PR makes the response consistent like "ROOT/domain1/domain2" Signed-off-by: Abhishek Kumar * more changes Signed-off-by: Abhishek Kumar --------- Signed-off-by: Abhishek Kumar --- .../api/command/user/vpn/AddVpnUserCmd.java | 18 +-- .../api/response/UserVmResponse.java | 2 +- .../cluster/KubernetesClusterManagerImpl.java | 15 +-- .../response/KubernetesClusterResponse.java | 2 +- .../java/com/cloud/api/ApiResponseHelper.java | 106 ++++++------------ .../api/query/dao/AccountJoinDaoImpl.java | 5 +- .../api/query/dao/AsyncJobJoinDaoImpl.java | 5 +- .../api/query/dao/DomainJoinDaoImpl.java | 5 +- .../query/dao/DomainRouterJoinDaoImpl.java | 4 - .../query/dao/ProjectAccountJoinDaoImpl.java | 3 +- .../dao/ProjectInvitationJoinDaoImpl.java | 3 +- .../api/query/dao/ResourceTagJoinDaoImpl.java | 4 - .../api/query/dao/TemplateJoinDaoImpl.java | 15 --- .../api/query/dao/UserVmJoinDaoImpl.java | 10 +- 14 files changed, 49 insertions(+), 148 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java index 59ba7e94b04..781edff52e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java @@ -28,7 +28,6 @@ import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.VpnUsersResponse; import org.apache.cloudstack.context.CallContext; -import com.cloud.domain.Domain; import com.cloud.event.EventTypes; import com.cloud.network.VpnUser; import com.cloud.user.Account; @@ -110,7 +109,6 @@ public class AddVpnUserCmd extends BaseAsyncCreateCmd { @Override public void execute() { VpnUser vpnUser = _entityMgr.findById(VpnUser.class, getEntityId()); - Account account = _entityMgr.findById(Account.class, vpnUser.getAccountId()); try { if (!_ravService.applyVpnUsers(vpnUser.getAccountId(), userName)) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add vpn user"); @@ -118,24 +116,10 @@ public class AddVpnUserCmd extends BaseAsyncCreateCmd { } catch (Exception ex) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } - - VpnUsersResponse vpnResponse = new VpnUsersResponse(); - vpnResponse.setId(vpnUser.getUuid()); - vpnResponse.setUserName(vpnUser.getUsername()); - vpnResponse.setAccountName(account.getAccountName()); // re-retrieve the vpnuser, as the call to `applyVpnUsers` might have changed the state vpnUser = _entityMgr.findById(VpnUser.class, getEntityId()); - vpnResponse.setState(vpnUser.getState().toString()); - - Domain domain = _entityMgr.findById(Domain.class, account.getDomainId()); - if (domain != null) { - vpnResponse.setDomainId(domain.getUuid()); - vpnResponse.setDomainName(domain.getName()); - vpnResponse.setDomainPath(domain.getPath()); - } - + VpnUsersResponse vpnResponse = _responseGenerator.createVpnUserResponse(vpnUser); vpnResponse.setResponseName(getCommandName()); - vpnResponse.setObjectName("vpnuser"); setResponseObject(vpnResponse); } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index ca5bd09a9aa..29681b5e38f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -41,7 +41,7 @@ import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") @EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class}) -public class UserVmResponse extends BaseResponseWithTagInformation implements ControlledEntityResponse, SetResourceIconResponse { +public class UserVmResponse extends BaseResponseWithTagInformation implements ControlledViewEntityResponse, SetResourceIconResponse { @SerializedName(ApiConstants.ID) @Param(description = "the ID of the virtual machine") private String id; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index c46b0c00fa1..a12648c8e84 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -118,6 +118,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import com.cloud.api.ApiDBUtils; +import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.dao.NetworkOfferingJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.dao.UserVmJoinDao; @@ -135,7 +136,6 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.deploy.DeployDestination; -import com.cloud.domain.Domain; import com.cloud.event.ActionEvent; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -814,18 +814,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne response.setKubernetesVersionId(version.getUuid()); response.setKubernetesVersionName(version.getName()); } - Account account = ApiDBUtils.findAccountById(kubernetesCluster.getAccountId()); - if (account.getType() == Account.Type.PROJECT) { - Project project = ApiDBUtils.findProjectByProjectAccountId(account.getId()); - response.setProjectId(project.getUuid()); - response.setProjectName(project.getName()); - } else { - response.setAccountName(account.getAccountName()); - } - Domain domain = ApiDBUtils.findDomainById(kubernetesCluster.getDomainId()); - response.setDomainId(domain.getUuid()); - response.setDomainName(domain.getName()); - response.setDomainPath(domain.getPath()); + ApiResponseHelper.populateOwner(response, kubernetesCluster); response.setKeypair(kubernetesCluster.getKeyPair()); response.setState(kubernetesCluster.getState().toString()); response.setCores(String.valueOf(kubernetesCluster.getCores())); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index b811f4f9dcb..b38fdcbc49e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -30,7 +30,7 @@ import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") @EntityReference(value = {KubernetesCluster.class}) -public class KubernetesClusterResponse extends BaseResponseWithAnnotations implements ControlledEntityResponse { +public class KubernetesClusterResponse extends BaseResponseWithAnnotations implements ControlledViewEntityResponse { @SerializedName(ApiConstants.ID) @Param(description = "the id of the Kubernetes cluster") private String id; diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index c44239b4447..cdb210c6dea 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -530,6 +530,15 @@ public class ApiResponseHelper implements ResponseGenerator { @Inject ResourceIconManager resourceIconManager; + public static String getPrettyDomainPath(String path) { + if (path == null) { + return null; + } + StringBuilder domainPath = new StringBuilder("ROOT"); + (domainPath.append(path)).deleteCharAt(domainPath.length() - 1); + return domainPath.toString(); + } + @Override public UserResponse createUserResponse(User user) { UserAccountJoinVO vUser = ApiDBUtils.newUserView(user); @@ -567,9 +576,7 @@ public class ApiResponseHelper implements ResponseGenerator { if (parentDomain != null) { domainResponse.setParentDomainId(parentDomain.getUuid()); } - StringBuilder domainPath = new StringBuilder("ROOT"); - (domainPath.append(domain.getPath())).deleteCharAt(domainPath.length() - 1); - domainResponse.setPath(domainPath.toString()); + domainResponse.setPath(getPrettyDomainPath(domain.getPath())); if (domain.getParent() != null) { domainResponse.setParentDomainName(ApiDBUtils.findDomainById(domain.getParent()).getName()); } @@ -822,21 +829,6 @@ public class ApiResponseHelper implements ResponseGenerator { } } populateOwner(vmSnapshotResponse, vmSnapshot); - Project project = ApiDBUtils.findProjectByProjectAccountId(vmSnapshot.getAccountId()); - if (project != null) { - vmSnapshotResponse.setProjectId(project.getUuid()); - vmSnapshotResponse.setProjectName(project.getName()); - } - Account account = ApiDBUtils.findAccountById(vmSnapshot.getAccountId()); - if (account != null) { - vmSnapshotResponse.setAccountName(account.getAccountName()); - } - DomainVO domain = ApiDBUtils.findDomainById(vmSnapshot.getDomainId()); - if (domain != null) { - vmSnapshotResponse.setDomainId(domain.getUuid()); - vmSnapshotResponse.setDomainName(domain.getName()); - vmSnapshotResponse.setDomainPath(domain.getPath()); - } List tags = _resourceTagDao.listBy(vmSnapshot.getId(), ResourceObjectType.VMSnapshot); List tagResponses = new ArrayList(); @@ -2350,18 +2342,7 @@ public class ApiResponseHelper implements ResponseGenerator { response.setName(securityGroup.getName()); response.setDescription(securityGroup.getDescription()); - Account account = securiytGroupAccounts.get(securityGroup.getAccountId()); - - if (securityGroup.getAccountType() == Account.Type.PROJECT) { - response.setProjectId(securityGroup.getProjectUuid()); - response.setProjectName(securityGroup.getProjectName()); - } else { - response.setAccountName(securityGroup.getAccountName()); - } - - response.setDomainId(securityGroup.getDomainUuid()); - response.setDomainName(securityGroup.getDomainName()); - response.setDomainPath(securityGroup.getDomainPath()); + populateOwner(response, securityGroup); for (SecurityRule securityRule : securityRules) { SecurityGroupRuleResponse securityGroupData = new SecurityGroupRuleResponse(); @@ -2758,32 +2739,18 @@ public class ApiResponseHelper implements ResponseGenerator { // get domain from network_domain table Pair domainNetworkDetails = ApiDBUtils.getDomainNetworkDetails(network.getId()); if (domainNetworkDetails.first() != null) { - Domain domain = ApiDBUtils.findDomainById(domainNetworkDetails.first()); - if (domain != null) { - response.setDomainId(domain.getUuid()); - - StringBuilder domainPath = new StringBuilder("ROOT"); - (domainPath.append(domain.getPath())).deleteCharAt(domainPath.length() - 1); - response.setDomainPath(domainPath.toString()); - } + populateDomain(response, domainNetworkDetails.first()); } response.setSubdomainAccess(domainNetworkDetails.second()); } Long dedicatedDomainId = ApiDBUtils.getDedicatedNetworkDomain(network.getId()); if (dedicatedDomainId != null) { - Domain domain = ApiDBUtils.findDomainById(dedicatedDomainId); - if (domain != null) { - response.setDomainId(domain.getUuid()); - response.setDomainName(domain.getName()); - response.setDomainPath(domain.getPath()); - } - + populateDomain(response, dedicatedDomainId); } response.setSpecifyIpRanges(network.getSpecifyIpRanges()); - setVpcIdInResponse(network.getVpcId(), response::setVpcId, response::setVpcName); setResponseAssociatedNetworkInformation(response, network.getId()); @@ -3045,14 +3012,10 @@ public class ApiResponseHelper implements ResponseGenerator { } else { response.setAccountName(account.getAccountName()); } - - Domain domain = ApiDBUtils.findDomainById(object.getDomainId()); - response.setDomainId(domain.getUuid()); - response.setDomainName(domain.getName()); - response.setDomainPath(domain.getPath()); + populateDomain(response, object.getDomainId()); } - private void populateOwner(ControlledViewEntityResponse response, ControlledEntity object) { + public static void populateOwner(ControlledViewEntityResponse response, ControlledEntity object) { Account account = ApiDBUtils.findAccountById(object.getAccountId()); if (account.getType() == Account.Type.PROJECT) { @@ -3064,10 +3027,7 @@ public class ApiResponseHelper implements ResponseGenerator { response.setAccountName(account.getAccountName()); } - Domain domain = ApiDBUtils.findDomainById(object.getDomainId()); - response.setDomainId(domain.getUuid()); - response.setDomainName(domain.getName()); - response.setDomainPath(domain.getPath()); + populateDomain(response, object.getDomainId()); } public static void populateOwner(ControlledViewEntityResponse response, ControlledViewEntity object) { @@ -3081,7 +3041,7 @@ public class ApiResponseHelper implements ResponseGenerator { response.setDomainId(object.getDomainUuid()); response.setDomainName(object.getDomainName()); - response.setDomainPath(object.getDomainPath()); + response.setDomainPath(getPrettyDomainPath(object.getDomainPath())); } private void populateAccount(ControlledEntityResponse response, long accountId) { @@ -3105,10 +3065,22 @@ public class ApiResponseHelper implements ResponseGenerator { private void populateDomain(ControlledEntityResponse response, long domainId) { Domain domain = ApiDBUtils.findDomainById(domainId); - + if (domain == null) { + return; + } response.setDomainId(domain.getUuid()); response.setDomainName(domain.getName()); - response.setDomainPath(domain.getPath()); + response.setDomainPath(getPrettyDomainPath(domain.getPath())); + } + + private static void populateDomain(ControlledViewEntityResponse response, long domainId) { + Domain domain = ApiDBUtils.findDomainById(domainId); + if (domain == null) { + return; + } + response.setDomainId(domain.getUuid()); + response.setDomainName(domain.getName()); + response.setDomainPath(getPrettyDomainPath(domain.getPath())); } @Override @@ -4100,12 +4072,7 @@ public class ApiResponseHelper implements ResponseGenerator { usageRecResponse.setAccountName(account.getAccountName()); } - Domain domain = ApiDBUtils.findDomainById(usageRecord.getDomainId()); - if (domain != null) { - usageRecResponse.setDomainId(domain.getUuid()); - usageRecResponse.setDomainName(domain.getName()); - usageRecResponse.setDomainPath(domain.getPath()); - } + populateDomain(usageRecResponse, account.getDomainId()); if (usageRecord.getZoneId() != null) { DataCenter zone = ApiDBUtils.findZoneById(usageRecord.getZoneId()); @@ -4892,18 +4859,11 @@ public class ApiResponseHelper implements ResponseGenerator { AffinityGroupResponse response = new AffinityGroupResponse(); - Account account = ApiDBUtils.findAccountById(group.getAccountId()); response.setId(group.getUuid()); - response.setAccountName(account.getAccountName()); response.setName(group.getName()); response.setType(group.getType()); response.setDescription(group.getDescription()); - Domain domain = ApiDBUtils.findDomainById(account.getDomainId()); - if (domain != null) { - response.setDomainId(domain.getUuid()); - response.setDomainName(domain.getName()); - response.setDomainPath(domain.getPath()); - } + populateOwner(response, group); response.setObjectName("affinitygroup"); return response; diff --git a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java index 9a301d440a9..de66b80ca55 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse; import org.apache.cloudstack.api.response.UserResponse; import com.cloud.api.ApiDBUtils; +import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.ViewResponseHelper; import com.cloud.api.query.vo.AccountJoinVO; import com.cloud.api.query.vo.UserAccountJoinVO; @@ -74,9 +75,7 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl accountResponse.setAccountType(account.getType().ordinal()); accountResponse.setDomainId(account.getDomainUuid()); accountResponse.setDomainName(account.getDomainName()); - StringBuilder domainPath = new StringBuilder("ROOT"); - (domainPath.append(account.getDomainPath())).deleteCharAt(domainPath.length() - 1); - accountResponse.setDomainPath(domainPath.toString()); + accountResponse.setDomainPath(ApiResponseHelper.getPrettyDomainPath(account.getDomainPath())); accountResponse.setState(account.getState().toString()); accountResponse.setCreated(account.getCreated()); accountResponse.setNetworkDomain(account.getNetworkDomain()); diff --git a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java index 08b896edb17..10ef67bbbea 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AsyncJobJoinDaoImpl.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.framework.jobs.AsyncJob; +import com.cloud.api.ApiResponseHelper; import com.cloud.api.ApiSerializerHelper; import com.cloud.api.SerializationContext; import com.cloud.api.query.vo.AsyncJobJoinVO; @@ -60,9 +61,7 @@ public class AsyncJobJoinDaoImpl extends GenericDaoBase im jobResponse.setAccountId(job.getAccountUuid()); jobResponse.setAccount(job.getAccountName()); jobResponse.setDomainId(job.getDomainUuid()); - StringBuilder domainPath = new StringBuilder("ROOT"); - (domainPath.append(job.getDomainPath())).deleteCharAt(domainPath.length() - 1); - jobResponse.setDomainPath(domainPath.toString()); + jobResponse.setDomainPath(ApiResponseHelper.getPrettyDomainPath(job.getDomainPath())); jobResponse.setUserId(job.getUserUuid()); jobResponse.setCmd(job.getCmd()); jobResponse.setCreated(job.getCreated()); diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java index 4a0929744cf..d4865c5550e 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java @@ -21,6 +21,7 @@ import java.util.EnumSet; import java.util.List; +import com.cloud.api.ApiResponseHelper; import com.cloud.configuration.Resource; import com.cloud.user.AccountManager; import org.apache.cloudstack.annotation.AnnotationService; @@ -79,9 +80,7 @@ public class DomainJoinDaoImpl extends GenericDaoBase implem if (domain.getParentUuid() != null) { domainResponse.setParentDomainId(domain.getParentUuid()); } - StringBuilder domainPath = new StringBuilder("ROOT"); - (domainPath.append(domain.getPath())).deleteCharAt(domainPath.length() - 1); - domainResponse.setPath(domainPath.toString()); + domainResponse.setPath(ApiResponseHelper.getPrettyDomainPath(domain.getPath())); if (domain.getParent() != null) { domainResponse.setParentDomainName(domain.getParentName()); } diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java index e7e2295923a..2c6c6481661 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java @@ -207,10 +207,6 @@ public class DomainRouterJoinDaoImpl extends GenericDaoBase details = new HashMap<>(); @@ -504,11 +494,6 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation Date: Thu, 9 Oct 2025 13:11:18 +0530 Subject: [PATCH 009/127] Delete template from storage pool instantly if no volume is using it (#11782) --- .../com/cloud/template/TemplateManager.java | 9 +++ .../cloud/storage/dao/VMTemplatePoolDao.java | 2 + .../storage/dao/VMTemplatePoolDaoImpl.java | 10 ++++ .../datastore/db/PrimaryDataStoreDao.java | 2 + .../datastore/db/PrimaryDataStoreDaoImpl.java | 14 +++++ .../template/HypervisorTemplateAdapter.java | 11 +++- .../cloud/template/TemplateManagerImpl.java | 55 +++++++++++++------ 7 files changed, 85 insertions(+), 18 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index b8912526fdf..a0f91882c4d 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -56,6 +56,13 @@ public interface TemplateManager { + "will validate if the provided URL is resolvable during the register of templates/ISOs before persisting them in the database.", true); + ConfigKey TemplateDeleteFromPrimaryStorage = new ConfigKey("Advanced", + Boolean.class, + "template.delete.from.primary.storage", "true", + "Template when deleted will be instantly deleted from the Primary Storage", + true, + ConfigKey.Scope.Global); + static final String VMWARE_TOOLS_ISO = "vmware-tools.iso"; static final String XS_TOOLS_ISO = "xs-tools.iso"; @@ -103,6 +110,8 @@ public interface TemplateManager { */ List getUnusedTemplatesInPool(StoragePoolVO pool); + void evictTemplateFromStoragePoolsForZones(Long templateId, List zoneId); + /** * Deletes a template in the specified storage pool. * diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java index a3ce03a74c3..a208590e23a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java @@ -35,6 +35,8 @@ public interface VMTemplatePoolDao extends GenericDao listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state); + List listByPoolIdsAndTemplate(List poolIds, Long templateId); + List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState); List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java index 5a2ec1163fb..5dfc138d8e1 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java @@ -150,6 +150,16 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase listByPoolIdsAndTemplate(List poolIds, Long templateId) { + SearchCriteria sc = PoolTemplateSearch.create(); + if (CollectionUtils.isNotEmpty(poolIds)) { + sc.setParameters("pool_id", poolIds.toArray()); + } + sc.setParameters("template_id", templateId); + return listBy(sc); + } + @Override public List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState) { SearchCriteria sc = TemplateStatusSearch.create(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index f0c235e842c..102df6848f8 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -154,4 +154,6 @@ public interface PrimaryDataStoreDao extends GenericDao { String keyword, Filter searchFilter); List listByIds(List ids); + + List listByDataCenterIds(List dataCenterIds); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index cb7313954dc..0d901c82306 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -63,6 +63,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase private final GenericSearchBuilder StatusCountSearch; private final SearchBuilder ClustersSearch; private final SearchBuilder IdsSearch; + private final SearchBuilder DcsSearch; @Inject private StoragePoolDetailsDao _detailsDao; @@ -155,6 +156,9 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase IdsSearch.and("ids", IdsSearch.entity().getId(), SearchCriteria.Op.IN); IdsSearch.done(); + DcsSearch = createSearchBuilder(); + DcsSearch.and("dataCenterId", DcsSearch.entity().getDataCenterId(), SearchCriteria.Op.IN); + DcsSearch.done(); } @Override @@ -733,6 +737,16 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase return listBy(sc); } + @Override + public List listByDataCenterIds(List dataCenterIds) { + if (CollectionUtils.isEmpty(dataCenterIds)) { + return Collections.emptyList(); + } + SearchCriteria sc = DcsSearch.create(); + sc.setParameters("dataCenterId", dataCenterIds.toArray()); + return listBy(sc); + } + private SearchCriteria createStoragePoolSearchCriteria(Long storagePoolId, String storagePoolName, Long zoneId, String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, String keyword) { diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index f7eb654141d..8d38aba0e7e 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -627,7 +627,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { boolean dataDiskDeletetionResult = true; List dataDiskTemplates = templateDao.listByParentTemplatetId(template.getId()); - if (dataDiskTemplates != null && dataDiskTemplates.size() > 0) { + if (CollectionUtils.isNotEmpty(dataDiskTemplates)) { logger.info("Template: {} has Datadisk template(s) associated with it. Delete Datadisk templates before deleting the template", template); for (VMTemplateVO dataDiskTemplate : dataDiskTemplates) { logger.info("Delete Datadisk template: {} from image store: {}", dataDiskTemplate, imageStore); @@ -693,6 +693,9 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { if (success) { if ((imageStores != null && imageStores.size() > 1) && (profile.getZoneIdList() != null)) { //if template is stored in more than one image stores, and the zone id is not null, then don't delete other templates. + if (templateMgr.TemplateDeleteFromPrimaryStorage.value()) { + templateMgr.evictTemplateFromStoragePoolsForZones(template.getId(), profile.getZoneIdList()); + } return cleanupTemplate(template, success); } @@ -705,7 +708,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { // find all eligible image stores for this template List iStores = templateMgr.getImageStoreByTemplate(template.getId(), null); - if (iStores == null || iStores.size() == 0) { + if (CollectionUtils.isEmpty(iStores)) { // remove any references from template_zone_ref List templateZones = templateZoneDao.listByTemplateId(template.getId()); if (templateZones != null) { @@ -726,6 +729,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } + if (templateMgr.TemplateDeleteFromPrimaryStorage.value()) { + templateMgr.evictTemplateFromStoragePoolsForZones(template.getId(), profile.getZoneIdList()); + } + // remove its related ACL permission Pair, Long> templateClassForId = new Pair<>(VirtualMachineTemplate.class, template.getId()); _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, templateClassForId); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index cf88ccec919..5aa31eff170 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1024,33 +1024,53 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return adapter.delete(new TemplateProfile(userId, template, zoneId)); } + private Boolean templateIsUnusedInPool(VMTemplateStoragePoolVO templatePoolVO) { + VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId()); + + // If this is a routing template, consider it in use + if (template.getTemplateType() == TemplateType.SYSTEM) { + return false; + } + + // If the template is not yet downloaded to the pool, consider it in use + if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) { + return false; + } + + if (template.getFormat() == ImageFormat.ISO || _volumeDao.isAnyVolumeActivelyUsingTemplateOnPool(template.getId(), templatePoolVO.getPoolId())) { + return false; + } + return true; + } + @Override public List getUnusedTemplatesInPool(StoragePoolVO pool) { List unusedTemplatesInPool = new ArrayList(); List allTemplatesInPool = _tmpltPoolDao.listByPoolId(pool.getId()); for (VMTemplateStoragePoolVO templatePoolVO : allTemplatesInPool) { - VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId()); - - // If this is a routing template, consider it in use - if (template.getTemplateType() == TemplateType.SYSTEM) { - continue; - } - - // If the template is not yet downloaded to the pool, consider it in - // use - if (templatePoolVO.getDownloadState() != Status.DOWNLOADED) { - continue; - } - - if (template.getFormat() != ImageFormat.ISO && !_volumeDao.isAnyVolumeActivelyUsingTemplateOnPool(template.getId(), pool.getId())) { + if (templateIsUnusedInPool(templatePoolVO)) { unusedTemplatesInPool.add(templatePoolVO); } } - return unusedTemplatesInPool; } + @Override + public void evictTemplateFromStoragePoolsForZones(Long templateId, List zoneIds) { + List poolIds = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(zoneIds)) { + List pools = _poolDao.listByDataCenterIds(zoneIds); + poolIds = pools.stream().map(StoragePoolVO::getId).collect(Collectors.toList()); + } + List templateStoragePoolVOS = _tmpltPoolDao.listByPoolIdsAndTemplate(poolIds, templateId); + for (VMTemplateStoragePoolVO templateStoragePoolVO: templateStoragePoolVOS) { + if (templateIsUnusedInPool(templateStoragePoolVO)) { + evictTemplateFromStoragePool(templateStoragePoolVO); + } + } + } + @Override @DB public void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO) { @@ -2368,7 +2388,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {AllowPublicUserTemplates, TemplatePreloaderPoolSize, ValidateUrlIsResolvableBeforeRegisteringTemplate}; + return new ConfigKey[] {AllowPublicUserTemplates, + TemplatePreloaderPoolSize, + ValidateUrlIsResolvableBeforeRegisteringTemplate, + TemplateDeleteFromPrimaryStorage}; } public List getTemplateAdapters() { From f67b738eb3d11b7ba6bdde67a32d0a56dc557775 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 9 Oct 2025 16:00:46 +0530 Subject: [PATCH 010/127] Migrate volume improvements, to bypass secondary storage when copy volume between pools is allowed directly (#11625) * Migrate volume improvements, to bypass secondary storage when copy volume between pools is allowed directly * Bypass secondary storage for copy volume between zone-wide pools and - local storage on host in the same zone - cluser-wide pools in the same zone * Bypass secondary storage for volumes on ceph/rdb pool when the scope permits * Fix dest disk format while migrating volume from ceph/rbd to nfs, and some code improvements * unit tests * Update suitable disk offering(s) for volume(s) after migrate VM with volumes when change in pool type (shared or local) Currently, Migrate VM with volume(s) bypasses the service and disk offerings of the volumes, as the target pools for migration are specified, which ignores the offerings. Offering change is required when pool type (shared or local) is changed, mainly - when volume on shared pool is migrated to local pool - when volume on local pool is migrated to shared pool * Update with proper message while migrate volume when target pool and offering type mismatches (both are not shared/local) * Consider host scope first during endpoint selection while copying between primary storages * Update disk offering count (for listDiskOfferings api) while removing offerings with tags mismatch with storage tags --- .../java/com/cloud/offering/DiskOffering.java | 5 +- .../com/cloud/storage/VolumeApiService.java | 2 + .../service/VolumeOrchestrationService.java | 5 +- .../subsystem/api/storage/ClusterScope.java | 6 + .../subsystem/api/storage/HostScope.java | 10 +- .../subsystem/api/storage/ZoneScope.java | 6 + .../cloud/vm/VirtualMachineManagerImpl.java | 1 + .../orchestration/VolumeOrchestrator.java | 9 +- .../orchestration/VolumeOrchestratorTest.java | 2 +- .../com/cloud/storage/DiskOfferingVO.java | 2 +- .../cloud/storage/dao/DiskOfferingDao.java | 2 + .../storage/dao/DiskOfferingDaoImpl.java | 16 ++ .../com/cloud/storage/dao/VolumeDaoImpl.java | 1 + .../motion/AncientDataMotionStrategy.java | 97 ++++++++- .../StorageSystemDataMotionStrategy.java | 34 ++- .../motion/AncientDataMotionStrategyTest.java | 200 ++++++++++++++++++ .../vmsnapshot/DefaultVMSnapshotStrategy.java | 1 + .../endpoint/DefaultEndPointSelector.java | 8 +- .../datastore/PrimaryDataStoreImpl.java | 1 + .../storage/volume/VolumeDataFactoryImpl.java | 8 + .../motion/HypervStorageMotionStrategy.java | 1 + .../kvm/storage/KVMStorageProcessor.java | 55 +++-- .../kvm/storage/LibvirtStorageAdaptor.java | 4 +- .../motion/VmwareStorageMotionStrategy.java | 1 + .../driver/AdaptiveDataStoreDriverImpl.java | 1 + .../CloudStackPrimaryDataStoreDriverImpl.java | 1 + .../LinstorPrimaryDataStoreDriverImpl.java | 1 + .../motion/StorPoolDataMotionStrategy.java | 1 + .../com/cloud/api/query/QueryManagerImpl.java | 1 + .../com/cloud/storage/StorageManagerImpl.java | 1 + .../cloud/storage/VolumeApiServiceImpl.java | 16 +- .../command/ReconcileCommandServiceImpl.java | 2 + .../VolumeImportUnmanageManagerImpl.java | 2 +- .../vm/UnmanagedVMsManagerImpl.java | 8 +- .../VolumeImportUnmanageManagerImplTest.java | 2 +- .../ReflectionToStringBuilderUtilsTest.java | 2 +- 36 files changed, 459 insertions(+), 56 deletions(-) diff --git a/api/src/main/java/com/cloud/offering/DiskOffering.java b/api/src/main/java/com/cloud/offering/DiskOffering.java index 9407acfdec1..d74f5703cc9 100644 --- a/api/src/main/java/com/cloud/offering/DiskOffering.java +++ b/api/src/main/java/com/cloud/offering/DiskOffering.java @@ -69,6 +69,8 @@ public interface DiskOffering extends InfrastructureEntity, Identity, InternalId boolean isCustomized(); + boolean isShared(); + void setDiskSize(long diskSize); long getDiskSize(); @@ -99,7 +101,6 @@ public interface DiskOffering extends InfrastructureEntity, Identity, InternalId Long getBytesReadRateMaxLength(); - void setBytesWriteRate(Long bytesWriteRate); Long getBytesWriteRate(); @@ -112,7 +113,6 @@ public interface DiskOffering extends InfrastructureEntity, Identity, InternalId Long getBytesWriteRateMaxLength(); - void setIopsReadRate(Long iopsReadRate); Long getIopsReadRate(); @@ -133,7 +133,6 @@ public interface DiskOffering extends InfrastructureEntity, Identity, InternalId Long getIopsWriteRateMax(); - void setIopsWriteRateMaxLength(Long iopsWriteRateMaxLength); Long getIopsWriteRateMaxLength(); diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 4140d51a800..19c2ebe455a 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -180,6 +180,8 @@ public interface VolumeApiService { */ boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags); + boolean validateConditionsToReplaceDiskOfferingOfVolume(Volume volume, DiskOffering newDiskOffering, StoragePool destPool); + Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge); void destroyVolume(long volumeId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index ccb5bba1c0a..168822c21eb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Set; import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.Storage; import com.cloud.utils.Pair; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -182,10 +183,10 @@ public interface VolumeOrchestrationService { */ DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, - Account owner, Long deviceId, Long poolId, String path, String chainInfo); + Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo); DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile); + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile); /** * Unmanage VM volumes diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ClusterScope.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ClusterScope.java index b0ed7d7d52b..68c7ed40d8f 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ClusterScope.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ClusterScope.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage; import com.cloud.storage.ScopeType; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; public class ClusterScope extends AbstractScope { private ScopeType type = ScopeType.CLUSTER; @@ -51,4 +52,9 @@ public class ClusterScope extends AbstractScope { return this.zoneId; } + @Override + public String toString() { + return String.format("ClusterScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "zoneId", "clusterId", "podId")); + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HostScope.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HostScope.java index 6e0bc618bfe..7e6ffe7d4f7 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HostScope.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/HostScope.java @@ -19,8 +19,10 @@ package org.apache.cloudstack.engine.subsystem.api.storage; import com.cloud.storage.ScopeType; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; public class HostScope extends AbstractScope { + private ScopeType type = ScopeType.HOST; private Long hostId; private Long clusterId; private Long zoneId; @@ -34,7 +36,7 @@ public class HostScope extends AbstractScope { @Override public ScopeType getScopeType() { - return ScopeType.HOST; + return this.type; } @Override @@ -49,4 +51,10 @@ public class HostScope extends AbstractScope { public Long getZoneId() { return zoneId; } + + @Override + public String toString() { + return String.format("HostScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "zoneId", "clusterId", "hostId")); + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ZoneScope.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ZoneScope.java index a0d75b5cc7c..fa704d05b1a 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ZoneScope.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ZoneScope.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage; import com.cloud.storage.ScopeType; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; public class ZoneScope extends AbstractScope { private ScopeType type = ScopeType.ZONE; @@ -39,4 +40,9 @@ public class ZoneScope extends AbstractScope { return this.zoneId; } + @Override + public String toString() { + return String.format("ZoneScope %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( + this, "zoneId")); + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index a8a92d6b3db..b3e672e2c17 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -2859,6 +2859,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } volume.setPath(result.getPath()); volume.setPoolId(pool.getId()); + volume.setPoolType(pool.getPoolType()); if (result.getChainInfo() != null) { volume.setChainInfo(result.getChainInfo()); } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 7af9b6b8492..2b759235ac8 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1423,7 +1423,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati String volumeToString = getVolumeIdentificationInfos(volume); VolumeInfo vol = volFactory.getVolume(volume.getId()); - if (vol == null){ + if (vol == null) { throw new CloudRuntimeException(String.format("Volume migration failed because volume [%s] is null.", volumeToString)); } if (destPool == null) { @@ -2308,6 +2308,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati StoragePoolVO pool = _storagePoolDao.findByUuid(updatedDataStoreUUID); if (pool != null) { vol.setPoolId(pool.getId()); + vol.setPoolType(pool.getPoolType()); } } _volsDao.update(volumeId, vol); @@ -2317,7 +2318,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @Override public DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId, Long poolId, String path, String chainInfo) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo) { if (sizeInBytes == null) { sizeInBytes = offering.getDiskSize(); } @@ -2358,6 +2359,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati vol.setFormat(getSupportedImageFormatForCluster(hypervisorType)); vol.setPoolId(poolId); + vol.setPoolType(poolType); vol.setPath(path); vol.setChainInfo(chainInfo); vol.setState(Volume.State.Ready); @@ -2367,7 +2369,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @Override public DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile) { VolumeVO vol = _volsDao.findById(diskProfile.getVolumeId()); if (vm != null) { @@ -2401,6 +2403,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType())); vol.setPoolId(poolId); + vol.setPoolType(poolType); vol.setPath(path); vol.setChainInfo(chainInfo); vol.setSize(diskProfile.getSize()); diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java index 817ff02ef74..b4a26c17e2e 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java @@ -241,7 +241,7 @@ public class VolumeOrchestratorTest { volumeOrchestrator.importVolume(volumeType, name, diskOffering, sizeInBytes, null, null, zoneId, hypervisorType, null, null, owner, - deviceId, poolId, path, chainInfo); + deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo); VolumeVO volume = volumeVOMockedConstructionConstruction.constructed().get(0); Mockito.verify(volume, Mockito.never()).setInstanceId(Mockito.anyLong()); diff --git a/engine/schema/src/main/java/com/cloud/storage/DiskOfferingVO.java b/engine/schema/src/main/java/com/cloud/storage/DiskOfferingVO.java index 79f5bcb5157..7f6b6d8adf0 100644 --- a/engine/schema/src/main/java/com/cloud/storage/DiskOfferingVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/DiskOfferingVO.java @@ -577,11 +577,11 @@ public class DiskOfferingVO implements DiskOffering { @Override public void setEncrypt(boolean encrypt) { this.encrypt = encrypt; } + @Override public boolean isShared() { return !useLocalStorage; } - public boolean getDiskSizeStrictness() { return diskSizeStrictness; } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java index f726bca3c5d..9beea003744 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDao.java @@ -31,6 +31,8 @@ public interface DiskOfferingDao extends GenericDao { List listAllBySizeAndProvisioningType(long size, Storage.ProvisioningType provisioningType); List findCustomDiskOfferings(); + List listByStorageTag(String tag); + List listAllActiveAndNonComputeDiskOfferings(); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java index 93e74766277..4ca3fe9f12a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/DiskOfferingDaoImpl.java @@ -26,6 +26,7 @@ import java.util.List; import javax.inject.Inject; import javax.persistence.EntityExistsException; +import com.cloud.offering.DiskOffering; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.springframework.stereotype.Component; @@ -45,6 +46,8 @@ public class DiskOfferingDaoImpl extends GenericDaoBase im protected DiskOfferingDetailsDao detailsDao; protected final SearchBuilder UniqueNameSearch; + protected final SearchBuilder ActiveAndNonComputeSearch; + private final String SizeDiskOfferingSearch = "SELECT * FROM disk_offering WHERE " + "disk_size = ? AND provisioning_type = ? AND removed IS NULL"; @@ -56,6 +59,11 @@ public class DiskOfferingDaoImpl extends GenericDaoBase im UniqueNameSearch.and("name", UniqueNameSearch.entity().getUniqueName(), SearchCriteria.Op.EQ); UniqueNameSearch.done(); + ActiveAndNonComputeSearch = createSearchBuilder(); + ActiveAndNonComputeSearch.and("state", ActiveAndNonComputeSearch.entity().getState(), SearchCriteria.Op.EQ); + ActiveAndNonComputeSearch.and("computeOnly", ActiveAndNonComputeSearch.entity().isComputeOnly(), SearchCriteria.Op.EQ); + ActiveAndNonComputeSearch.done(); + _computeOnlyAttr = _allAttributes.get("computeOnly"); } @@ -164,4 +172,12 @@ public class DiskOfferingDaoImpl extends GenericDaoBase im sc.setParameters("tagEndLike", "%," + tag); return listBy(sc); } + + @Override + public List listAllActiveAndNonComputeDiskOfferings() { + SearchCriteria sc = ActiveAndNonComputeSearch.create(); + sc.setParameters("state", DiskOffering.State.Active); + sc.setParameters("computeOnly", false); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index f8b6bb3ed68..6814636bc66 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -822,6 +822,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol if (volume.getState() != Volume.State.Destroy) { volume.setState(Volume.State.Destroy); volume.setPoolId(null); + volume.setPoolType(null); volume.setInstanceId(null); update(volume.getId(), volume); remove(volume.getId()); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 0b0065361d0..c494ca31594 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -18,9 +18,11 @@ */ package org.apache.cloudstack.storage.motion; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import javax.inject.Inject; @@ -67,6 +69,7 @@ import com.cloud.configuration.Config; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; import com.cloud.storage.Snapshot.Type; import com.cloud.storage.SnapshotVO; import com.cloud.storage.StorageManager; @@ -85,6 +88,11 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { protected Logger logger = LogManager.getLogger(getClass()); private static final String NO_REMOTE_ENDPOINT_SSVM = "No remote endpoint to send command, check if host or ssvm is down?"; private static final String NO_REMOTE_ENDPOINT_WITH_ENCRYPTION = "No remote endpoint to send command, unable to find a valid endpoint. Requires encryption support: %s"; + private static final List SUPPORTED_POOL_TYPES_TO_BYPASS_SECONDARY_STORE = Arrays.asList( + StoragePoolType.NetworkFilesystem, + StoragePoolType.Filesystem, + StoragePoolType.RBD + ); @Inject EndPointSelector selector; @@ -240,7 +248,6 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { return dataTO; } - protected Answer copyObject(DataObject srcData, DataObject destData) { return copyObject(srcData, destData, null); } @@ -352,14 +359,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { Scope destScope = getZoneScope(destData.getDataStore().getScope()); DataStore cacheStore = cacheMgr.getCacheStorage(destScope); - boolean bypassSecondaryStorage = false; - if (srcData instanceof VolumeInfo && ((VolumeInfo)srcData).isDirectDownload()) { - bypassSecondaryStorage = true; - } + boolean bypassSecondaryStorage = canBypassSecondaryStorage(srcData, destData); boolean encryptionRequired = anyVolumeRequiresEncryption(srcData, destData); if (cacheStore == null) { if (bypassSecondaryStorage) { + logger.debug("Secondary storage is bypassed, copy volume between pools directly"); CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value()); EndPoint ep = selector.select(srcData, destData, encryptionRequired); Answer answer = null; @@ -388,8 +393,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { answer = copyObject(srcData, objOnImageStore); if (answer == null || !answer.getResult()) { - if (answer != null) { - if (logger.isDebugEnabled()) logger.debug("copy to image store failed: " + answer.getDetails()); + if (answer != null && logger.isDebugEnabled()) { + logger.debug("copy to image store failed: {}", answer.getDetails()); } objOnImageStore.processEvent(Event.OperationFailed); imageStore.delete(objOnImageStore); @@ -411,8 +416,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } if (answer == null || !answer.getResult()) { - if (answer != null) { - if (logger.isDebugEnabled()) logger.debug("copy to primary store failed: " + answer.getDetails()); + if (answer != null && logger.isDebugEnabled()) { + logger.debug("copy to primary store failed: {}", answer.getDetails()); } objOnImageStore.processEvent(Event.OperationFailed); imageStore.delete(objOnImageStore); @@ -423,7 +428,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { objOnImageStore.processEvent(Event.OperationFailed); imageStore.delete(objOnImageStore); } - logger.error("Failed to perform operation: "+ e.getLocalizedMessage()); + logger.error("Failed to perform operation: {}", e.getLocalizedMessage()); throw e; } @@ -448,7 +453,78 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } return answer; } + } + private boolean canBypassSecondaryStorage(DataObject srcData, DataObject destData) { + if (srcData instanceof VolumeInfo) { + if (((VolumeInfo)srcData).isDirectDownload()) { + return true; + } + + if (destData instanceof VolumeInfo) { + Scope srcDataStoreScope = srcData.getDataStore().getScope(); + Scope destDataStoreScope = destData.getDataStore().getScope(); + logger.info("srcDataStoreScope: {}, srcData pool type: {}; destDataStoreScope: {}, destData pool type: {}", + srcDataStoreScope, ((VolumeInfo)srcData).getStoragePoolType(), destDataStoreScope, ((VolumeInfo)destData).getStoragePoolType()); + + if (srcDataStoreScope != null && destDataStoreScope != null && + SUPPORTED_POOL_TYPES_TO_BYPASS_SECONDARY_STORE.contains(((VolumeInfo)srcData).getStoragePoolType()) && + SUPPORTED_POOL_TYPES_TO_BYPASS_SECONDARY_STORE.contains(((VolumeInfo)destData).getStoragePoolType())) { + + return canDirectlyCopyBetweenDataStoreScopes(srcDataStoreScope, destDataStoreScope); + } + } + } + + return false; + } + + private boolean canDirectlyCopyBetweenDataStoreScopes(Scope srcDataStoreScope, Scope destDataStoreScope) { + if (srcDataStoreScope == null || destDataStoreScope == null) { + return false; + } + + if (srcDataStoreScope.isSameScope(destDataStoreScope)) { + return true; + } + + if (srcDataStoreScope.getScopeType() == ScopeType.HOST) { + if (destDataStoreScope.getScopeType() == ScopeType.CLUSTER && + (Objects.equals(((HostScope) srcDataStoreScope).getClusterId(), ((ClusterScope) destDataStoreScope).getScopeId()))) { + return true; + } + if (destDataStoreScope.getScopeType() == ScopeType.ZONE && + (Objects.equals(((HostScope) srcDataStoreScope).getZoneId(), ((ZoneScope) destDataStoreScope).getScopeId()))) { + return true; + } + } + + if (destDataStoreScope.getScopeType() == ScopeType.HOST) { + if (srcDataStoreScope.getScopeType() == ScopeType.CLUSTER && + (Objects.equals(((ClusterScope) srcDataStoreScope).getScopeId(), ((HostScope) destDataStoreScope).getClusterId()))) { + return true; + } + if (srcDataStoreScope.getScopeType() == ScopeType.ZONE && + (Objects.equals(((ZoneScope) srcDataStoreScope).getScopeId(), ((HostScope) destDataStoreScope).getZoneId()))) { + return true; + } + } + + if (srcDataStoreScope.getScopeType() == ScopeType.CLUSTER) { + if (destDataStoreScope.getScopeType() == ScopeType.ZONE && + (Objects.equals(((ClusterScope) srcDataStoreScope).getZoneId(), ((ZoneScope) destDataStoreScope).getScopeId()))) { + return true; + } + } + + if (destDataStoreScope.getScopeType() == ScopeType.CLUSTER) { + if (srcDataStoreScope.getScopeType() == ScopeType.ZONE && + (Objects.equals(((ZoneScope) srcDataStoreScope).getScopeId(), ((ClusterScope) destDataStoreScope).getZoneId()))) { + return true; + } + } + + return false; } protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) { @@ -492,6 +568,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } volumeVo.setPodId(destPool.getPodId()); volumeVo.setPoolId(destPool.getId()); + volumeVo.setPoolType(destPool.getPoolType()); volumeVo.setLastPoolId(oldPoolId); // For SMB, pool credentials are also stored in the uri query string. We trim the query string // part here to make sure the credentials do not get stored in the db unencrypted. diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 0a211ab1934..fa8a7a79aa3 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -122,6 +122,7 @@ import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; @@ -194,6 +195,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { @Inject private VolumeService _volumeService; @Inject + public VolumeApiService _volumeApiService; + @Inject private StorageCacheManager cacheMgr; @Inject private EndPointSelector selector; @@ -796,6 +799,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { volumeVO.setPodId(destPool.getPodId()); volumeVO.setPoolId(destPool.getId()); + volumeVO.setPoolType(destPool.getPoolType()); volumeVO.setLastPoolId(srcVolumeInfo.getPoolId()); _volumeDao.update(srcVolumeInfo.getId(), volumeVO); @@ -2348,11 +2352,22 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { volumeVO.setFormat(ImageFormat.QCOW2); volumeVO.setLastId(srcVolumeInfo.getId()); + if (Objects.equals(srcVolumeInfo.getDiskOfferingId(), destVolumeInfo.getDiskOfferingId())) { + StoragePoolVO srcPoolVO = _storagePoolDao.findById(srcVolumeInfo.getPoolId()); + StoragePoolVO destPoolVO = _storagePoolDao.findById(destVolumeInfo.getPoolId()); + if (srcPoolVO != null && destPoolVO != null && + ((srcPoolVO.isShared() && destPoolVO.isLocal()) || (srcPoolVO.isLocal() && destPoolVO.isShared()))) { + Long offeringId = getSuitableDiskOfferingForVolumeOnPool(volumeVO, destPoolVO); + if (offeringId != null) { + volumeVO.setDiskOfferingId(offeringId); + } + } + } + _volumeDao.update(volumeVO.getId(), volumeVO); _volumeService.copyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigration(Event.OperationSuccessed, null, srcVolumeInfo, destVolumeInfo, false); - // Update the volume ID for snapshots on secondary storage if (!_snapshotDao.listByVolumeId(srcVolumeInfo.getId()).isEmpty()) { _snapshotDao.updateVolumeIds(srcVolumeInfo.getId(), destVolumeInfo.getId()); @@ -2394,17 +2409,32 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } } + private Long getSuitableDiskOfferingForVolumeOnPool(VolumeVO volume, StoragePoolVO pool) { + List diskOfferings = _diskOfferingDao.listAllActiveAndNonComputeDiskOfferings(); + for (DiskOfferingVO diskOffering : diskOfferings) { + try { + if (_volumeApiService.validateConditionsToReplaceDiskOfferingOfVolume(volume, diskOffering, pool)) { + logger.debug("Found suitable disk offering {} for the volume {}", diskOffering, volume); + return diskOffering.getId(); + } + } catch (Exception ignore) { + } + } + logger.warn("Unable to find suitable disk offering for the volume {}", volume); + return null; + } + private VolumeVO duplicateVolumeOnAnotherStorage(Volume volume, StoragePoolVO storagePoolVO) { Long lastPoolId = volume.getPoolId(); VolumeVO newVol = new VolumeVO(volume); - newVol.setInstanceId(null); newVol.setChainInfo(null); newVol.setPath(null); newVol.setFolder(null); newVol.setPodId(storagePoolVO.getPodId()); newVol.setPoolId(storagePoolVO.getId()); + newVol.setPoolType(storagePoolVO.getPoolType()); newVol.setLastPoolId(lastPoolId); newVol.setLastId(volume.getId()); diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java index 67e3ea844d5..e167cc0a965 100755 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java @@ -21,20 +21,34 @@ package org.apache.cloudstack.storage.motion; import static org.mockito.Mockito.when; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.never; import static org.mockito.Mockito.any; +import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.volume.VolumeObject; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -57,6 +71,10 @@ public class AncientDataMotionStrategyTest { StorageManager storageManager; @Mock StoragePool storagePool; + @Mock + StorageCacheManager cacheMgr; + @Mock + ConfigurationDao configDao; private static final long POOL_ID = 1l; private static final Boolean FULL_CLONE_FLAG = true; @@ -88,4 +106,186 @@ public class AncientDataMotionStrategyTest { verify(dataStoreTO, never()).setFullCloneFlag(any(Boolean.class)); } + @Test + public void testCanBypassSecondaryStorageForDirectDownload() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(true).when(srcVolumeInfo).isDirectDownload(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageForUnsupportedDataObject() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + + TemplateObject destTemplateInfo = Mockito.spy(new TemplateObject()); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destTemplateInfo); + Assert.assertFalse(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageForUnsupportedSrcPoolType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore srcDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope(); + Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.PowerFlex).when(srcVolumeInfo).getStoragePoolType(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore destDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); + Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(destVolumeInfo).getStoragePoolType(); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertFalse(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageForUnsupportedDestPoolType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore srcDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope(); + Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore destDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); + Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.Iscsi).when(destVolumeInfo).getStoragePoolType(); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertFalse(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageWithZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore srcDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope(); + Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore destDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); + Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(destVolumeInfo).getStoragePoolType(); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageWithClusterWideNFSPoolsInSameCluster() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore srcDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(srcDataStore).getScope(); + Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore destDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(destDataStore).getScope(); + Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(destVolumeInfo).getStoragePoolType(); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageWithLocalAndClusterWideNFSPoolsInSameCluster() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore srcDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new HostScope(1L, 1L, 1L)).when(srcDataStore).getScope(); + Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.Filesystem).when(srcVolumeInfo).getStoragePoolType(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore destDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ClusterScope(1L, 1L, 1L)).when(destDataStore).getScope(); + Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(destVolumeInfo).getStoragePoolType(); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + + canBypassSecondaryStorage = (boolean) method.invoke(strategy, destVolumeInfo, srcVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageWithLocalAndZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore srcDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new HostScope(1L, 1L, 1L)).when(srcDataStore).getScope(); + Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.Filesystem).when(srcVolumeInfo).getStoragePoolType(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore destDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); + Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(destVolumeInfo).getStoragePoolType(); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + + canBypassSecondaryStorage = (boolean) method.invoke(strategy, destVolumeInfo, srcVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + } + + @Test + public void testCanBypassSecondaryStorageWithClusterWideNFSAndZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { + VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore srcDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(srcDataStore).getScope(); + Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); + + VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + DataStore destDataStore = Mockito.mock(DataStore.class); + Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); + Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); + Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(destVolumeInfo).getStoragePoolType(); + + Method method; + method = AncientDataMotionStrategy.class.getDeclaredMethod("canBypassSecondaryStorage", DataObject.class, DataObject.class); + method.setAccessible(true); + boolean canBypassSecondaryStorage = (boolean) method.invoke(strategy, srcVolumeInfo, destVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + + canBypassSecondaryStorage = (boolean) method.invoke(strategy, destVolumeInfo, srcVolumeInfo); + Assert.assertTrue(canBypassSecondaryStorage); + } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java index a19fcb9eda8..6e933396871 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java @@ -345,6 +345,7 @@ public class DefaultVMSnapshotStrategy extends ManagerBase implements VMSnapshot StoragePool pool = primaryDataStoreDao.findPoolByUUID(volume.getDataStoreUuid()); if (pool != null && pool.getId() != volumeVO.getPoolId()) { volumeVO.setPoolId(pool.getId()); + volumeVO.setPoolType(pool.getPoolType()); } } if (StringUtils.isNotEmpty(volume.getPath())) { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index da6b1cfede3..061d18dc376 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -232,7 +232,13 @@ public class DefaultEndPointSelector implements EndPointSelector { // assumption, at least one of scope should be zone, find the least // scope - if (srcScope.getScopeType() != ScopeType.ZONE) { + if (srcScope.getScopeType() == ScopeType.HOST) { + selectedScope = srcScope; + poolId = srcStore.getId(); + } else if (destScope.getScopeType() == ScopeType.HOST) { + selectedScope = destScope; + poolId = destStore.getId(); + } else if (srcScope.getScopeType() != ScopeType.ZONE) { selectedScope = srcScope; poolId = srcStore.getId(); } else if (destScope.getScopeType() != ScopeType.ZONE) { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 6a10c26cc0b..d0184359c8b 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -334,6 +334,7 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { VolumeVO vol = volumeDao.findById(obj.getId()); if (vol != null) { vol.setPoolId(getId()); + vol.setPoolType(getPoolType()); volumeDao.update(vol.getId(), vol); } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 53fa21f3a79..5c2a774f8a2 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -30,6 +30,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; @@ -46,6 +48,8 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { DataStoreManager storeMgr; @Inject VMTemplateDao templateDao; + @Inject + PrimaryDataStoreDao storagePoolDao; @Override public VolumeInfo getVolume(long volumeId, DataStore store) { @@ -92,6 +96,10 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { vol = VolumeObject.getVolumeObject(store, volumeVO); } else { DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary); + StoragePoolVO pool = storagePoolDao.findById(volumeVO.getPoolId()); + if (pool != null) { + volumeVO.setPoolType(pool.getPoolType()); + } vol = VolumeObject.getVolumeObject(store, volumeVO); } if (vol.getTemplateId() != null) { diff --git a/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java b/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java index 0e189d05000..55944a08242 100644 --- a/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java +++ b/plugins/hypervisors/hyperv/src/main/java/org/apache/cloudstack/storage/motion/HypervStorageMotionStrategy.java @@ -155,6 +155,7 @@ public class HypervStorageMotionStrategy implements DataMotionStrategy { volumeVO.setPath(volumeTo.getPath()); volumeVO.setPodId(pool.getPodId()); volumeVO.setPoolId(pool.getId()); + volumeVO.setPoolType(pool.getPoolType()); volumeVO.setLastPoolId(oldPoolId); // For SMB, pool credentials are also stored in the uri query string. We trim the query string // part here to make sure the credentials do not get stored in the db unencrypted. diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 0bb621af171..7b56ba47db0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -364,16 +364,7 @@ public class KVMStorageProcessor implements StorageProcessor { final TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(primaryVol.getName()); newTemplate.setSize(primaryVol.getSize()); - - if(List.of( - StoragePoolType.RBD, - StoragePoolType.PowerFlex, - StoragePoolType.Linstor, - StoragePoolType.FiberChannel).contains(primaryPool.getType())) { - newTemplate.setFormat(ImageFormat.RAW); - } else { - newTemplate.setFormat(ImageFormat.QCOW2); - } + newTemplate.setFormat(getFormat(primaryPool.getType())); data = newTemplate; } else if (destData.getObjectType() == DataObjectType.VOLUME) { final VolumeObjectTO volumeObjectTO = new VolumeObjectTO(); @@ -2990,7 +2981,7 @@ public class KVMStorageProcessor implements StorageProcessor { final VolumeObjectTO srcVol = (VolumeObjectTO)srcData; final VolumeObjectTO destVol = (VolumeObjectTO)destData; final ImageFormat srcFormat = srcVol.getFormat(); - final ImageFormat destFormat = destVol.getFormat(); + ImageFormat destFormat = destVol.getFormat(); final DataStoreTO srcStore = srcData.getDataStore(); final DataStoreTO destStore = destData.getDataStore(); final PrimaryDataStoreTO srcPrimaryStore = (PrimaryDataStoreTO)srcStore; @@ -3025,33 +3016,35 @@ public class KVMStorageProcessor implements StorageProcessor { volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString())); volume.setDispName(srcVol.getName()); volume.setVmName(srcVol.getVmName()); - - String destVolumeName = null; + KVMPhysicalDisk newVolume; + String destVolumeName; + destPool = storagePoolMgr.getStoragePool(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid()); if (destPrimaryStore.isManaged()) { if (!storagePoolMgr.connectPhysicalDisk(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid(), destVolumePath, destPrimaryStore.getDetails())) { logger.warn("Failed to connect dest volume {}, in storage pool {}", destVol, destPrimaryStore); } destVolumeName = derivePath(destPrimaryStore, destData, destPrimaryStore.getDetails()); } else { + PhysicalDiskFormat destPoolDefaultFormat = destPool.getDefaultFormat(); + destFormat = getFormat(destPoolDefaultFormat); final String volumeName = UUID.randomUUID().toString(); destVolumeName = volumeName + "." + destFormat.getFileExtension(); // Update path in the command for reconciliation - if (destData.getPath() == null) { + if (StringUtils.isBlank(destVolumePath)) { ((VolumeObjectTO) destData).setPath(destVolumeName); } } - destPool = storagePoolMgr.getStoragePool(destPrimaryStore.getPoolType(), destPrimaryStore.getUuid()); try { Volume.Type volumeType = srcVol.getVolumeType(); resource.createOrUpdateLogFileForCommand(cmd, Command.State.PROCESSING_IN_BACKEND); if (srcVol.getPassphrase() != null && (Volume.Type.ROOT.equals(volumeType) || Volume.Type.DATADISK.equals(volumeType))) { volume.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); - storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds(), srcVol.getPassphrase(), destVol.getPassphrase(), srcVol.getProvisioningType()); + newVolume = storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds(), srcVol.getPassphrase(), destVol.getPassphrase(), srcVol.getProvisioningType()); } else { - storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); + newVolume = storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds()); } resource.createOrUpdateLogFileForCommand(cmd, Command.State.COMPLETED); } catch (Exception e) { // Any exceptions while copying the disk, should send failed answer with the error message @@ -3071,9 +3064,13 @@ public class KVMStorageProcessor implements StorageProcessor { } final VolumeObjectTO newVol = new VolumeObjectTO(); - String path = destPrimaryStore.isManaged() ? destVolumeName : destVolumePath + File.separator + destVolumeName; + String path = destVolumeName; + if (!destPrimaryStore.isManaged() && StringUtils.isNotBlank(destVolumePath)) { + path = destVolumePath + File.separator + destVolumeName; + } newVol.setPath(path); - newVol.setFormat(destFormat); + ImageFormat newVolumeFormat = getFormat(newVolume.getFormat()); + newVol.setFormat(newVolumeFormat); newVol.setEncryptFormat(destVol.getEncryptFormat()); return new CopyCmdAnswer(newVol); } catch (final CloudRuntimeException e) { @@ -3085,6 +3082,26 @@ public class KVMStorageProcessor implements StorageProcessor { } } + private Storage.ImageFormat getFormat(PhysicalDiskFormat format) { + if (format == null) { + return null; + } + + return ImageFormat.valueOf(format.toString().toUpperCase()); + } + + private Storage.ImageFormat getFormat(StoragePoolType poolType) { + if(List.of( + StoragePoolType.RBD, + StoragePoolType.PowerFlex, + StoragePoolType.Linstor, + StoragePoolType.FiberChannel).contains(poolType)) { + return ImageFormat.RAW; + } else { + return ImageFormat.QCOW2; + } + } + /** * True if location exists */ diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index d5119ea55b7..02e5c08fa33 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -1116,7 +1116,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { // make room for encryption header on raw format, use LUKS if (format == PhysicalDiskFormat.RAW) { - destFile.setSize(destFile.getSize() - (16<<20)); + destFile.setSize(destFile.getSize() - (16 << 20)); destFile.setFormat(PhysicalDiskFormat.LUKS); } @@ -1593,7 +1593,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { String sourcePath = disk.getPath(); KVMPhysicalDisk newDisk; - logger.debug("copyPhysicalDisk: disk size:" + toHumanReadableSize(disk.getSize()) + ", virtualsize:" + toHumanReadableSize(disk.getVirtualSize())+" format:"+disk.getFormat()); + logger.debug("copyPhysicalDisk: disk size:{}, virtualsize:{} format:{}", toHumanReadableSize(disk.getSize()), toHumanReadableSize(disk.getVirtualSize()), disk.getFormat()); if (destPool.getType() != StoragePoolType.RBD) { if (disk.getFormat() == PhysicalDiskFormat.TAR) { newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, Storage.ProvisioningType.THIN, disk.getVirtualSize(), null); diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java index b0cacf60a17..d2d319ed9d0 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java @@ -433,6 +433,7 @@ public class VmwareStorageMotionStrategy implements DataMotionStrategy { volumeVO.setFolder(pool.getPath()); volumeVO.setPodId(pool.getPodId()); volumeVO.setPoolId(pool.getId()); + volumeVO.setPoolType(pool.getPoolType()); volDao.update(volume.getId(), volumeVO); updated = true; break; diff --git a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/driver/AdaptiveDataStoreDriverImpl.java b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/driver/AdaptiveDataStoreDriverImpl.java index 40d99526394..2ccd2bab6cd 100644 --- a/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/driver/AdaptiveDataStoreDriverImpl.java +++ b/plugins/storage/volume/adaptive/src/main/java/org/apache/cloudstack/storage/datastore/driver/AdaptiveDataStoreDriverImpl.java @@ -857,6 +857,7 @@ public class AdaptiveDataStoreDriverImpl extends CloudStackPrimaryDataStoreDrive volumeVO.setPath(finalPath); volumeVO.setFormat(ImageFormat.RAW); volumeVO.setPoolId(storagePool.getId()); + volumeVO.setPoolType(storagePool.getPoolType()); volumeVO.setExternalUuid(managedVolume.getExternalUuid()); volumeVO.setDisplay(true); volumeVO.setDisplayVolume(true); diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java index d2f0076f95f..e407bc6c2f3 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackPrimaryDataStoreDriverImpl.java @@ -505,6 +505,7 @@ public class CloudStackPrimaryDataStoreDriverImpl implements PrimaryDataStoreDri StoragePoolVO storagePoolVO = primaryStoreDao.findByUuid(datastoreUUID); if (storagePoolVO != null) { volumeVO.setPoolId(storagePoolVO.getId()); + volumeVO.setPoolType(storagePoolVO.getPoolType()); } else { logger.warn("Unable to find datastore {} while updating the new datastore of the volume {}", datastoreUUID, vol); } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 306e9259936..d3b797e319f 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -868,6 +868,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver devPath = createVolume(volumeInfo, storagePool); volume.setFolder("/dev/"); volume.setPoolId(storagePool.getId()); + volume.setPoolType(storagePool.getPoolType()); volume.setUuid(vol.getUuid()); volume.setPath(vol.getUuid()); diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java index f260c566986..5789ac50871 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java @@ -425,6 +425,7 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy { newVol.setFolder(null); newVol.setPodId(storagePoolVO.getPodId()); newVol.setPoolId(storagePoolVO.getId()); + newVol.setPoolType(storagePoolVO.getPoolType()); newVol.setLastPoolId(lastPoolId); return _volumeDao.persist(newVol); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index f51aca02af0..ef7d8505420 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -3675,6 +3675,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q String[] offeringTagsArray = (offeringTags == null || offeringTags.isEmpty()) ? new String[0] : offeringTags.split(","); if (!CollectionUtils.isSubCollection(Arrays.asList(requiredTagsArray), Arrays.asList(offeringTagsArray))) { iteratorForTagsChecking.remove(); + count--; } } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index abaf0e3d0e5..77d947b2b3b 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3049,6 +3049,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C StoragePoolVO storagePoolVO = _storagePoolDao.findByUuid(datastoreName); if (storagePoolVO != null) { volumeVO.setPoolId(storagePoolVO.getId()); + volumeVO.setPoolType(storagePoolVO.getPoolType()); } else { logger.warn("Unable to find datastore {} while updating the new datastore of the volume {}", datastoreName, volumeVO); } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index f4536e8549f..de6dd4ec67f 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2937,8 +2937,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(storageId); Collections.shuffle(childDatastores); volume.setPoolId(childDatastores.get(0).getId()); + volume.setPoolType(childDatastores.get(0).getPoolType()); } else { volume.setPoolId(pool.getId()); + volume.setPoolType(pool.getPoolType()); } } @@ -3225,6 +3227,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (storagePoolVO != null) { VolumeVO volumeVO = _volsDao.findById(volumeId); volumeVO.setPoolId(storagePoolVO.getId()); + volumeVO.setPoolType(storagePoolVO.getPoolType()); _volsDao.update(volumeVO.getId(), volumeVO); } else { logger.warn("Unable to find datastore {} while updating the new datastore of the volume {}", datastoreName, volume); @@ -3645,12 +3648,16 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic * * If all of the above validations pass, we check if the size of the new disk offering is different from the volume. If it is, we log a warning message. */ - protected void validateConditionsToReplaceDiskOfferingOfVolume(VolumeVO volume, DiskOfferingVO newDiskOffering, StoragePool destPool) { + @Override + public boolean validateConditionsToReplaceDiskOfferingOfVolume(Volume volume, DiskOffering newDiskOffering, StoragePool destPool) { if (newDiskOffering == null) { - return; + return false; } - if ((destPool.isShared() && newDiskOffering.isUseLocalStorage()) || destPool.isLocal() && newDiskOffering.isShared()) { - throw new InvalidParameterValueException("You cannot move the volume to a shared storage and assign a disk offering for local storage and vice versa."); + if (destPool.isShared() && newDiskOffering.isUseLocalStorage()) { + throw new InvalidParameterValueException("You cannot move the volume to shared storage, with the disk offering configured for local storage."); + } + if (destPool.isLocal() && newDiskOffering.isShared()) { + throw new InvalidParameterValueException("You cannot move the volume to local storage, with the disk offering configured for shared storage."); } if (!doesStoragePoolSupportDiskOffering(destPool, newDiskOffering)) { throw new InvalidParameterValueException(String.format("Migration failed: target pool [%s, tags:%s] has no matching tags for volume [%s, uuid:%s, tags:%s]", destPool.getName(), @@ -3675,6 +3682,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic volume, oldDiskOffering, newDiskOffering); } logger.info("Changing disk offering to [{}] while migrating volume [{}].", newDiskOffering, volume); + return true; } /** diff --git a/server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java b/server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java index d0dcc1f86de..5edf05d1a5c 100644 --- a/server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/command/ReconcileCommandServiceImpl.java @@ -1064,6 +1064,7 @@ public class ReconcileCommandServiceImpl extends ManagerBase implements Reconcil logger.debug(String.format("Updating volume %s to %s state", sourceVolume, Volume.State.Ready)); sourceVolume.setState(Volume.State.Ready); sourceVolume.setPoolId(srcDataStore.getId()); // restore pool_id and update path + sourceVolume.setPoolType(srcDataStore.getPoolType()); sourceVolume.setPath(srcData.getPath()); sourceVolume.set_iScsiName(srcData.getPath()); sourceVolume.setUpdated(new Date()); @@ -1075,6 +1076,7 @@ public class ReconcileCommandServiceImpl extends ManagerBase implements Reconcil VolumeVO newVolume = (VolumeVO) newVol; newVolume.setInstanceId(null); newVolume.setPoolId(destDataStore.getId()); + newVolume.setPoolType(destDataStore.getPoolType()); newVolume.setState(Volume.State.Creating); newVolume.setPath(destData.getPath()); newVolume.set_iScsiName(destData.getPath()); diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 9e1fc46e02e..383644f9aa2 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -452,7 +452,7 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ Account owner, StoragePoolVO pool, String volumeName) { DiskProfile diskProfile = volumeManager.importVolume(Volume.Type.DATADISK, volumeName, diskOffering, volume.getVirtualSize(), null, null, pool.getDataCenterId(), volume.getHypervisorType(), null, null, - owner, null, pool.getId(), volume.getPath(), null); + owner, null, pool.getId(), pool.getPoolType(), volume.getPath(), null); return volumeDao.findById(diskProfile.getVolumeId()); } diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 9f38182e4d5..187b315dd35 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -820,7 +820,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } diskProfile.setSize(copyRemoteVolumeAnswer.getSize()); DiskProfile profile = volumeManager.updateImportedVolume(type, diskOffering, vm, template, deviceId, - storagePool.getId(), copyRemoteVolumeAnswer.getFilename(), chainInfo, diskProfile); + storagePool.getId(), storagePool.getPoolType(), copyRemoteVolumeAnswer.getFilename(), chainInfo, diskProfile); return new Pair<>(profile, storagePool); } @@ -836,7 +836,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { StoragePool storagePool = storagePools.get(0); DiskProfile profile = volumeManager.updateImportedVolume(type, diskOffering, vm, template, deviceId, - storagePool.getId(), diskPath, null, diskProfile); + storagePool.getId(), storagePool.getPoolType(), diskPath, null, diskProfile); return new Pair<>(profile, storagePool); } @@ -847,7 +847,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { StoragePool storagePool = primaryDataStoreDao.findById(poolId); DiskProfile profile = volumeManager.updateImportedVolume(type, diskOffering, vm, template, deviceId, - poolId, diskPath, null, diskProfile); + poolId, storagePool.getPoolType(), diskPath, null, diskProfile); return new Pair<>(profile, storagePool); } @@ -866,7 +866,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } StoragePool storagePool = getStoragePool(disk, zone, cluster, diskOffering); DiskProfile profile = volumeManager.importVolume(type, name, diskOffering, diskSize, - minIops, maxIops, vm.getDataCenterId(), vm.getHypervisorType(), vm, template, owner, deviceId, storagePool.getId(), path, chainInfo); + minIops, maxIops, vm.getDataCenterId(), vm.getHypervisorType(), vm, template, owner, deviceId, storagePool.getId(), storagePool.getPoolType(), path, chainInfo); return new Pair(profile, storagePool); } diff --git a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java index 419acc0ca0b..9ee34d32a17 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImplTest.java @@ -277,7 +277,7 @@ public class VolumeImportUnmanageManagerImplTest { doNothing().when(volumeApiService).validateCustomDiskOfferingSizeRange(anyLong()); doReturn(true).when(volumeApiService).doesStoragePoolSupportDiskOffering(any(), any()); doReturn(diskProfile).when(volumeManager).importVolume(any(), anyString(), any(), eq(virtualSize), isNull(), isNull(), anyLong(), - any(), isNull(), isNull(), any(), isNull(), anyLong(), anyString(), isNull()); + any(), isNull(), isNull(), any(), isNull(), anyLong(), any(), anyString(), isNull()); when(diskProfile.getVolumeId()).thenReturn(volumeId); when(volumeDao.findById(volumeId)).thenReturn(volumeVO); diff --git a/utils/src/test/java/org/apache/cloudstack/utils/reflectiontostringbuilderutils/ReflectionToStringBuilderUtilsTest.java b/utils/src/test/java/org/apache/cloudstack/utils/reflectiontostringbuilderutils/ReflectionToStringBuilderUtilsTest.java index afd033c7b04..9168ec0d2fa 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/reflectiontostringbuilderutils/ReflectionToStringBuilderUtilsTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/reflectiontostringbuilderutils/ReflectionToStringBuilderUtilsTest.java @@ -53,7 +53,7 @@ public class ReflectionToStringBuilderUtilsTest extends TestCase { private static final String DEFAULT_MULTIPLE_VALUES_SEPARATOR = ","; @Before - public void setup(){ + public void setup() { classToReflect = String.class; classToReflectFieldsNamesList = ReflectionUtils.getAllFields(classToReflect).stream().map(objectField -> objectField.getName()).collect(Collectors.toList()); classToReflectRemovedField = classToReflectFieldsNamesList.remove(0); From 973819dad61b1322ccc7ca54fad019ee39c5810b Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 9 Oct 2025 07:52:17 -0400 Subject: [PATCH 011/127] API: Add support to list all snapshot policies & backup schedules (#11587) * API: Add support to list all snapshot policies & backup schedules * Add support for backup policy listing without tying it to the vmid * add tests for snapshot policy listing * update tests for listbackupschedules * remove trailing spaces and fix lint failure * Add upgrade test * remove unused import * add create policy - snap/backup in the list view with resource (volume/vm) selection * add translations * refresh parent list * remove unnecessary alert info * fix checks for UI backup schedule list view * fix checks for UI backup schedule list view * add back access checks * add since param * fix failing test * update snapshot policy and backup schedule ownership when VM is moved * fix issue with showing vm selection * fix unit test failure * Update list snappolicy & backup schedule logic to list only those that belong to a proj or for root admin those that belong to it, unless listall & projid is passed * fix test * support snap / backup policy search using keyword * fix tests --- .../storage/snapshot/SnapshotApiService.java | 2 +- .../storage/snapshot/SnapshotPolicy.java | 3 +- .../user/backup/ListBackupScheduleCmd.java | 30 ++- .../snapshot/ListSnapshotPoliciesCmd.java | 7 +- .../api/response/SnapshotPolicyResponse.java | 8 + .../cloudstack/backup/BackupManager.java | 3 +- .../cloudstack/backup/BackupSchedule.java | 3 +- .../backup/ListBackupScheduleCmdTest.java | 98 +++++++ .../snapshot/ListSnapshotPoliciesCmdTest.java | 79 ++++++ .../com/cloud/storage/SnapshotPolicyVO.java | 38 ++- .../upgrade/dao/Upgrade42100to42200.java | 73 ++++++ .../cloudstack/backup/BackupScheduleVO.java | 38 ++- .../META-INF/db/schema-42100to42200.sql | 6 + .../upgrade/dao/Upgrade42100to42200Test.java | 242 ++++++++++++++++++ .../test/SnapshotTestWithFakeData.java | 2 +- .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../storage/snapshot/SnapshotManagerImpl.java | 63 +++-- .../java/com/cloud/vm/UserVmManagerImpl.java | 41 +++ .../cloudstack/backup/BackupManagerImpl.java | 60 ++++- .../snapshot/SnapshotManagerImplTest.java | 125 +++++++++ .../storage/snapshot/SnapshotManagerTest.java | 6 +- .../com/cloud/vm/UserVmManagerImplTest.java | 9 + .../cloudstack/backup/BackupManagerTest.java | 75 ++++++ ui/public/locales/en.json | 15 +- ui/src/components/view/DetailsTab.vue | 7 + ui/src/components/view/ListView.vue | 86 ++++++- ui/src/components/view/SearchView.vue | 50 +++- ui/src/config/section/storage.js | 90 +++++++ ui/src/views/compute/BackupScheduleWizard.vue | 30 ++- .../compute/backup/CreateBackupSchedule.vue | 183 +++++++++++++ .../views/storage/RecurringSnapshotVolume.vue | 153 ++++++++++- 31 files changed, 1553 insertions(+), 73 deletions(-) create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmdTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmdTest.java create mode 100644 engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42100to42200Test.java create mode 100644 ui/src/views/compute/backup/CreateBackupSchedule.vue diff --git a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java index 67afd6aa4e2..d52e645ec79 100644 --- a/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java +++ b/api/src/main/java/com/cloud/storage/snapshot/SnapshotApiService.java @@ -85,7 +85,7 @@ public interface SnapshotApiService { * the command that specifies the volume criteria * @return list of snapshot policies */ - Pair, Integer> listPoliciesforVolume(ListSnapshotPoliciesCmd cmd); + Pair, Integer> listSnapshotPolicies(ListSnapshotPoliciesCmd cmd); boolean deleteSnapshotPolicies(DeleteSnapshotPoliciesCmd cmd); diff --git a/api/src/main/java/com/cloud/storage/snapshot/SnapshotPolicy.java b/api/src/main/java/com/cloud/storage/snapshot/SnapshotPolicy.java index 22d5dfb9c1b..13009a9808a 100644 --- a/api/src/main/java/com/cloud/storage/snapshot/SnapshotPolicy.java +++ b/api/src/main/java/com/cloud/storage/snapshot/SnapshotPolicy.java @@ -16,11 +16,12 @@ // under the License. package com.cloud.storage.snapshot; +import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.Displayable; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; -public interface SnapshotPolicy extends Identity, InternalIdentity, Displayable { +public interface SnapshotPolicy extends ControlledEntity, Identity, InternalIdentity, Displayable { long getVolumeId(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java index fa6e3ea5d45..d68e4a9296e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java @@ -24,7 +24,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.BackupScheduleResponse; @@ -39,7 +39,6 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.utils.exception.CloudRuntimeException; import java.util.ArrayList; import java.util.List; @@ -48,10 +47,10 @@ import java.util.List; description = "List backup schedule of a VM", responseObject = BackupScheduleResponse.class, since = "4.14.0", authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) -public class ListBackupScheduleCmd extends BaseCmd { +public class ListBackupScheduleCmd extends BaseListProjectAndAccountResourcesCmd { @Inject - private BackupManager backupManager; + BackupManager backupManager; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -60,10 +59,16 @@ public class ListBackupScheduleCmd extends BaseCmd { @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, - required = true, description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupScheduleResponse.class, + description = "the ID of the backup schedule", + since = "4.22.0") + private Long id; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -72,6 +77,10 @@ public class ListBackupScheduleCmd extends BaseCmd { return vmId; } + public Long getId() { + return id; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -79,19 +88,18 @@ public class ListBackupScheduleCmd extends BaseCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try{ - List schedules = backupManager.listBackupSchedule(getVmId()); + List schedules = backupManager.listBackupSchedules(this); ListResponse response = new ListResponse<>(); List scheduleResponses = new ArrayList<>(); + if (!CollectionUtils.isNullOrEmpty(schedules)) { for (BackupSchedule schedule : schedules) { scheduleResponses.add(_responseGenerator.createBackupScheduleResponse(schedule)); } - response.setResponses(scheduleResponses, schedules.size()); - response.setResponseName(getCommandName()); - setResponseObject(response); - } else { - throw new CloudRuntimeException("No backup schedule exists for the VM"); } + response.setResponses(scheduleResponses, schedules.size()); + response.setResponseName(getCommandName()); + setResponseObject(response); } catch (Exception e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmd.java index 126a4080e6d..f4dfbe58cf2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmd.java @@ -23,7 +23,7 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.SnapshotPolicyResponse; @@ -34,7 +34,7 @@ import com.cloud.utils.Pair; @APICommand(name = "listSnapshotPolicies", description = "Lists snapshot policies.", responseObject = SnapshotPolicyResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListSnapshotPoliciesCmd extends BaseListCmd { +public class ListSnapshotPoliciesCmd extends BaseListProjectAndAccountResourcesCmd { ///////////////////////////////////////////////////// @@ -69,13 +69,14 @@ public class ListSnapshotPoliciesCmd extends BaseListCmd { public Long getId() { return id; } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override public void execute() { - Pair, Integer> result = _snapshotService.listPoliciesforVolume(this); + Pair, Integer> result = _snapshotService.listSnapshotPolicies(this); ListResponse response = new ListResponse(); List policyResponses = new ArrayList(); for (SnapshotPolicy policy : result.first()) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java index 4ce77cfdf6e..998d6af871d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotPolicyResponse.java @@ -37,6 +37,10 @@ public class SnapshotPolicyResponse extends BaseResponseWithTagInformation { @Param(description = "the ID of the disk volume") private String volumeId; + @SerializedName("volumename") + @Param(description = "the name of the disk volume") + private String volumeName; + @SerializedName("schedule") @Param(description = "time the snapshot is scheduled to be taken.") private String schedule; @@ -87,6 +91,10 @@ public class SnapshotPolicyResponse extends BaseResponseWithTagInformation { this.volumeId = volumeId; } + public void setVolumeName(String volumeName) { + this.volumeName = volumeName; + } + public String getSchedule() { return schedule; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index f1f0c3c31ee..db051313d96 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd; +import org.apache.cloudstack.api.command.user.backup.ListBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupsCmd; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.framework.config.ConfigKey; @@ -174,7 +175,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer * @param vmId * @return */ - List listBackupSchedule(Long vmId); + List listBackupSchedules(ListBackupScheduleCmd cmd); /** * Deletes VM backup schedule for a VM diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java index b5138d34de1..44fdf70c4c1 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java @@ -19,11 +19,12 @@ package org.apache.cloudstack.backup; import java.util.Date; +import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.InternalIdentity; import com.cloud.utils.DateUtil; -public interface BackupSchedule extends InternalIdentity { +public interface BackupSchedule extends ControlledEntity, InternalIdentity { Long getVmId(); DateUtil.IntervalType getScheduleType(); String getSchedule(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmdTest.java new file mode 100644 index 00000000000..a0d88bbc84e --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmdTest.java @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.backup; + +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.BackupSchedule; +import org.apache.cloudstack.context.CallContext; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class ListBackupScheduleCmdTest { + + @Mock + private BackupManager backupManager; + + @Mock + private ResponseGenerator responseGenerator; + + private ListBackupScheduleCmd cmd; + + @Before + public void setUp() { + cmd = new ListBackupScheduleCmd(); + cmd.backupManager = backupManager; + cmd._responseGenerator = responseGenerator; + } + + @Test + public void testExecuteWithSchedules() throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException, NetworkRuleConflictException { + BackupSchedule schedule = Mockito.mock(BackupSchedule.class); + BackupScheduleResponse scheduleResponse = Mockito.mock(BackupScheduleResponse.class); + List schedules = new ArrayList<>(); + schedules.add(schedule); + + Mockito.when(backupManager.listBackupSchedules(cmd)).thenReturn(schedules); + Mockito.when(responseGenerator.createBackupScheduleResponse(schedule)).thenReturn(scheduleResponse); + + Account mockAccount = Mockito.mock(Account.class); + CallContext callContext = Mockito.mock(CallContext.class); + try (org.mockito.MockedStatic mocked = Mockito.mockStatic(CallContext.class)) { + cmd.execute(); + } + + ListResponse response = (ListResponse) cmd.getResponseObject(); + Assert.assertNotNull(response); + Assert.assertEquals(1, response.getResponses().size()); + Assert.assertEquals(scheduleResponse, response.getResponses().get(0)); + } + + @Test + public void testExecuteWithNoSchedules() { + Mockito.when(backupManager.listBackupSchedules(cmd)).thenReturn(new ArrayList<>()); + CallContext callContext = Mockito.mock(CallContext.class); + + try (org.mockito.MockedStatic mocked = Mockito.mockStatic(CallContext.class)) { + mocked.when(CallContext::current).thenReturn(callContext); + cmd.execute(); + } catch (ResourceUnavailableException | InsufficientCapacityException | ResourceAllocationException | + NetworkRuleConflictException e) { + throw new RuntimeException(e); + } + + ListResponse response = (ListResponse) cmd.getResponseObject(); + Assert.assertNotNull(response); + Assert.assertEquals(0, response.getResponses().size()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmdTest.java new file mode 100644 index 00000000000..36cfcf5cb9a --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotPoliciesCmdTest.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.snapshot; + +import com.cloud.storage.snapshot.SnapshotApiService; +import com.cloud.storage.snapshot.SnapshotPolicy; +import com.cloud.utils.Pair; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SnapshotPolicyResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.List; + +public class ListSnapshotPoliciesCmdTest { + private ListSnapshotPoliciesCmd cmd; + private SnapshotApiService snapshotService; + private ResponseGenerator responseGenerator; + + @Before + public void setUp() { + cmd = new ListSnapshotPoliciesCmd(); + snapshotService = Mockito.mock(SnapshotApiService.class); + responseGenerator = Mockito.mock(ResponseGenerator.class); + + cmd._snapshotService = snapshotService; + cmd._responseGenerator = responseGenerator; + } + + @Test + public void testExecuteWithPolicies() { + SnapshotPolicy policy = Mockito.mock(SnapshotPolicy.class); + SnapshotPolicyResponse policyResponse = Mockito.mock(SnapshotPolicyResponse.class); + List policies = new ArrayList<>(); + policies.add(policy); + + Mockito.when(snapshotService.listSnapshotPolicies(cmd)) + .thenReturn(new Pair<>(policies, 1)); + Mockito.when(responseGenerator.createSnapshotPolicyResponse(policy)) + .thenReturn(policyResponse); + + cmd.execute(); + + ListResponse response = (ListResponse) cmd.getResponseObject(); + Assert.assertNotNull(response); + Assert.assertEquals(1, response.getResponses().size()); + Assert.assertEquals(policyResponse, response.getResponses().get(0)); + } + + @Test + public void testExecuteWithNoPolicies() { + Mockito.when(snapshotService.listSnapshotPolicies(cmd)) + .thenReturn(new Pair<>(new ArrayList<>(), 0)); + + cmd.execute(); + + ListResponse response = (ListResponse) cmd.getResponseObject(); + Assert.assertNotNull(response); + Assert.assertTrue(response.getResponses().isEmpty()); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/SnapshotPolicyVO.java b/engine/schema/src/main/java/com/cloud/storage/SnapshotPolicyVO.java index f57d9d3dccf..299c6380ab6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/SnapshotPolicyVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/SnapshotPolicyVO.java @@ -59,6 +59,12 @@ public class SnapshotPolicyVO implements SnapshotPolicy { @Column(name = "uuid") String uuid; + @Column(name = "account_id") + long accountId; + + @Column(name = "domain_id") + long domainId; + @Column(name = "display", updatable = true, nullable = false) protected boolean display = true; @@ -66,7 +72,7 @@ public class SnapshotPolicyVO implements SnapshotPolicy { this.uuid = UUID.randomUUID().toString(); } - public SnapshotPolicyVO(long volumeId, String schedule, String timezone, IntervalType intvType, int maxSnaps, boolean display) { + public SnapshotPolicyVO(long volumeId, String schedule, String timezone, IntervalType intvType, int maxSnaps, long accountId, long domainId, boolean display) { this.volumeId = volumeId; this.schedule = schedule; this.timezone = timezone; @@ -75,6 +81,8 @@ public class SnapshotPolicyVO implements SnapshotPolicy { this.active = true; this.display = display; this.uuid = UUID.randomUUID().toString(); + this.accountId = accountId; + this.domainId = domainId; } @Override @@ -160,4 +168,32 @@ public class SnapshotPolicyVO implements SnapshotPolicy { public void setDisplay(boolean display) { this.display = display; } + + @Override + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + + @Override + public long getDomainId() { + return domainId; + } + + public void setDomainId(long domainId) { + this.domainId = domainId; + } + + @Override + public Class getEntityType() { + return SnapshotPolicy.class; + } + + @Override + public String getName() { + return null; + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42100to42200.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42100to42200.java index c2cfd02c15c..5138d51bb1b 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42100to42200.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42100to42200.java @@ -16,6 +16,14 @@ // under the License. package com.cloud.upgrade.dao; +import com.cloud.utils.exception.CloudRuntimeException; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + public class Upgrade42100to42200 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { @Override @@ -27,4 +35,69 @@ public class Upgrade42100to42200 extends DbUpgradeAbstractImpl implements DbUpgr public String getUpgradedVersion() { return "4.22.0.0"; } + + @Override + public InputStream[] getPrepareScripts() { + final String scriptFile = "META-INF/db/schema-42100to42200.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + @Override + public void performDataMigration(Connection conn) { + updateSnapshotPolicyOwnership(conn); + updateBackupScheduleOwnership(conn); + } + + protected void updateSnapshotPolicyOwnership(Connection conn) { + // set account_id and domain_id in snapshot_policy table from volume table + String selectSql = "SELECT sp.id, v.account_id, v.domain_id FROM snapshot_policy sp, volumes v WHERE sp.volume_id = v.id AND (sp.account_id IS NULL AND sp.domain_id IS NULL)"; + String updateSql = "UPDATE snapshot_policy SET account_id = ?, domain_id = ? WHERE id = ?"; + + try (PreparedStatement selectPstmt = conn.prepareStatement(selectSql); + ResultSet rs = selectPstmt.executeQuery(); + PreparedStatement updatePstmt = conn.prepareStatement(updateSql)) { + + while (rs.next()) { + long policyId = rs.getLong(1); + long accountId = rs.getLong(2); + long domainId = rs.getLong(3); + + updatePstmt.setLong(1, accountId); + updatePstmt.setLong(2, domainId); + updatePstmt.setLong(3, policyId); + updatePstmt.executeUpdate(); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to update snapshot_policy table with account_id and domain_id", e); + } + } + + protected void updateBackupScheduleOwnership(Connection conn) { + // Set account_id and domain_id in backup_schedule table from vm_instance table + String selectSql = "SELECT bs.id, vm.account_id, vm.domain_id FROM backup_schedule bs, vm_instance vm WHERE bs.vm_id = vm.id AND (bs.account_id IS NULL AND bs.domain_id IS NULL)"; + String updateSql = "UPDATE backup_schedule SET account_id = ?, domain_id = ? WHERE id = ?"; + + try (PreparedStatement selectPstmt = conn.prepareStatement(selectSql); + ResultSet rs = selectPstmt.executeQuery(); + PreparedStatement updatePstmt = conn.prepareStatement(updateSql)) { + + while (rs.next()) { + long scheduleId = rs.getLong(1); + long accountId = rs.getLong(2); + long domainId = rs.getLong(3); + + updatePstmt.setLong(1, accountId); + updatePstmt.setLong(2, domainId); + updatePstmt.setLong(3, scheduleId); + updatePstmt.executeUpdate(); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Unable to update backup_schedule table with account_id and domain_id", e); + } + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java index 37e8105e3d5..1ee2cff78b6 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java @@ -68,10 +68,16 @@ public class BackupScheduleVO implements BackupSchedule { @Column(name = "quiescevm") Boolean quiesceVM = false; + @Column(name = "account_id") + Long accountId; + + @Column(name = "domain_id") + Long domainId; + public BackupScheduleVO() { } - public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, int maxBackups, Boolean quiesceVM) { + public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, int maxBackups, Boolean quiesceVM, Long accountId, Long domainId) { this.vmId = vmId; this.scheduleType = (short) scheduleType.ordinal(); this.schedule = schedule; @@ -79,6 +85,8 @@ public class BackupScheduleVO implements BackupSchedule { this.scheduledTimestamp = scheduledTimestamp; this.maxBackups = maxBackups; this.quiesceVM = quiesceVM; + this.accountId = accountId; + this.domainId = domainId; } @Override @@ -161,4 +169,32 @@ public class BackupScheduleVO implements BackupSchedule { public Boolean getQuiesceVM() { return quiesceVM; } + + @Override + public Class getEntityType() { + return BackupSchedule.class; + } + + @Override + public String getName() { + return null; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 405f2af9564..bdf3bc3a63c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -26,6 +26,12 @@ CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', ' -- Increase length of scripts_version column to 128 due to md5sum to sha512sum change CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.domain_router', 'scripts_version', 'scripts_version', 'VARCHAR(128)'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.snapshot_policy','domain_id', 'BIGINT(20) DEFAULT NULL'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.snapshot_policy','account_id', 'BIGINT(20) DEFAULT NULL'); + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule','domain_id', 'BIGINT(20) DEFAULT NULL'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule','account_id', 'BIGINT(20) DEFAULT NULL'); + -- Increase the cache_mode column size from cloud.disk_offering table CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.disk_offering', 'cache_mode', 'cache_mode', 'varchar(18) DEFAULT "none" COMMENT "The disk cache mode to use for disks created with this offering"'); diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42100to42200Test.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42100to42200Test.java new file mode 100644 index 00000000000..cff34fb4e20 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42100to42200Test.java @@ -0,0 +1,242 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.inOrder; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class Upgrade42100to42200Test { + + @Spy + Upgrade42100to42200 upgrade; + + @Mock + private Connection conn; + + @Mock + private PreparedStatement selectStmt; + + @Mock + private PreparedStatement updateStmt; + + @Mock + private ResultSet resultSet; + + @Test + public void testUpdateSnapshotPolicyOwnership() throws SQLException { + // Setup mock data for snapshot policies without ownership + when(conn.prepareStatement("SELECT sp.id, v.account_id, v.domain_id FROM snapshot_policy sp, volumes v WHERE sp.volume_id = v.id AND (sp.account_id IS NULL AND sp.domain_id IS NULL)")) + .thenReturn(selectStmt); + when(conn.prepareStatement("UPDATE snapshot_policy SET account_id = ?, domain_id = ? WHERE id = ?")) + .thenReturn(updateStmt); + when(selectStmt.executeQuery()).thenReturn(resultSet); + + when(resultSet.next()) + .thenReturn(true) + .thenReturn(true) + .thenReturn(false); + + when(resultSet.getLong(1)) + .thenReturn(1L) + .thenReturn(2L); + when(resultSet.getLong(2)) + .thenReturn(100L) + .thenReturn(200L); + when(resultSet.getLong(3)) + .thenReturn(1L) + .thenReturn(2L); + + upgrade.updateSnapshotPolicyOwnership(conn); + + verify(conn).prepareStatement("SELECT sp.id, v.account_id, v.domain_id FROM snapshot_policy sp, volumes v WHERE sp.volume_id = v.id AND (sp.account_id IS NULL AND sp.domain_id IS NULL)"); + verify(conn).prepareStatement("UPDATE snapshot_policy SET account_id = ?, domain_id = ? WHERE id = ?"); + + InOrder inOrder = inOrder(updateStmt); + + inOrder.verify(updateStmt).setLong(1, 100L); // account_id + inOrder.verify(updateStmt).setLong(2, 1L); // domain_id + inOrder.verify(updateStmt).setLong(3, 1L); // policy_id + inOrder.verify(updateStmt).executeUpdate(); + + inOrder.verify(updateStmt).setLong(1, 200L); // account_id + inOrder.verify(updateStmt).setLong(2, 2L); // domain_id + inOrder.verify(updateStmt).setLong(3, 2L); // policy_id + inOrder.verify(updateStmt).executeUpdate(); + + verify(updateStmt, times(2)).executeUpdate(); + } + + @Test + public void testUpdateBackupScheduleOwnership() throws SQLException { + when(conn.prepareStatement("SELECT bs.id, vm.account_id, vm.domain_id FROM backup_schedule bs, vm_instance vm WHERE bs.vm_id = vm.id AND (bs.account_id IS NULL AND bs.domain_id IS NULL)")) + .thenReturn(selectStmt); + when(conn.prepareStatement("UPDATE backup_schedule SET account_id = ?, domain_id = ? WHERE id = ?")) + .thenReturn(updateStmt); + when(selectStmt.executeQuery()).thenReturn(resultSet); + + when(resultSet.next()) + .thenReturn(true) + .thenReturn(true) + .thenReturn(true) + .thenReturn(false); + + when(resultSet.getLong(1)) + .thenReturn(10L) + .thenReturn(20L) + .thenReturn(30L); + when(resultSet.getLong(2)) + .thenReturn(500L) + .thenReturn(600L) + .thenReturn(700L); + when(resultSet.getLong(3)) + .thenReturn(5L) + .thenReturn(6L) + .thenReturn(7L); + + upgrade.updateBackupScheduleOwnership(conn); + + verify(conn).prepareStatement("SELECT bs.id, vm.account_id, vm.domain_id FROM backup_schedule bs, vm_instance vm WHERE bs.vm_id = vm.id AND (bs.account_id IS NULL AND bs.domain_id IS NULL)"); + verify(conn).prepareStatement("UPDATE backup_schedule SET account_id = ?, domain_id = ? WHERE id = ?"); + + InOrder inOrder = inOrder(updateStmt); + + inOrder.verify(updateStmt).setLong(1, 500L); + inOrder.verify(updateStmt).setLong(2, 5L); + inOrder.verify(updateStmt).setLong(3, 10L); + inOrder.verify(updateStmt).executeUpdate(); + + inOrder.verify(updateStmt).setLong(1, 600L); + inOrder.verify(updateStmt).setLong(2, 6L); + inOrder.verify(updateStmt).setLong(3, 20L); + inOrder.verify(updateStmt).executeUpdate(); + + inOrder.verify(updateStmt).setLong(1, 700L); + inOrder.verify(updateStmt).setLong(2, 7L); + inOrder.verify(updateStmt).setLong(3, 30L); + inOrder.verify(updateStmt).executeUpdate(); + + verify(updateStmt, times(3)).executeUpdate(); + } + + @Test + public void testUpdateSnapshotPolicyOwnershipNoResults() throws SQLException { + when(conn.prepareStatement("SELECT sp.id, v.account_id, v.domain_id FROM snapshot_policy sp, volumes v WHERE sp.volume_id = v.id AND (sp.account_id IS NULL AND sp.domain_id IS NULL)")) + .thenReturn(selectStmt); + when(conn.prepareStatement("UPDATE snapshot_policy SET account_id = ?, domain_id = ? WHERE id = ?")) + .thenReturn(updateStmt); + when(selectStmt.executeQuery()).thenReturn(resultSet); + + when(resultSet.next()).thenReturn(false); + + upgrade.updateSnapshotPolicyOwnership(conn); + + verify(selectStmt).executeQuery(); + verify(updateStmt, times(0)).executeUpdate(); + } + + @Test + public void testUpdateBackupScheduleOwnershipNoResults() throws SQLException { + when(conn.prepareStatement("SELECT bs.id, vm.account_id, vm.domain_id FROM backup_schedule bs, vm_instance vm WHERE bs.vm_id = vm.id AND (bs.account_id IS NULL AND bs.domain_id IS NULL)")) + .thenReturn(selectStmt); + when(conn.prepareStatement("UPDATE backup_schedule SET account_id = ?, domain_id = ? WHERE id = ?")) + .thenReturn(updateStmt); + when(selectStmt.executeQuery()).thenReturn(resultSet); + + when(resultSet.next()).thenReturn(false); + + upgrade.updateBackupScheduleOwnership(conn); + + verify(selectStmt).executeQuery(); + verify(updateStmt, times(0)).executeUpdate(); + } + + @Test + public void testPerformDataMigration() throws SQLException { + when(conn.prepareStatement(anyString())).thenReturn(selectStmt); + when(selectStmt.executeQuery()).thenReturn(resultSet); + when(resultSet.next()).thenReturn(false); + + upgrade.performDataMigration(conn); + + verify(conn).prepareStatement("SELECT sp.id, v.account_id, v.domain_id FROM snapshot_policy sp, volumes v WHERE sp.volume_id = v.id AND (sp.account_id IS NULL AND sp.domain_id IS NULL)"); + verify(conn).prepareStatement("SELECT bs.id, vm.account_id, vm.domain_id FROM backup_schedule bs, vm_instance vm WHERE bs.vm_id = vm.id AND (bs.account_id IS NULL AND bs.domain_id IS NULL)"); + } + + @Test + public void testUpdateSnapshotPolicyOwnershipSingleRecord() throws SQLException { + when(conn.prepareStatement("SELECT sp.id, v.account_id, v.domain_id FROM snapshot_policy sp, volumes v WHERE sp.volume_id = v.id AND (sp.account_id IS NULL AND sp.domain_id IS NULL)")) + .thenReturn(selectStmt); + when(conn.prepareStatement("UPDATE snapshot_policy SET account_id = ?, domain_id = ? WHERE id = ?")) + .thenReturn(updateStmt); + when(selectStmt.executeQuery()).thenReturn(resultSet); + + when(resultSet.next()) + .thenReturn(true) + .thenReturn(false); + + when(resultSet.getLong(1)).thenReturn(42L); + when(resultSet.getLong(2)).thenReturn(999L); + when(resultSet.getLong(3)).thenReturn(10L); + + upgrade.updateSnapshotPolicyOwnership(conn); + + verify(updateStmt).setLong(1, 999L); + verify(updateStmt).setLong(2, 10L); + verify(updateStmt).setLong(3, 42L); + verify(updateStmt, times(1)).executeUpdate(); + } + + @Test + public void testUpdateBackupScheduleOwnershipSingleRecord() throws SQLException { + when(conn.prepareStatement("SELECT bs.id, vm.account_id, vm.domain_id FROM backup_schedule bs, vm_instance vm WHERE bs.vm_id = vm.id AND (bs.account_id IS NULL AND bs.domain_id IS NULL)")) + .thenReturn(selectStmt); + when(conn.prepareStatement("UPDATE backup_schedule SET account_id = ?, domain_id = ? WHERE id = ?")) + .thenReturn(updateStmt); + when(selectStmt.executeQuery()).thenReturn(resultSet); + + when(resultSet.next()) + .thenReturn(true) + .thenReturn(false); + + when(resultSet.getLong(1)).thenReturn(55L); + when(resultSet.getLong(2)).thenReturn(888L); + when(resultSet.getLong(3)).thenReturn(15L); + + upgrade.updateBackupScheduleOwnership(conn); + + verify(updateStmt).setLong(1, 888L); + verify(updateStmt).setLong(2, 15L); + verify(updateStmt).setLong(3, 55L); + verify(updateStmt, times(1)).executeUpdate(); + } +} diff --git a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTestWithFakeData.java b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTestWithFakeData.java index 152c279547c..9868ccdf29a 100644 --- a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTestWithFakeData.java +++ b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTestWithFakeData.java @@ -305,7 +305,7 @@ public class SnapshotTestWithFakeData { } protected SnapshotPolicyVO createSnapshotPolicy(Long volId) { - SnapshotPolicyVO policyVO = new SnapshotPolicyVO(volId, "jfkd", "fdfd", DateUtil.IntervalType.DAILY, 8, true); + SnapshotPolicyVO policyVO = new SnapshotPolicyVO(volId, "jfkd", "fdfd", DateUtil.IntervalType.DAILY, 8, 1, 1, true); policyVO = snapshotPolicyDao.persist(policyVO); return policyVO; } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index cdb210c6dea..74a252fe244 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -853,6 +853,7 @@ public class ApiResponseHelper implements ResponseGenerator { Volume vol = ApiDBUtils.findVolumeById(policy.getVolumeId()); if (vol != null) { policyResponse.setVolumeId(vol.getUuid()); + policyResponse.setVolumeName(vol.getName()); } policyResponse.setSchedule(policy.getSchedule()); policyResponse.setIntervalType(policy.getInterval()); diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index a27885527d4..57209fe58ed 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -1303,7 +1303,8 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement } protected SnapshotPolicyVO createSnapshotPolicy(long volumeId, String schedule, String timezone, IntervalType intervalType, int maxSnaps, boolean display, List zoneIds, List poolIds) { - SnapshotPolicyVO policy = new SnapshotPolicyVO(volumeId, schedule, timezone, intervalType, maxSnaps, display); + VolumeVO volume = _volsDao.findById(volumeId); + SnapshotPolicyVO policy = new SnapshotPolicyVO(volumeId, schedule, timezone, intervalType, maxSnaps, volume.getAccountId(), volume.getDomainId(), display); policy = _snapshotPolicyDao.persist(policy); if (CollectionUtils.isNotEmpty(zoneIds)) { List details = new ArrayList<>(); @@ -1388,28 +1389,54 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement } @Override - public Pair, Integer> listPoliciesforVolume(ListSnapshotPoliciesCmd cmd) { + public Pair, Integer> listSnapshotPolicies(ListSnapshotPoliciesCmd cmd) { Long volumeId = cmd.getVolumeId(); - boolean display = cmd.isDisplay(); Long id = cmd.getId(); - Pair, Integer> result = null; - // TODO - Have a better way of doing this. - if (id != null) { - result = _snapshotPolicyDao.listAndCountById(id, display, null); - if (result != null && result.first() != null && !result.first().isEmpty()) { - SnapshotPolicyVO snapshotPolicy = result.first().get(0); - volumeId = snapshotPolicy.getVolumeId(); + Account caller = CallContext.current().getCallingAccount(); + List permittedAccounts = new ArrayList<>(); + String keyword = cmd.getKeyword(); + + // Verify parameters + if (volumeId != null) { + VolumeVO volume = _volsDao.findById(volumeId); + if (volume != null) { + _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); } } - VolumeVO volume = _volsDao.findById(volumeId); - if (volume == null) { - throw new InvalidParameterValueException("Unable to find a volume with id " + volumeId); + + Ternary domainIdRecursiveListProject = + new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); + _accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); + Long domainId = domainIdRecursiveListProject.first(); + Boolean isRecursive = domainIdRecursiveListProject.second(); + ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + + Filter searchFilter = new Filter(SnapshotPolicyVO.class, "id", false, cmd.getStartIndex(), cmd.getPageSizeVal()); + SearchBuilder policySearch = _snapshotPolicyDao.createSearchBuilder(); + _accountMgr.buildACLSearchBuilder(policySearch, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + + policySearch.and("id", policySearch.entity().getId(), SearchCriteria.Op.EQ); + policySearch.and("volumeId", policySearch.entity().getVolumeId(), SearchCriteria.Op.EQ); + + SearchBuilder volumeSearch = _volsDao.createSearchBuilder(); + volumeSearch.and("name", volumeSearch.entity().getName(), SearchCriteria.Op.LIKE); + policySearch.join("volumeJoin", volumeSearch, policySearch.entity().getVolumeId(), volumeSearch.entity().getId(), JoinBuilder.JoinType.INNER); + + SearchCriteria sc = policySearch.create(); + _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + + if (volumeId != null) { + sc.setParameters("volumeId", volumeId); } - _accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, volume); - if (result != null) - return new Pair, Integer>(result.first(), result.second()); - result = _snapshotPolicyDao.listAndCountByVolumeId(volumeId, display); - return new Pair, Integer>(result.first(), result.second()); + if (id != null) { + sc.setParameters("id", id); + } + if (keyword != null) { + sc.setJoinParameters("volumeJoin", "name", "%" + keyword + "%"); + } + + Pair, Integer> result = _snapshotPolicyDao.searchAndCount(sc, searchFilter); + return new Pair<>(result.first(), result.second()); } private List listPoliciesforVolume(long volumeId) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index f1bbfa07292..6769a8066dd 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -60,6 +60,8 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.storage.SnapshotPolicyVO; +import com.cloud.storage.dao.SnapshotPolicyDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -103,8 +105,10 @@ import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.volume.ChangeOfferingForVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.BackupScheduleVO; import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntity; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMNetworkMapDao; @@ -607,6 +611,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir ReservationDao reservationDao; @Inject ResourceLimitService resourceLimitService; + @Inject + SnapshotPolicyDao snapshotPolicyDao; + @Inject + BackupScheduleDao backupScheduleDao; @Inject private StatsCollector statsCollector; @@ -8045,6 +8053,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir updateVolumesOwner(volumes, oldAccount, newAccount, newAccountId); + updateSnapshotPolicyOwnership(volumes, newAccount); + updateBackupScheduleOwnership(vm, newAccount); + try { updateVmNetwork(cmd, caller, vm, newAccount, template); } catch (InsufficientCapacityException | ResourceAllocationException e) { @@ -8519,6 +8530,36 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } + protected void updateSnapshotPolicyOwnership(List volumes, Account newAccount) { + logger.debug("Updating snapshot policy ownership for volumes of VM being assigned to account [{}]", newAccount); + + for (VolumeVO volume : volumes) { + List snapshotPolicies = snapshotPolicyDao.listByVolumeId(volume.getId()); + for (SnapshotPolicyVO policy : snapshotPolicies) { + logger.trace("Updating snapshot policy [{}] ownership from account [{}] to account [{}]", + policy.getId(), policy.getAccountId(), newAccount.getAccountId()); + + policy.setAccountId(newAccount.getAccountId()); + policy.setDomainId(newAccount.getDomainId()); + snapshotPolicyDao.update(policy.getId(), policy); + } + } + } + + protected void updateBackupScheduleOwnership(UserVmVO vm, Account newAccount) { + logger.debug("Updating backup schedule ownership for VM [{}] being assigned to account [{}]", vm, newAccount); + + List backupSchedules = backupScheduleDao.listByVM(vm.getId()); + for (BackupScheduleVO schedule : backupSchedules) { + logger.trace("Updating backup schedule [{}] ownership from account [{}] to account [{}]", + schedule.getId(), schedule.getAccountId(), newAccount.getAccountId()); + + schedule.setAccountId(newAccount.getAccountId()); + schedule.setDomainId(newAccount.getDomainId()); + backupScheduleDao.update(schedule.getId(), schedule); + } + } + /** * Attempts to create a network suitable for the creation of a VM ({@link NetworkOrchestrationService#createGuestNetwork}). * If no physical network is found, throws a {@link InvalidParameterValueException}. diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index aff838a3cdf..b78ce450f1d 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -588,7 +588,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); if (schedule == null) { - return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime, maxBackups, cmd.getQuiesceVM())); + return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime, maxBackups, cmd.getQuiesceVM(), vm.getAccountId(), vm.getDomainId())); } schedule.setScheduleType((short) intervalType.ordinal()); @@ -638,13 +638,59 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { return maxBackups; } - @Override - public List listBackupSchedule(final Long vmId) { - final VMInstanceVO vm = findVmById(vmId); - validateBackupForZone(vm.getDataCenterId()); - accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + public List listBackupSchedules(ListBackupScheduleCmd cmd) { + Account caller = CallContext.current().getCallingAccount(); + Long id = cmd.getId(); + Long vmId = cmd.getVmId(); + List permittedAccounts = new ArrayList<>(); + Long domainId = null; + Boolean isRecursive = null; + String keyword = cmd.getKeyword(); + Project.ListProjectResourcesCriteria listProjectResourcesCriteria = null; - return backupScheduleDao.listByVM(vmId).stream().map(BackupSchedule.class::cast).collect(Collectors.toList()); + if (vmId != null) { + final VMInstanceVO vm = findVmById(vmId); + validateBackupForZone(vm.getDataCenterId()); + accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); + } + + Ternary domainIdRecursiveListProject = + new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); + accountManager.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, true, false); + domainId = domainIdRecursiveListProject.first(); + isRecursive = domainIdRecursiveListProject.second(); + listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + + Filter searchFilter = new Filter(BackupScheduleVO.class, "id", false, null, null); + SearchBuilder searchBuilder = backupScheduleDao.createSearchBuilder(); + + accountManager.buildACLSearchBuilder(searchBuilder, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + + searchBuilder.and("id", searchBuilder.entity().getId(), SearchCriteria.Op.EQ); + if (vmId != null) { + searchBuilder.and("vmId", searchBuilder.entity().getVmId(), SearchCriteria.Op.EQ); + } + if (keyword != null && !keyword.isEmpty()) { + SearchBuilder vmSearch = vmInstanceDao.createSearchBuilder(); + vmSearch.and("hostName", vmSearch.entity().getHostName(), SearchCriteria.Op.LIKE); + searchBuilder.join("vmJoin", vmSearch, searchBuilder.entity().getVmId(), vmSearch.entity().getId(), JoinBuilder.JoinType.INNER); + } + + SearchCriteria sc = searchBuilder.create(); + accountManager.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + + if (id != null) { + sc.setParameters("id", id); + } + if (vmId != null) { + sc.setParameters("vmId", vmId); + } + if (keyword != null && !keyword.isEmpty()) { + sc.setJoinParameters("vmJoin", "hostName", "%" + keyword + "%"); + } + + Pair, Integer> result = backupScheduleDao.searchAndCount(sc, searchFilter); + return new ArrayList<>(result.first()); } @Override diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java index f178c6b8912..6dadbfe96eb 100644 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java @@ -27,18 +27,25 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.org.Grouping; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.SnapshotZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; +import com.cloud.user.User; import com.cloud.user.dao.AccountDao; import com.cloud.utils.Pair; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -51,6 +58,8 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.junit.Assert; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -89,9 +98,23 @@ public class SnapshotManagerImplTest { SnapshotZoneDao snapshotZoneDao; @Mock VolumeDao volumeDao; + @Mock + SnapshotPolicyDao snapshotPolicyDao; @InjectMocks SnapshotManagerImpl snapshotManager = new SnapshotManagerImpl(); + @Before + public void setUp() { + snapshotManager._snapshotPolicyDao = snapshotPolicyDao; + snapshotManager._volsDao = volumeDao; + snapshotManager._accountMgr = accountManager; + } + + @After + public void tearDown() { + CallContext.unregister(); + } + @Test public void testGetSnapshotZoneImageStoreValid() { final long snapshotId = 1L; @@ -395,4 +418,106 @@ public class SnapshotManagerImplTest { Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(dataCenterVO); Assert.assertNotNull(snapshotManager.getCheckedDestinationZoneForSnapshotCopy(zoneId, false)); } + + @Test + public void testListSnapshotPolicies() { + long volumeId = 42L; + ListSnapshotPoliciesCmd cmd = Mockito.mock(ListSnapshotPoliciesCmd.class); + Mockito.when(cmd.getVolumeId()).thenReturn(volumeId); + Mockito.when(cmd.getId()).thenReturn(null); + Mockito.when(cmd.getStartIndex()).thenReturn(0L); + Mockito.when(cmd.getPageSizeVal()).thenReturn(10L); + + Account caller = Mockito.mock(Account.class); + Mockito.when(caller.getId()).thenReturn(1L); + CallContext.register(Mockito.mock(User.class), caller); + + SnapshotPolicyVO policy1 = Mockito.mock(SnapshotPolicyVO.class); + SnapshotPolicyVO policy2 = Mockito.mock(SnapshotPolicyVO.class); + List mockPolicies = List.of(policy1, policy2); + + SearchBuilder mockSearchBuilder = Mockito.mock(SearchBuilder.class); + SearchBuilder mockVolumeSearchBuilder = Mockito.mock(SearchBuilder.class); + SearchCriteria mockSearchCriteria = Mockito.mock(SearchCriteria.class); + + Mockito.when(snapshotPolicyDao.createSearchBuilder()).thenReturn(mockSearchBuilder); + Mockito.when(mockSearchBuilder.entity()).thenReturn(Mockito.mock(SnapshotPolicyVO.class)); + Mockito.when(mockSearchBuilder.create()).thenReturn(mockSearchCriteria); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(mockVolumeSearchBuilder); + Mockito.when(mockVolumeSearchBuilder.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + Mockito.when(snapshotPolicyDao.searchAndCount(Mockito.any(), Mockito.any())).thenReturn(new Pair<>(mockPolicies, 2)); + + Pair, Integer> result = snapshotManager.listSnapshotPolicies(cmd); + + Assert.assertNotNull(result); + Assert.assertEquals(2, result.first().size()); + Assert.assertEquals(Integer.valueOf(2), result.second()); + Assert.assertEquals(mockPolicies, result.first()); + } + + @Test + public void testListSnapshotPolicies_NonRootAdmin() { + ListSnapshotPoliciesCmd cmd = Mockito.mock(ListSnapshotPoliciesCmd.class); + Mockito.when(cmd.getVolumeId()).thenReturn(1L); + Mockito.when(cmd.getId()).thenReturn(null); + Mockito.when(cmd.getStartIndex()).thenReturn(0L); + Mockito.when(cmd.getPageSizeVal()).thenReturn(10L); + + Account caller = Mockito.mock(Account.class); + Mockito.when(caller.getId()).thenReturn(2L); + CallContext.register(Mockito.mock(User.class), caller); + + SnapshotPolicyVO policy1 = Mockito.mock(SnapshotPolicyVO.class); + SnapshotPolicyVO policy2 = Mockito.mock(SnapshotPolicyVO.class); + List mockPolicies = List.of(policy1, policy2); + + SearchBuilder mockSearchBuilder = Mockito.mock(SearchBuilder.class); + SearchBuilder mockVolumeSearchBuilder = Mockito.mock(SearchBuilder.class); + SearchCriteria mockSearchCriteria = Mockito.mock(SearchCriteria.class); + + Mockito.when(snapshotPolicyDao.createSearchBuilder()).thenReturn(mockSearchBuilder); + Mockito.when(mockSearchBuilder.entity()).thenReturn(Mockito.mock(SnapshotPolicyVO.class)); + Mockito.when(mockSearchBuilder.create()).thenReturn(mockSearchCriteria); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(mockVolumeSearchBuilder); + Mockito.when(mockVolumeSearchBuilder.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + Mockito.when(snapshotPolicyDao.searchAndCount(Mockito.any(), Mockito.any())).thenReturn(new Pair<>(mockPolicies, 2)); + + Pair, Integer> result = snapshotManager.listSnapshotPolicies(cmd); + + Assert.assertNotNull(result); + Assert.assertEquals(2, result.first().size()); + Assert.assertEquals(Integer.valueOf(2), result.second()); + Assert.assertEquals(mockPolicies, result.first()); + } + + @Test + public void testListSnapshotPolicies_RootAdmin() { + ListSnapshotPoliciesCmd cmd = Mockito.mock(ListSnapshotPoliciesCmd.class); + Mockito.when(cmd.getVolumeId()).thenReturn(1L); + Mockito.when(cmd.getId()).thenReturn(null); + Mockito.when(cmd.getStartIndex()).thenReturn(0L); + Mockito.when(cmd.getPageSizeVal()).thenReturn(10L); + + Account caller = Mockito.mock(Account.class); + Mockito.when(caller.getId()).thenReturn(1L); + CallContext.register(Mockito.mock(User.class), caller); + + SnapshotPolicyVO policy = Mockito.mock(SnapshotPolicyVO.class); + SearchBuilder mockSearchBuilder = Mockito.mock(SearchBuilder.class); + SearchBuilder mockVolumeSearchBuilder = Mockito.mock(SearchBuilder.class); + SearchCriteria mockSearchCriteria = Mockito.mock(SearchCriteria.class); + + Mockito.when(snapshotPolicyDao.createSearchBuilder()).thenReturn(mockSearchBuilder); + Mockito.when(mockSearchBuilder.entity()).thenReturn(Mockito.mock(SnapshotPolicyVO.class)); + Mockito.when(mockSearchBuilder.create()).thenReturn(mockSearchCriteria); + Mockito.when(volumeDao.createSearchBuilder()).thenReturn(mockVolumeSearchBuilder); + Mockito.when(mockVolumeSearchBuilder.entity()).thenReturn(Mockito.mock(VolumeVO.class)); + Mockito.when(snapshotPolicyDao.searchAndCount(Mockito.any(), Mockito.any())).thenReturn(new Pair<>(List.of(policy), 1)); + + Pair, Integer> result = snapshotManager.listSnapshotPolicies(cmd); + + Assert.assertNotNull(result); + Assert.assertEquals(1, result.first().size()); + Assert.assertEquals(Integer.valueOf(1), result.second()); + } } diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java index 4d802319935..3b5d92103e7 100755 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerTest.java @@ -209,6 +209,8 @@ public class SnapshotManagerTest { private static final String TEST_SNAPSHOT_POLICY_TIMEZONE = ""; private static final IntervalType TEST_SNAPSHOT_POLICY_INTERVAL = IntervalType.MONTHLY; private static final int TEST_SNAPSHOT_POLICY_MAX_SNAPS = 1; + private static final long TEST_SNAPSHOT_POLICY_ACCOUNT_ID = 1; + private static final long TEST_SNAPSHOT_POLICY_DOMAIN_ID = 1; private static final boolean TEST_SNAPSHOT_POLICY_DISPLAY = true; private static final boolean TEST_SNAPSHOT_POLICY_ACTIVE = true; private static final long TEST_ZONE_ID = 7L; @@ -251,7 +253,7 @@ public class SnapshotManagerTest { when(_resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(any(HypervisorType.class), anyLong())).thenReturn(null); snapshotPolicyVoInstance = new SnapshotPolicyVO(TEST_VOLUME_ID, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY); + TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_ACCOUNT_ID, TEST_SNAPSHOT_POLICY_DOMAIN_ID, TEST_SNAPSHOT_POLICY_DISPLAY); apiDBUtilsMock = Mockito.mockStatic(ApiDBUtils.class); } @@ -442,7 +444,7 @@ public class SnapshotManagerTest { Mockito.doReturn(true).when(taggedResourceServiceMock).deleteTags(any(), any(), any()); SnapshotPolicyVO snapshotPolicyVo = new SnapshotPolicyVO(TEST_VOLUME_ID, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, - TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY); + TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_ACCOUNT_ID, TEST_SNAPSHOT_POLICY_DOMAIN_ID, TEST_SNAPSHOT_POLICY_DISPLAY); _snapshotMgr.updateSnapshotPolicy(snapshotPolicyVo, TEST_SNAPSHOT_POLICY_SCHEDULE, TEST_SNAPSHOT_POLICY_TIMEZONE, TEST_SNAPSHOT_POLICY_INTERVAL, TEST_SNAPSHOT_POLICY_MAX_SNAPS, TEST_SNAPSHOT_POLICY_DISPLAY, TEST_SNAPSHOT_POLICY_ACTIVE, null, null); diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 360800fac34..a21477aeb80 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -59,6 +59,7 @@ import java.util.Map; import java.util.TimeZone; import java.util.UUID; +import com.cloud.storage.dao.SnapshotPolicyDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -79,6 +80,7 @@ import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.backup.BackupVO; import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; @@ -435,6 +437,13 @@ public class UserVmManagerImplTest { @Mock private UUIDManager uuidMgr; + + @Mock + private SnapshotPolicyDao snapshotPolicyDao; + + @Mock + private BackupScheduleDao backupScheduleDao; + MockedStatic unmanagedVMsManagerMockedStatic; private static final long vmId = 1l; diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 5ee08d185bd..c9391211fac 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -60,6 +60,8 @@ import com.cloud.user.User; import com.cloud.user.dao.AccountDao; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VMInstanceDetailVO; @@ -77,6 +79,7 @@ import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; +import org.apache.cloudstack.api.command.user.backup.ListBackupScheduleCmd; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupDetailsDao; @@ -1815,6 +1818,78 @@ public class BackupManagerTest { ); } + @Test + public void testListBackupSchedulesAsRootAdmin() { + long vmId = 1L; + ListBackupScheduleCmd cmd = Mockito.mock(ListBackupScheduleCmd.class); + Mockito.when(cmd.getVmId()).thenReturn(vmId); + Mockito.when(cmd.getId()).thenReturn(1L); + + // Mock VM for validation + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vmInstanceDao.findById(vmId)).thenReturn(vm); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + overrideBackupFrameworkConfigValue(); + Mockito.doNothing().when(accountManager).checkAccess(Mockito.any(), Mockito.any(), Mockito.anyBoolean(), Mockito.any()); + + BackupScheduleVO schedule1 = Mockito.mock(BackupScheduleVO.class); + BackupScheduleVO schedule2 = Mockito.mock(BackupScheduleVO.class); + List schedules = List.of(schedule1, schedule2); + + SearchBuilder searchBuilder = Mockito.mock(SearchBuilder.class); + SearchCriteria searchCriteria = Mockito.mock(SearchCriteria.class); + BackupScheduleVO entity = Mockito.mock(BackupScheduleVO.class); + + Mockito.when(backupScheduleDao.createSearchBuilder()).thenReturn(searchBuilder); + Mockito.when(searchBuilder.entity()).thenReturn(entity); + Mockito.when(searchBuilder.and(Mockito.anyString(), (Object) any(), Mockito.any())).thenReturn(searchBuilder); + Mockito.when(searchBuilder.create()).thenReturn(searchCriteria); + + Mockito.when(backupScheduleDao.searchAndCount(Mockito.any(), Mockito.any())).thenReturn(new Pair<>(schedules, schedules.size())); + List result = backupManager.listBackupSchedules(cmd); + + assertEquals(2, result.size()); + assertTrue(result.contains(schedule1)); + assertTrue(result.contains(schedule2)); + } + + @Test + public void testListBackupSchedulesAsNonAdmin() { + long vmId = 1L; + ListBackupScheduleCmd cmd = Mockito.mock(ListBackupScheduleCmd.class); + Mockito.when(cmd.getVmId()).thenReturn(vmId); + Mockito.when(cmd.getId()).thenReturn(1L); + + // Mock VM for validation + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vmInstanceDao.findById(vmId)).thenReturn(vm); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + overrideBackupFrameworkConfigValue(); + Mockito.doNothing().when(accountManager).checkAccess(Mockito.any(), Mockito.any(), Mockito.anyBoolean(), Mockito.any()); + + BackupScheduleVO schedule = Mockito.mock(BackupScheduleVO.class); + List schedules = List.of(schedule); + + SearchBuilder searchBuilder = Mockito.mock(SearchBuilder.class); + SearchCriteria searchCriteria = Mockito.mock(SearchCriteria.class); + BackupScheduleVO entity = Mockito.mock(BackupScheduleVO.class); + + Mockito.when(backupScheduleDao.createSearchBuilder()).thenReturn(searchBuilder); + Mockito.when(searchBuilder.create()).thenReturn(searchCriteria); + Mockito.when(searchBuilder.entity()).thenReturn(entity); + Mockito.when(searchBuilder.and(Mockito.anyString(), (Object) any(), Mockito.any())).thenReturn(searchBuilder); + Mockito.lenient().when(backupScheduleDao.search(Mockito.eq(searchCriteria), Mockito.any())).thenReturn(schedules); + + Mockito.doNothing().when(accountManager).buildACLSearchBuilder(Mockito.any(), Mockito.anyLong(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any()); + Mockito.doNothing().when(accountManager).buildACLSearchCriteria(Mockito.any(), Mockito.anyLong(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any()); + + Mockito.when(backupScheduleDao.searchAndCount(Mockito.any(), Mockito.any())).thenReturn(new Pair<>(schedules, schedules.size())); + List result = backupManager.listBackupSchedules(cmd); + + assertEquals(1, result.size()); + assertTrue(result.contains(schedule)); + } + @Test public void testCanCreateInstanceFromBackupAcrossZonesSuccess() { Long backupId = 1L; diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index c372076e0f3..3d00580dceb 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -78,6 +78,8 @@ "label.action.copy.iso": "Copy ISO", "label.action.copy.snapshot": "Copy Snapshot", "label.action.copy.template": "Copy Template", +"label.action.create.backup.schedule": "Create Backup Schedule", +"label.action.create.recurring.snapshot": "Create Recurring Snapshot", "label.action.create.snapshot.from.vmsnapshot": "Create Snapshot from Instance Snapshot", "label.action.create.template.from.volume": "Create Template from volume", "label.action.create.volume": "Create Volume", @@ -455,6 +457,7 @@ "label.backup.restore": "Restore Instance backup", "label.backup.schedule.create.failed": "Failed to create Backup Schedule", "label.backuplimit": "Backup Limits", +"label.backup.schedules": "Backup Schedules", "label.backup.storage": "Backup Storage", "label.backupstoragelimit": "Backup Storage Limits (GiB)", "label.backupofferingid": "Backup Offering ID", @@ -749,6 +752,7 @@ "label.delete.asnrange": "Delete AS Range", "label.delete.autoscale.vmgroup": "Delete AutoScaling Group", "label.delete.backup": "Delete backup", +"label.delete.backup.schedule": "Delete backup schedule", "label.delete.bgp.peer": "Delete BGP peer", "label.delete.bigswitchbcf": "Remove BigSwitch BCF controller", "label.delete.brocadevcs": "Remove Brocade Vcs switch", @@ -1510,7 +1514,7 @@ "label.max.primary.storage": "Max. primary (GiB)", "label.max.secondary.storage": "Max. secondary (GiB)", "label.max.migrations": "Max. migrations", -"label.maxbackup": "Max. Backups", +"label.maxbackups": "Max. Backups", "label.maxbackupstorage": "Max. Backup Storage (GiB)", "label.maxbackups.to.retain": "Max. Backups to retain", "label.maxbucket": "Max. Buckets", @@ -1536,6 +1540,7 @@ "label.maxresolutiony": "Max. resolution Y", "label.maxsecondarystorage": "Max. secondary storage (GiB)", "label.maxsize": "Maximum size", +"label.maxsnaps": "Max. Snapshots", "label.maxsnapshot": "Max. Snapshots", "label.maxtemplate": "Max. Templates", "label.maxuservm": "Max. User Instances", @@ -2214,6 +2219,8 @@ "label.select.root.disk": "Select the ROOT disk", "label.select.source.vcenter.datacenter": "Select the source VMware vCenter Datacenter", "label.select.tier": "Select Network Tier", +"label.select.vm": "Select Instance", +"label.select.volume": "Select Volume", "label.select.zones": "Select zones", "label.select.storagepools": "Select storage pools", "label.select.2fa.provider": "Select the provider", @@ -2284,6 +2291,8 @@ "label.snapshot.name": "Snapshot name", "label.snapshotlimit": "Snapshot limits", "label.snapshotmemory": "Snapshot memory", +"label.snapshotpolicy": "Snapshot policy", +"label.snapshotpolicies": "Snapshot policies", "label.snapshots": "Volume Snapshots", "label.snapshottype": "Snapshot Type", "label.sockettimeout": "Socket timeout", @@ -2880,6 +2889,7 @@ "message.action.delete.autoscale.vmgroup": "Please confirm that you want to delete this autoscaling group.", "message.action.delete.backup.offering": "Please confirm that you want to delete this backup offering?", "message.action.delete.backup.repository": "Please confirm that you want to delete this backup repository?", +"message.action.delete.backup.schedule": "Please confirm that you want to delete this backup schedule?", "message.action.delete.cluster": "Please confirm that you want to delete this Cluster.", "message.action.delete.custom.action": "Please confirm that you want to delete this custom action.", "message.action.delete.domain": "Please confirm that you want to delete this domain.", @@ -2905,6 +2915,7 @@ "message.action.delete.secondary.storage": "Please confirm that you want to delete this secondary storage.", "message.action.delete.security.group": "Please confirm that you want to delete this security group.", "message.action.delete.snapshot": "Please confirm that you want to delete this Snapshot.", +"message.action.delete.snapshot.policy": "Please confirm that you want to delete the selected Snapshot Policy.", "message.action.delete.template": "Please confirm that you want to delete this Template.", "message.action.delete.tungsten.router.table": "Please confirm that you want to remove Route Table from this Network?", "message.action.delete.vgpu.profile": "Please confirm that you want to delete this vGPU profile.", @@ -3731,6 +3742,8 @@ "message.select.security.groups": "Please select security group(s) for your new Instance.", "message.select.start.date.and.time": "Select a start date & time.", "message.select.temporary.storage.instance.conversion": "(Optional) Select a Storage temporary destination for the converted disks through virt-v2v", +"message.select.volume.to.continue": "Please select a volume to continue.", +"message.select.vm.to.continue": "Please select an Instance to continue.", "message.select.zone.description": "Select type of Zone basic/advanced.", "message.select.zone.hint": "This is the type of Zone deployment that you want to use. Basic zone: provides a single Network where each Instance is assigned an IP directly from the Network. Guest isolation can be provided through layer-3 means such as security groups (IP address source filtering). Advanced zone: For more sophisticated Network topologies. This Network model provides the most flexibility in defining guest Networks and providing custom Network offerings such as firewall, VPN, or load balancer support.", "message.server": "Server : ", diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 7829121fdb2..135ea7384fa 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -161,6 +161,13 @@
{{ dataResource[item] }}
+ +
+ {{ $t('label.' + String(item).toLowerCase()) }} +
+
{{ dataResource[item] }}
+
+
{{ $t('label.' + item.replace('date', '.date.and.time'))}} diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 141dfd69530..47aa3d2ddef 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -233,11 +233,49 @@ >{{ $t(text.toLowerCase()) }} {{ text }} - + +