From fd4223295a05878fbfeef7a9c1a00dd380de351b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 30 Jul 2025 10:56:33 +0530 Subject: [PATCH 01/53] ui: fix advance setting behaviour in autoscale form (#11306) Fixes #11269 The current dysfunctional behaviour was introduced in #6571. In advanced settings interface for ssh keypairs, userdata, affinity group, etc are show but the toggle to show/hide them was not working correctly. Signed-off-by: Abhishek Kumar --- ui/src/views/compute/CreateAutoScaleVmGroup.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/views/compute/CreateAutoScaleVmGroup.vue b/ui/src/views/compute/CreateAutoScaleVmGroup.vue index 981c2b0ab18..ae295e7f94f 100644 --- a/ui/src/views/compute/CreateAutoScaleVmGroup.vue +++ b/ui/src/views/compute/CreateAutoScaleVmGroup.vue @@ -738,7 +738,7 @@ {{ $t('label.isadvanced') }} -
+
Date: Wed, 30 Jul 2025 11:04:58 +0530 Subject: [PATCH 02/53] kvm, ui: fix interface when using vlan subnet for storage traffic type (#11245) * kvm, ui: fix interface when using vlan subnet for storage traffic type Fixes #7816 Signed-off-by: Abhishek Kumar --- .../cloud/hypervisor/kvm/resource/BridgeVifDriver.java | 9 +++++++++ ui/src/views/infra/network/IpRangesTabStorage.vue | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 39ecc9182f0..67f093a933a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -252,6 +252,15 @@ public class BridgeVifDriver extends VifDriverBase { intf.defBridgeNet(_bridges.get("private"), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter)); } else if (nic.getType() == Networks.TrafficType.Storage) { String storageBrName = nic.getName() == null ? _bridges.get("private") : nic.getName(); + if (nic.getBroadcastType() == Networks.BroadcastDomainType.Storage) { + vNetId = Networks.BroadcastDomainType.getValue(nic.getBroadcastUri()); + protocol = Networks.BroadcastDomainType.Vlan.scheme(); + } + if (isValidProtocolAndVnetId(vNetId, protocol)) { + s_logger.debug(String.format("creating a vNet dev and bridge for %s traffic per traffic label %s", + Networks.TrafficType.Storage.name(), trafficLabel)); + storageBrName = createVnetBr(vNetId, storageBrName, protocol); + } intf.defBridgeNet(storageBrName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter)); } if (nic.getPxeDisable()) { diff --git a/ui/src/views/infra/network/IpRangesTabStorage.vue b/ui/src/views/infra/network/IpRangesTabStorage.vue index c8665d6d9b5..881f9ceadea 100644 --- a/ui/src/views/infra/network/IpRangesTabStorage.vue +++ b/ui/src/views/infra/network/IpRangesTabStorage.vue @@ -166,7 +166,7 @@ export default { }, { title: this.$t('label.vlan'), - dataIndex: 'vlanid' + dataIndex: 'vlan' }, { title: this.$t('label.startip'), From 8497f70b46c8d0463b0d0619f551e909204dae6a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 30 Jul 2025 11:07:38 +0530 Subject: [PATCH 03/53] ui: make events tab selected columns persistent using cache (#11317) Fixes #10308 Signed-off-by: Abhishek Kumar --- ui/src/components/view/EventsTab.vue | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ui/src/components/view/EventsTab.vue b/ui/src/components/view/EventsTab.vue index c3a6ac5c17f..59080a4988a 100644 --- a/ui/src/components/view/EventsTab.vue +++ b/ui/src/components/view/EventsTab.vue @@ -52,6 +52,8 @@ import { getAPI } from '@/api' import { genericCompare } from '@/utils/sort.js' import ListView from '@/components/view/ListView' +const EVENTS_TAB_COLUMNS_KEY = 'events_tab_columns' + export default { name: 'EventsTab', components: { @@ -98,8 +100,7 @@ export default { } }, created () { - this.selectedColumnKeys = this.columnKeys - this.updateSelectedColumns('description') + this.setDefaultColumns() this.pageSize = this.pageSizeOptions[0] * 1 this.fetchData() }, @@ -111,6 +112,15 @@ export default { } }, methods: { + setDefaultColumns () { + const savedColumns = this.$localStorage.get(EVENTS_TAB_COLUMNS_KEY) + if (savedColumns && Array.isArray(savedColumns) && savedColumns.length > 0) { + this.selectedColumnKeys = savedColumns + } else { + this.selectedColumnKeys = this.columnKeys.filter(x => x !== 'description') + } + this.updateColumns() + }, fetchData () { this.fetchEvents() }, @@ -145,6 +155,7 @@ export default { } else { this.selectedColumnKeys.push(key) } + this.$localStorage.set(EVENTS_TAB_COLUMNS_KEY, this.selectedColumnKeys) this.updateColumns() }, updateColumns () { From 4d2beea7773d739e7946d75472e4ccfc57a05d69 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 30 Jul 2025 14:50:41 +0530 Subject: [PATCH 04/53] logger fix --- .../java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 2c4aafd4c8c..0602fd6322f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -255,7 +255,7 @@ public class BridgeVifDriver extends VifDriverBase { protocol = Networks.BroadcastDomainType.Vlan.scheme(); } if (isValidProtocolAndVnetId(vNetId, protocol)) { - s_logger.debug(String.format("creating a vNet dev and bridge for %s traffic per traffic label %s", + logger.debug(String.format("creating a vNet dev and bridge for %s traffic per traffic label %s", Networks.TrafficType.Storage.name(), trafficLabel)); storageBrName = createVnetBr(vNetId, storageBrName, protocol); } From 2d025bd0749026afc4a7f71b161383a17fc59a8b Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 30 Jul 2025 17:03:10 +0530 Subject: [PATCH 05/53] kvm: fix regression 5a52ca78ae5e165211c618525613c3d62cfd1b28 (#11342) Somehow the commit 5a52ca78ae5e165211c618525613c3d62cfd1b28 was reverted so cloud-init templates don't work on arm64 anymore :( Signed-off-by: Rohit Yadav --- .../java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 93ad084b437..1383190933d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -249,9 +249,7 @@ public class LibvirtVMDef { guestDef.append("\n"); } } - if (_arch == null || !_arch.equals("aarch64")) { - guestDef.append("\n"); - } + guestDef.append("\n"); guestDef.append("\n"); if (iothreads) { guestDef.append(String.format("%s", NUMBER_OF_IOTHREADS)); From 294aef5ecff1356976eff819cc3028f3439faa24 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:35:03 +0530 Subject: [PATCH 06/53] Fix listCapacity sort by usage (#11316) --- .../api/command/admin/resource/ListCapacityCmd.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java index 253677616f0..fdc1087377f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java @@ -127,14 +127,16 @@ public class ListCapacityCmd extends BaseListCmd { Collections.sort(capacityResponses, new Comparator() { public int compare(CapacityResponse resp1, CapacityResponse resp2) { int res = resp1.getZoneName().compareTo(resp2.getZoneName()); + // Group by zone if (res != 0) { return res; - } else { - return resp1.getCapacityType().compareTo(resp2.getCapacityType()); } + // Sort by capacity type only if not already sorted by usage + return (getSortBy() != null) ? 0 : resp1.getCapacityType().compareTo(resp2.getCapacityType()); } }); + response.setResponses(capacityResponses); response.setResponseName(getCommandName()); this.setResponseObject(response); From 1dc134a3ecf645b743334213aeb35530b8e9c2de Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Wed, 30 Jul 2025 08:11:13 -0400 Subject: [PATCH 07/53] UI: Display NSX Provider only when NSX is the selected Isolation method (#11142) --- .../com/cloud/network/NetworkServiceImpl.java | 1 - ui/src/views/infra/zone/ZoneWizardLaunchZone.vue | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index ccafc2da258..254c03c9317 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -5658,7 +5658,6 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C } addProviderToPhysicalNetwork(physicalNetworkId, Provider.Nsx.getName(), null, null); - enableProvider(Provider.Nsx.getName()); } return null; } diff --git a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue index e479246777f..31e3fadc9f6 100644 --- a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue +++ b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue @@ -487,7 +487,6 @@ export default { if (physicalNetwork.isolationMethod === 'NSX' && physicalNetwork.traffics.findIndex(traffic => traffic.type === 'public' || traffic.type === 'guest') > -1) { this.stepData.isNsxZone = true - this.stepData.tungstenPhysicalNetworkId = physicalNetworkReturned.id } } else { this.stepData.physicalNetworkReturned = this.stepData.physicalNetworkItem['createPhysicalNetwork' + index] @@ -980,7 +979,7 @@ export default { return } - if (idx === 0) { + if (idx === 0 && this.stepData.isNsxZone) { await this.stepConfigurePublicTraffic('message.configuring.nsx.public.traffic', 'nsxPublicTraffic', 1) } else { if (this.stepData.isTungstenZone) { @@ -1080,6 +1079,7 @@ export default { providerParams.transportzone = this.prefillContent?.transportZone || '' await this.addNsxController(providerParams) + await this.updateNsxServiceProviderStatus() this.stepData.stepMove.push('addNsxController') } this.stepData.stepMove.push('nsx') @@ -1090,6 +1090,18 @@ export default { this.setStepStatus(STATUS_FAILED) } }, + async updateNsxServiceProviderStatus () { + const listParams = {} + listParams.name = 'Nsx' + const nsxPhysicalNetwork = this.stepData.physicalNetworksReturned.find(net => net.isolationmethods.trim().toUpperCase() === 'NSX') + const nsxPhysicalNetworkId = nsxPhysicalNetwork?.id || null + listParams.physicalNetworkId = nsxPhysicalNetworkId + const nsxProviderId = await this.listNetworkServiceProviders(listParams, 'nsxProvider') + console.log(nsxProviderId) + if (nsxProviderId !== null) { + await this.updateNetworkServiceProvider(nsxProviderId) + } + }, async stepConfigureStorageTraffic () { let targetNetwork = false this.prefillContent.physicalNetworks.forEach(physicalNetwork => { From f73cb5621df6819453ec4002134610ae947ccefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernardo=20De=20Marco=20Gon=C3=A7alves?= Date: Wed, 30 Jul 2025 09:42:04 -0300 Subject: [PATCH 08/53] Refactoring retention of backup schedules (#11223) * refactor backup schedule retention workflows Co-authored-by: Fabricio Duarte --- .../apache/cloudstack/api/ApiConstants.java | 4 + .../command/user/backup/CreateBackupCmd.java | 19 +- .../user/backup/CreateBackupScheduleCmd.java | 6 +- .../org/apache/cloudstack/backup/Backup.java | 23 +- .../cloudstack/backup/BackupManager.java | 36 +- .../cloudstack/backup/BackupSchedule.java | 2 +- .../cloudstack/backup/BackupScheduleVO.java | 8 +- .../apache/cloudstack/backup/BackupVO.java | 23 +- .../cloudstack/backup/dao/BackupDao.java | 3 +- .../cloudstack/backup/dao/BackupDaoImpl.java | 23 +- .../META-INF/db/schema-42010to42100.sql | 5 +- .../ParamGenericValidationWorker.java | 1 + .../cloudstack/backup/BackupManagerImpl.java | 240 +++++++----- .../cloudstack/backup/BackupManagerTest.java | 361 ++++++++++++++---- ui/src/views/compute/backup/FormSchedule.vue | 2 +- 15 files changed, 466 insertions(+), 290 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4fef598d311..a8e45c8112e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1307,6 +1307,10 @@ public class ApiConstants { "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " + "added, it will be interpreted as \"23:59:59\"). If the recommended format is not used, the date will be considered in the server timezone."; + public static final String PARAMETER_DESCRIPTION_MAX_BACKUPS = "The maximum number of backups to keep for a VM. " + + "If \"0\", no retention policy will be applied and, thus, no backups from the schedule will be automatically deleted. " + + "This parameter is only supported for the Dummy, NAS and EMC Networker backup provider."; + public static final String VMWARE_DC = "vmwaredc"; public static final String CSS = "css"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index 2d387788243..af25f873f87 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.api.command.user.backup; import javax.inject.Inject; -import com.cloud.storage.Snapshot; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -28,7 +27,6 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; @@ -62,13 +60,6 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { description = "ID of the VM") private Long vmId; - @Parameter(name = ApiConstants.SCHEDULE_ID, - type = CommandType.LONG, - entityType = BackupScheduleResponse.class, - description = "backup schedule ID of the VM, if this is null, it indicates that it is a manual backup.", - since = "4.21.0") - private Long scheduleId; - ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -77,14 +68,6 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { return vmId; } - public Long getScheduleId() { - if (scheduleId != null) { - return scheduleId; - } else { - return Snapshot.MANUAL_POLICY_ID; - } - } - ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -92,7 +75,7 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try { - boolean result = backupManager.createBackup(getVmId(), getScheduleId()); + boolean result = backupManager.createBackup(getVmId(), getJob()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 1d0741e6217..f9903eaf1ec 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -75,10 +75,8 @@ public class CreateBackupScheduleCmd extends BaseCmd { description = "Specifies a timezone for this command. For more information on the timezone parameter, see TimeZone Format.") private String timezone; - @Parameter(name = ApiConstants.MAX_BACKUPS, - type = CommandType.INTEGER, - description = "maximum number of backups to retain", - since = "4.21.0") + @Parameter(name = ApiConstants.MAX_BACKUPS, type = CommandType.INTEGER, + since = "4.21.0", description = ApiConstants.PARAMETER_DESCRIPTION_MAX_BACKUPS) private Integer maxBackups; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index dffe8a03213..53ac0ae960e 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -33,28 +33,6 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged } - public enum Type { - MANUAL, HOURLY, DAILY, WEEKLY, MONTHLY; - private int max = 8; - - public void setMax(int max) { - this.max = max; - } - - public int getMax() { - return max; - } - - @Override - public String toString() { - return this.name(); - } - - public boolean equals(String snapshotType) { - return this.toString().equalsIgnoreCase(snapshotType); - } - } - class Metric { private Long backupSize = 0L; private Long dataSize = 0L; @@ -166,4 +144,5 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { Long getProtectedSize(); List getBackedUpVolumes(); long getZoneId(); + Long getBackupScheduleId(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index eebad3af067..2ed4d5af5d1 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -58,38 +58,6 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer "false", "Enable volume attach/detach operations for VMs that are assigned to Backup Offerings.", true); - ConfigKey BackupHourlyMax = new ConfigKey("Advanced", Integer.class, - "backup.max.hourly", - "8", - "Maximum recurring hourly backups to be retained for an instance. If the limit is reached, early backups from the start of the hour are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring hourly backups can not be scheduled.", - false, - ConfigKey.Scope.Global, - null); - - ConfigKey BackupDailyMax = new ConfigKey("Advanced", Integer.class, - "backup.max.daily", - "8", - "Maximum recurring daily backups to be retained for an instance. If the limit is reached, backups from the start of the day are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring daily backups can not be scheduled.", - false, - ConfigKey.Scope.Global, - null); - - ConfigKey BackupWeeklyMax = new ConfigKey("Advanced", Integer.class, - "backup.max.weekly", - "8", - "Maximum recurring weekly backups to be retained for an instance. If the limit is reached, backups from the beginning of the week are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring weekly backups can not be scheduled.", - false, - ConfigKey.Scope.Global, - null); - - ConfigKey BackupMonthlyMax = new ConfigKey("Advanced", Integer.class, - "backup.max.monthly", - "8", - "Maximum recurring monthly backups to be retained for an instance. If the limit is reached, backups from the beginning of the month are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring monthly backups can not be scheduled.", - false, - ConfigKey.Scope.Global, - null); - ConfigKey DefaultMaxAccountBackups = new ConfigKey("Account Defaults", Long.class, "max.account.backups", "20", @@ -201,10 +169,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer /** * Creates backup of a VM * @param vmId Virtual Machine ID - * @param scheduleId Virtual Machine Backup Schedule ID + * @param job The async job associated with the backup retention * @return returns operation success */ - boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException; + boolean createBackup(final Long vmId, Object job) throws ResourceAllocationException; /** * List existing backups for a VM diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java index f439b3a9139..26adc80db37 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java @@ -30,6 +30,6 @@ public interface BackupSchedule extends InternalIdentity { String getTimezone(); Date getScheduledTimestamp(); Long getAsyncJobId(); - Integer getMaxBackups(); + int getMaxBackups(); String getUuid(); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java index 75c7a8be55c..06e1dcfb1ed 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java @@ -63,12 +63,12 @@ public class BackupScheduleVO implements BackupSchedule { Long asyncJobId; @Column(name = "max_backups") - Integer maxBackups = 0; + private int maxBackups = 0; public BackupScheduleVO() { } - public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, Integer maxBackups) { + public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, int maxBackups) { this.vmId = vmId; this.scheduleType = (short) scheduleType.ordinal(); this.schedule = schedule; @@ -142,11 +142,11 @@ public class BackupScheduleVO implements BackupSchedule { this.asyncJobId = asyncJobId; } - public Integer getMaxBackups() { + public int getMaxBackups() { return maxBackups; } - public void setMaxBackups(Integer maxBackups) { + public void setMaxBackups(int maxBackups) { this.maxBackups = maxBackups; } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 9ef442baff9..40bf9712137 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -88,12 +88,12 @@ public class BackupVO implements Backup { @Column(name = "zone_id") private long zoneId; - @Column(name = "backup_interval_type") - private short backupIntervalType; - @Column(name = "backed_volumes", length = 65535) protected String backedUpVolumes; + @Column(name = "backup_schedule_id") + private Long backupScheduleId; + public BackupVO() { this.uuid = UUID.randomUUID().toString(); } @@ -211,14 +211,6 @@ public class BackupVO implements Backup { this.zoneId = zoneId; } - public short getBackupIntervalType() { - return backupIntervalType; - } - - public void setBackupIntervalType(short backupIntervalType) { - this.backupIntervalType = backupIntervalType; - } - @Override public Class getEntityType() { return Backup.class; @@ -247,4 +239,13 @@ public class BackupVO implements Backup { public void setRemoved(Date removed) { this.removed = removed; } + + @Override + public Long getBackupScheduleId() { + return backupScheduleId; + } + + public void setBackupScheduleId(Long backupScheduleId) { + this.backupScheduleId = backupScheduleId; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java index ffd5e5a4a66..64b8d5da5a9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java @@ -36,9 +36,8 @@ public interface BackupDao extends GenericDao { BackupVO getBackupVO(Backup backup); List listByOfferingId(Long backupOfferingId); - List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType); - BackupResponse newBackupResponse(Backup backup); public Long countBackupsForAccount(long accountId); public Long calculateBackupStorageForAccount(long accountId); + List listBySchedule(Long backupScheduleId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index b4e1a760282..0e8d8242f9c 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -24,6 +24,7 @@ import java.util.Objects; import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericSearchBuilder; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.Backup; @@ -63,7 +64,7 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac private SearchBuilder backupSearch; private GenericSearchBuilder CountBackupsByAccount; private GenericSearchBuilder CalculateBackupStorageByAccount; - private SearchBuilder ListBackupsByVMandIntervalType; + private SearchBuilder listBackupsBySchedule; public BackupDaoImpl() { } @@ -91,12 +92,11 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac CalculateBackupStorageByAccount.and("removed", CalculateBackupStorageByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); CalculateBackupStorageByAccount.done(); - ListBackupsByVMandIntervalType = createSearchBuilder(); - ListBackupsByVMandIntervalType.and("vmId", ListBackupsByVMandIntervalType.entity().getVmId(), SearchCriteria.Op.EQ); - ListBackupsByVMandIntervalType.and("intervalType", ListBackupsByVMandIntervalType.entity().getBackupIntervalType(), SearchCriteria.Op.EQ); - ListBackupsByVMandIntervalType.and("status", ListBackupsByVMandIntervalType.entity().getStatus(), SearchCriteria.Op.EQ); - ListBackupsByVMandIntervalType.and("removed", ListBackupsByVMandIntervalType.entity().getRemoved(), SearchCriteria.Op.NULL); - ListBackupsByVMandIntervalType.done(); + listBackupsBySchedule = createSearchBuilder(); + listBackupsBySchedule.and("backup_schedule_id", listBackupsBySchedule.entity().getBackupScheduleId(), SearchCriteria.Op.EQ); + listBackupsBySchedule.and("status", listBackupsBySchedule.entity().getStatus(), SearchCriteria.Op.EQ); + listBackupsBySchedule.and("removed", listBackupsBySchedule.entity().getRemoved(), SearchCriteria.Op.NULL); + listBackupsBySchedule.done(); } @Override @@ -184,12 +184,11 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac } @Override - public List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType) { - SearchCriteria sc = ListBackupsByVMandIntervalType.create(); - sc.setParameters("vmId", vmId); - sc.setParameters("type", backupType.ordinal()); + public List listBySchedule(Long backupScheduleId) { + SearchCriteria sc = listBackupsBySchedule.create(); + sc.setParameters("backup_schedule_id", backupScheduleId); sc.setParameters("status", Backup.Status.BackedUp); - return listBy(sc, null); + return listBy(sc, new Filter(BackupVO.class, "date", true)); } @Override diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4c7fe74cbcd..4fed45c91fb 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -19,9 +19,8 @@ -- Schema upgrade from 4.20.1.0 to 4.21.0.0 --; --- Add columns max_backup and backup_interval_type to backup table -ALTER TABLE `cloud`.`backup_schedule` ADD COLUMN `max_backups` int(8) default NULL COMMENT 'maximum number of backups to maintain'; -ALTER TABLE `cloud`.`backups` ADD COLUMN `backup_interval_type` int(5) COMMENT 'type of backup, e.g. manual, recurring - hourly, daily, weekly or monthly'; +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_schedule', 'max_backups', 'INT(8) UNSIGNED NOT NULL DEFAULT 0 COMMENT ''Maximum number of backups to be retained'''); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backups', 'backup_schedule_id', 'BIGINT(20) UNSIGNED'); -- Update default value for the config 'vm.network.nic.max.secondary.ipaddresses' (and value to default value if value is null) UPDATE `cloud`.`configuration` SET default_value = '10' WHERE name = 'vm.network.nic.max.secondary.ipaddresses'; diff --git a/server/src/main/java/com/cloud/api/dispatch/ParamGenericValidationWorker.java b/server/src/main/java/com/cloud/api/dispatch/ParamGenericValidationWorker.java index bfe256305d5..bfd8b827ec5 100644 --- a/server/src/main/java/com/cloud/api/dispatch/ParamGenericValidationWorker.java +++ b/server/src/main/java/com/cloud/api/dispatch/ParamGenericValidationWorker.java @@ -69,6 +69,7 @@ public class ParamGenericValidationWorker implements DispatchWorker { defaultParamNames.add(ApiConstants.ID); defaultParamNames.add(ApiConstants.SIGNATURE_VERSION); defaultParamNames.add(ApiConstants.EXPIRES); + defaultParamNames.add(ApiConstants.SCHEDULE_ID); defaultParamNames.add("_"); } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index a7d03f1a9a3..52a12a6ff9b 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -33,6 +33,8 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.serializer.GsonHelper; +import com.google.gson.reflect.TypeToken; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.InternalIdentity; @@ -71,6 +73,7 @@ import org.apache.cloudstack.poll.BackgroundPollTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang.math.NumberUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -97,7 +100,6 @@ import com.cloud.hypervisor.HypervisorGuru; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.projects.Project; import com.cloud.storage.ScopeType; -import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; @@ -430,7 +432,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final DateUtil.IntervalType intervalType = cmd.getIntervalType(); final String scheduleString = cmd.getSchedule(); final TimeZone timeZone = TimeZone.getTimeZone(cmd.getTimezone()); - final Integer maxBackups = cmd.getMaxBackups(); if (intervalType == null) { throw new CloudRuntimeException("Invalid interval type provided"); @@ -443,40 +444,13 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { if (vm.getBackupOfferingId() == null) { throw new CloudRuntimeException("Cannot configure backup schedule for the VM without having any backup offering"); } - if (maxBackups != null && maxBackups <= 0) { - throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s should be greater than 0.", maxBackups, vm.getName())); - } - - Backup.Type backupType = Backup.Type.valueOf(intervalType.name()); - int intervalMaxBackups = backupType.getMax(); - if (maxBackups != null && maxBackups > intervalMaxBackups) { - throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s exceeds limit [%s] for interval type [%s].", maxBackups, vm.getName(), - intervalMaxBackups, intervalType)); - } - - Account owner = accountManager.getAccount(vm.getAccountId()); - - long accountLimit = resourceLimitMgr.findCorrectResourceLimitForAccount(owner, Resource.ResourceType.backup, null); - long domainLimit = resourceLimitMgr.findCorrectResourceLimitForDomain(domainManager.getDomain(owner.getDomainId()), Resource.ResourceType.backup, null); - if (maxBackups != null && !accountManager.isRootAdmin(owner.getId()) && ((accountLimit != -1 && maxBackups > accountLimit) || (domainLimit != -1 && maxBackups > domainLimit))) { - String message = "domain/account"; - if (owner.getType() == Account.Type.PROJECT) { - message = "domain/project"; - } - throw new InvalidParameterValueException("Max number of backups shouldn't exceed the " + message + " level backup limit"); - } final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); if (offering == null || !offering.isUserDrivenBackupAllowed()) { throw new CloudRuntimeException("The selected backup offering does not allow user-defined backup schedule"); } - if (maxBackups == null && !"veeam".equals(offering.getProvider())) { - throw new CloudRuntimeException("Please specify the maximum number of buckets to retain."); - } - if (maxBackups != null && "veeam".equals(offering.getProvider())) { - throw new CloudRuntimeException("The maximum backups to retain cannot be configured through CloudStack for Veeam. Retention is managed directly in Veeam based on the settings specified when creating the backup job."); - } + final int maxBackups = validateAndGetDefaultBackupRetentionIfRequired(cmd.getMaxBackups(), offering, vm); final String timezoneId = timeZone.getID(); if (!timezoneId.equals(cmd.getTimezone())) { @@ -504,6 +478,43 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { return backupScheduleDao.findById(schedule.getId()); } + /** + * Validates the provided backup retention value and returns 0 as the default value if required. + * + * @param maxBackups The number of backups to retain, can be null + * @param offering The backup offering + * @param vm The VM associated with the backup schedule + * @return The validated number of backups to retain. If maxBackups is null, returns 0 as the default value + * @throws InvalidParameterValueException if the backup offering's provider is Veeam, or maxBackups is less than 0 or greater than the account and domain backup limits + */ + protected int validateAndGetDefaultBackupRetentionIfRequired(Integer maxBackups, BackupOffering offering, VirtualMachine vm) { + if (maxBackups == null) { + return 0; + } + if ("veeam".equals(offering.getProvider())) { + throw new InvalidParameterValueException("The maximum amount of backups to retain cannot be directly configured via Apache CloudStack for Veeam. " + + "Retention is managed directly in Veeam based on the settings specified when creating the backup job."); + } + if (maxBackups < 0) { + throw new InvalidParameterValueException("maxbackups value for backup schedule must be a non-negative integer."); + } + + Account owner = accountManager.getAccount(vm.getAccountId()); + long accountLimit = resourceLimitMgr.findCorrectResourceLimitForAccount(owner, Resource.ResourceType.backup, null); + boolean exceededAccountLimit = accountLimit != -1 && maxBackups > accountLimit; + + long domainLimit = resourceLimitMgr.findCorrectResourceLimitForDomain(domainManager.getDomain(owner.getDomainId()), Resource.ResourceType.backup, null); + boolean exceededDomainLimit = domainLimit != -1 && maxBackups > domainLimit; + + if (!accountManager.isRootAdmin(owner.getId()) && (exceededAccountLimit || exceededDomainLimit)) { + throw new InvalidParameterValueException( + String.format("'maxbackups' should not exceed the domain/%s backup limit.", owner.getType() == Account.Type.PROJECT ? "project" : "account") + ); + } + + return maxBackups; + } + @Override public List listBackupSchedule(final Long vmId) { final VMInstanceVO vm = findVmById(vmId); @@ -564,30 +575,9 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { return success; } - private void postCreateScheduledBackup(Backup.Type backupType, Long vmId) { - DateUtil.IntervalType intervalType = DateUtil.IntervalType.valueOf(backupType.name()); - final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); - if (schedule == null) { - return; - } - Integer maxBackups = schedule.getMaxBackups(); - if (maxBackups == null) { - return; - } - List backups = backupDao.listBackupsByVMandIntervalType(vmId, backupType); - while (backups.size() > maxBackups) { - BackupVO oldestBackup = backups.get(0); - if (deleteBackup(oldestBackup.getId(), false)) { - ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, oldestBackup.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_DELETE, - "Successfully deleted oldest backup: " + oldestBackup.getId(), oldestBackup.getId(), ApiCommandResourceType.Backup.toString(), 0); - } - backups.remove(oldestBackup); - } - } - @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true) - public boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException { + public boolean createBackup(final Long vmId, Object job) throws ResourceAllocationException { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -605,16 +595,14 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { throw new CloudRuntimeException("The assigned backup offering does not allow ad-hoc user backup"); } - Backup.Type type = getBackupType(scheduleId); + Long backupScheduleId = getBackupScheduleId(job); + boolean isScheduledBackup = backupScheduleId != null; Account owner = accountManager.getAccount(vm.getAccountId()); try { resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup); } catch (ResourceAllocationException e) { - if (type != Backup.Type.MANUAL) { - String msg = "Backup resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; - logger.warn(msg); - alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup resource limit exceeded for account id : " + owner.getId() - + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + if (isScheduledBackup) { + sendExceededBackupLimitAlert(owner.getUuid(), Resource.ResourceType.backup); } throw e; } @@ -632,11 +620,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { try { resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup_storage, backupSize); } catch (ResourceAllocationException e) { - if (type != Backup.Type.MANUAL) { - String msg = "Backup storage space resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; - logger.warn(msg); - alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + owner.getId() - + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + if (isScheduledBackup) { + sendExceededBackupLimitAlert(owner.getUuid(), Resource.ResourceType.backup_storage); } throw e; } @@ -646,7 +631,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); - final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); if (backupProvider != null) { Pair result = backupProvider.takeBackup(vm); @@ -656,19 +640,108 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { Backup backup = result.second(); if (backup != null) { BackupVO vmBackup = backupDao.findById(result.second().getId()); - vmBackup.setBackupIntervalType((short) type.ordinal()); + vmBackup.setBackupScheduleId(backupScheduleId); backupDao.update(vmBackup.getId(), vmBackup); resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); } - if (type != Backup.Type.MANUAL) { - postCreateScheduledBackup(type, vm.getId()); + if (isScheduledBackup) { + deleteOldestBackupFromScheduleIfRequired(vmId, backupScheduleId); } return true; } throw new CloudRuntimeException("Failed to create VM backup"); } + /** + * Sends an alert when the backup limit has been exceeded for a given account. + * + * @param ownerUuid The UUID of the account owner that exceeded the limit + * @param resourceType The type of resource limit that was exceeded (either {@link Resource.ResourceType#backup} or {@link Resource.ResourceType#backup_storage}) + * + */ + protected void sendExceededBackupLimitAlert(String ownerUuid, Resource.ResourceType resourceType) { + String message = String.format("Failed to create backup: backup %s limit exceeded for account with ID: %s.", + resourceType == Resource.ResourceType.backup ? "resource" : "storage space resource" , ownerUuid); + logger.warn(message); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, + message, message + " Please, use the 'updateResourceLimit' API to increase the backup limit."); + } + + /** + * Gets the backup schedule ID from the async job's payload. + * + * @param job The asynchronous job associated with the creation of the backup + * @return The backup schedule ID. Returns null if the backup has been manually created + */ + protected Long getBackupScheduleId(Object job) { + if (!(job instanceof AsyncJobVO)) { + return null; + } + + AsyncJobVO asyncJob = (AsyncJobVO) job; + logger.debug("Trying to retrieve [{}] parameter from the job [ID: {}] parameters.", ApiConstants.SCHEDULE_ID, asyncJob.getId()); + String jobParamsRaw = asyncJob.getCmdInfo(); + + if (!jobParamsRaw.contains(ApiConstants.SCHEDULE_ID)) { + logger.info("Job [ID: {}] parameters do not include the [{}] parameter. Thus, the current backup is a manual backup.", asyncJob.getId(), ApiConstants.SCHEDULE_ID); + return null; + } + + TypeToken> jobParamsType = new TypeToken<>(){}; + Map jobParams = GsonHelper.getGson().fromJson(jobParamsRaw, jobParamsType.getType()); + long backupScheduleId = NumberUtils.toLong(jobParams.get(ApiConstants.SCHEDULE_ID)); + logger.info("Job [ID: {}] parameters include the [{}] parameter, whose value is equal to [{}]. Thus, the current backup is a scheduled backup.", asyncJob.getId(), ApiConstants.SCHEDULE_ID, backupScheduleId); + return backupScheduleId == 0L ? null : backupScheduleId; + } + + /** + * Deletes the oldest backups from the schedule. If the backup schedule is not active, the schedule's retention is equal to 0, + * or the number of backups to be deleted is lower than one, then no backups are deleted. + * + * @param vmId The ID of the VM associated with the backups + * @param backupScheduleId Backup schedule ID of the backups + */ + protected void deleteOldestBackupFromScheduleIfRequired(Long vmId, long backupScheduleId) { + BackupScheduleVO backupScheduleVO = backupScheduleDao.findById(backupScheduleId); + if (backupScheduleVO == null || backupScheduleVO.getMaxBackups() == 0) { + logger.info("The schedule does not have a retention specified and, hence, not deleting any backups from it.", vmId); + return; + } + + logger.debug("Checking if it is required to delete the oldest backups from the schedule with ID [{}], to meet its retention requirement of [{}] backups.", backupScheduleId, backupScheduleVO.getMaxBackups()); + List backups = backupDao.listBySchedule(backupScheduleId); + int amountOfBackupsToDelete = backups.size() - backupScheduleVO.getMaxBackups(); + if (amountOfBackupsToDelete > 0) { + deleteExcessBackups(backups, amountOfBackupsToDelete, backupScheduleId); + } else { + logger.debug("Not required to delete any backups from the schedule [ID: {}]: [backups size: {}] and [retention: {}].", backupScheduleId, backups.size(), backupScheduleVO.getMaxBackups()); + } + } + + /** + * Deletes a certain number of backups associated with a schedule. + * + * @param backups List of backups associated with a schedule + * @param amountOfBackupsToDelete Number of backups to be deleted from the list of backups + * @param backupScheduleId ID of the backup schedule associated with the backups + */ + protected void deleteExcessBackups(List backups, int amountOfBackupsToDelete, long backupScheduleId) { + logger.debug("Deleting the [{}] oldest backups from the schedule [ID: {}].", amountOfBackupsToDelete, backupScheduleId); + + for (int i = 0; i < amountOfBackupsToDelete; i++) { + BackupVO backup = backups.get(i); + if (deleteBackup(backup.getId(), false)) { + String eventDescription = String.format("Successfully deleted backup for VM [ID: %s], suiting the retention specified in the backup schedule [ID: %s]", backup.getVmId(), backupScheduleId); + logger.info(eventDescription); + ActionEventUtils.onCompletedActionEvent( + User.UID_SYSTEM, backup.getAccountId(), EventVO.LEVEL_INFO, + EventTypes.EVENT_VM_BACKUP_DELETE, eventDescription, backup.getId(), ApiCommandResourceType.Backup.toString(), 0 + ); + } + } + } + @Override public Pair, Integer> listBackups(final ListBackupsCmd cmd) { final Long id = cmd.getId(); @@ -839,29 +912,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { } } - private Backup.Type getBackupType(Long scheduleId) { - if (scheduleId.equals(Snapshot.MANUAL_POLICY_ID)) { - return Backup.Type.MANUAL; - } else { - BackupScheduleVO scheduleVO = backupScheduleDao.findById(scheduleId); - DateUtil.IntervalType intvType = scheduleVO.getScheduleType(); - return getBackupType(intvType); - } - } - - private Backup.Type getBackupType(DateUtil.IntervalType intvType) { - if (intvType.equals(DateUtil.IntervalType.HOURLY)) { - return Backup.Type.HOURLY; - } else if (intvType.equals(DateUtil.IntervalType.DAILY)) { - return Backup.Type.DAILY; - } else if (intvType.equals(DateUtil.IntervalType.WEEKLY)) { - return Backup.Type.WEEKLY; - } else if (intvType.equals(DateUtil.IntervalType.MONTHLY)) { - return Backup.Type.MONTHLY; - } - return null; - } - /** * Tries to update the state of given VM, given specified event * @param vm The VM to update its state @@ -1107,10 +1157,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); backgroundPollManager.submitTask(new BackupSyncTask(this)); - Backup.Type.HOURLY.setMax(BackupHourlyMax.value()); - Backup.Type.DAILY.setMax(BackupDailyMax.value()); - Backup.Type.WEEKLY.setMax(BackupWeeklyMax.value()); - Backup.Type.MONTHLY.setMax(BackupMonthlyMax.value()); return true; } @@ -1191,10 +1237,6 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { BackupProviderPlugin, BackupSyncPollingInterval, BackupEnableAttachDetachVolumes, - BackupHourlyMax, - BackupDailyMax, - BackupWeeklyMax, - BackupMonthlyMax, DefaultMaxAccountBackups, DefaultMaxAccountBackupStorage, DefaultMaxProjectBackups, @@ -1333,7 +1375,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { true, 0); final Map params = new HashMap(); params.put(ApiConstants.VIRTUAL_MACHINE_ID, "" + vmId); - params.put(ApiConstants.SCHEDULE_ID, "" + backupScheduleId); + params.put(ApiConstants.SCHEDULE_ID, String.valueOf(backupScheduleId)); params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + vm.getAccountId()); params.put("ctxStartEventId", String.valueOf(eventId)); diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 969a4202c51..609635ef3ab 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -44,6 +44,8 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; +import com.google.gson.Gson; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; @@ -54,6 +56,7 @@ import org.apache.cloudstack.backup.dao.BackupScheduleDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -74,9 +77,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; @@ -128,25 +133,39 @@ public class BackupManagerTest { DataCenterDao dataCenterDao; @Mock - AlertManager alertManager; + private AlertManager alertManagerMock; + + @Mock + private Domain domainMock; @Mock private VMInstanceVO vmInstanceVOMock; @Mock - private CallContext callContextMock; + private CreateBackupScheduleCmd createBackupScheduleCmdMock; + + @Mock + private BackupOfferingVO backupOfferingVOMock; + + @Mock + private AsyncJobVO asyncJobVOMock; + + @Mock + private BackupScheduleVO backupScheduleVOMock; @Mock private AccountVO accountVOMock; @Mock - private DeleteBackupScheduleCmd deleteBackupScheduleCmdMock; + private CallContext callContextMock; @Mock - private BackupScheduleVO backupScheduleVOMock; + private DeleteBackupScheduleCmd deleteBackupScheduleCmdMock; private UserVO user; + private Gson gson; + private String[] hostPossibleValues = {"127.0.0.1", "hostname"}; private String[] datastoresPossibleValues = {"e9804933-8609-4de3-bccc-6278072a496c", "datastore-name"}; private AutoCloseable closeable; @@ -155,6 +174,8 @@ public class BackupManagerTest { @Before public void setup() throws Exception { + gson = new Gson(); + closeable = MockitoAnnotations.openMocks(this); when(backupOfferingDao.findById(null)).thenReturn(null); when(backupOfferingDao.findById(123l)).thenReturn(null); @@ -452,98 +473,84 @@ public class BackupManagerTest { } @Test - public void testConfigureBackupScheduleLimitReached() { - Long vmId = 1L; - Long zoneId = 2L; - Long accountId = 3L; - Long domainId = 4L; + public void configureBackupScheduleTestEnsureLimitCheckIsPerformed() { + long vmId = 1L; + long zoneId = 2L; + long accountId = 3L; + long domainId = 4L; + long backupOfferingId = 5L; - CreateBackupScheduleCmd cmd = Mockito.mock(CreateBackupScheduleCmd.class); - when(cmd.getVmId()).thenReturn(vmId); - when(cmd.getTimezone()).thenReturn("GMT"); - when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); - when(cmd.getMaxBackups()).thenReturn(8); + when(createBackupScheduleCmdMock.getVmId()).thenReturn(vmId); + when(createBackupScheduleCmdMock.getTimezone()).thenReturn("GMT"); + when(createBackupScheduleCmdMock.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); + when(createBackupScheduleCmdMock.getMaxBackups()).thenReturn(8); - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - when(vmInstanceDao.findById(vmId)).thenReturn(vm); - when(vm.getDataCenterId()).thenReturn(zoneId); - when(vm.getAccountId()).thenReturn(accountId); + when(vmInstanceDao.findById(vmId)).thenReturn(vmInstanceVOMock); + when(vmInstanceVOMock.getDataCenterId()).thenReturn(zoneId); + when(vmInstanceVOMock.getAccountId()).thenReturn(accountId); + when(vmInstanceVOMock.getBackupOfferingId()).thenReturn(backupOfferingId); + + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(backupOfferingVOMock); + when(backupOfferingVOMock.isUserDrivenBackupAllowed()).thenReturn(true); overrideBackupFrameworkConfigValue(); - Account account = Mockito.mock(Account.class); - when(accountManager.getAccount(accountId)).thenReturn(account); - when(account.getDomainId()).thenReturn(domainId); - Domain domain = Mockito.mock(Domain.class); - when(domainManager.getDomain(domainId)).thenReturn(domain); - when(resourceLimitMgr.findCorrectResourceLimitForAccount(account, Resource.ResourceType.backup, null)).thenReturn(10L); - when(resourceLimitMgr.findCorrectResourceLimitForDomain(domain, Resource.ResourceType.backup, null)).thenReturn(1L); + when(accountManager.getAccount(accountId)).thenReturn(accountVOMock); + when(accountVOMock.getDomainId()).thenReturn(domainId); + when(domainManager.getDomain(domainId)).thenReturn(domainMock); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(accountVOMock, Resource.ResourceType.backup, null)).thenReturn(10L); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domainMock, Resource.ResourceType.backup, null)).thenReturn(1L); InvalidParameterValueException exception = Assert.assertThrows(InvalidParameterValueException.class, - () -> backupManager.configureBackupSchedule(cmd)); - Assert.assertEquals(exception.getMessage(), "Max number of backups shouldn't exceed the domain/account level backup limit"); + () -> backupManager.configureBackupSchedule(createBackupScheduleCmdMock)); + Assert.assertEquals("'maxbackups' should not exceed the domain/account backup limit.", exception.getMessage()); } @Test - public void testCreateScheduledBackup() throws ResourceAllocationException { + public void createBackupTestCreateScheduledBackup() throws ResourceAllocationException { Long vmId = 1L; Long zoneId = 2L; Long scheduleId = 3L; Long backupOfferingId = 4L; Long accountId = 5L; Long backupId = 6L; - Long oldestBackupId = 7L; Long newBackupSize = 1000000000L; - Long oldBackupSize = 400000000L; - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - when(vmInstanceDao.findById(vmId)).thenReturn(vm); - when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); - when(vm.getId()).thenReturn(vmId); - when(vm.getDataCenterId()).thenReturn(zoneId); - when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); - when(vm.getAccountId()).thenReturn(accountId); + when(vmInstanceDao.findById(vmId)).thenReturn(vmInstanceVOMock); + when(vmInstanceVOMock.getDataCenterId()).thenReturn(zoneId); + when(vmInstanceVOMock.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vmInstanceVOMock.getAccountId()).thenReturn(accountId); overrideBackupFrameworkConfigValue(); - BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); - when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); - when(offering.isUserDrivenBackupAllowed()).thenReturn(true); - when(offering.getProvider()).thenReturn("test"); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(backupOfferingVOMock); + when(backupOfferingVOMock.isUserDrivenBackupAllowed()).thenReturn(true); + when(backupOfferingVOMock.getProvider()).thenReturn("test"); - Account account = Mockito.mock(Account.class); - when(accountManager.getAccount(accountId)).thenReturn(account); + Mockito.doReturn(scheduleId).when(backupManager).getBackupScheduleId(asyncJobVOMock); + + when(accountManager.getAccount(accountId)).thenReturn(accountVOMock); BackupScheduleVO schedule = mock(BackupScheduleVO.class); - when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); - when(schedule.getMaxBackups()).thenReturn(0); when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); - when(backupScheduleDao.findByVMAndIntervalType(vmId, DateUtil.IntervalType.DAILY)).thenReturn(schedule); + when(schedule.getMaxBackups()).thenReturn(2); BackupProvider backupProvider = mock(BackupProvider.class); Backup backup = mock(Backup.class); when(backup.getId()).thenReturn(backupId); when(backup.getSize()).thenReturn(newBackupSize); when(backupProvider.getName()).thenReturn("test"); - when(backupProvider.takeBackup(vm)).thenReturn(new Pair<>(true, backup)); + when(backupProvider.takeBackup(vmInstanceVOMock)).thenReturn(new Pair<>(true, backup)); Map backupProvidersMap = new HashMap<>(); backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); BackupVO backupVO = mock(BackupVO.class); when(backupVO.getId()).thenReturn(backupId); - BackupVO oldestBackupVO = mock(BackupVO.class); - when(oldestBackupVO.getSize()).thenReturn(oldBackupSize); - when(oldestBackupVO.getId()).thenReturn(oldestBackupId); - when(oldestBackupVO.getVmId()).thenReturn(vmId); - when(oldestBackupVO.getBackupOfferingId()).thenReturn(backupOfferingId); + BackupVO oldestBackupVO = mock(BackupVO.class);; when(backupDao.findById(backupId)).thenReturn(backupVO); List backups = new ArrayList<>(List.of(oldestBackupVO)); - when(backupDao.listBackupsByVMandIntervalType(vmId, Backup.Type.DAILY)).thenReturn(backups); - when(backupDao.findByIdIncludingRemoved(oldestBackupId)).thenReturn(oldestBackupVO); - when(backupOfferingDao.findByIdIncludingRemoved(backupOfferingId)).thenReturn(offering); - when(backupProvider.deleteBackup(oldestBackupVO, false)).thenReturn(true); - when(backupDao.remove(oldestBackupVO.getId())).thenReturn(true); + when(backupDao.listBySchedule(scheduleId)).thenReturn(backups); try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), @@ -551,51 +558,39 @@ public class BackupManagerTest { Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); - Assert.assertEquals(backupManager.createBackup(vmId, scheduleId), true); - + assertTrue(backupManager.createBackup(vmId, asyncJobVOMock)); Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); Mockito.verify(backupDao, times(1)).update(backupVO.getId(), backupVO); - - Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup); - Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, oldBackupSize); - Mockito.verify(backupDao, times(1)).remove(oldestBackupId); + Mockito.verify(backupManager, times(1)).deleteOldestBackupFromScheduleIfRequired(vmId, scheduleId); } } - @Test (expected = ResourceAllocationException.class) - public void testCreateBackupLimitReached() throws ResourceAllocationException { + @Test(expected = ResourceAllocationException.class) + public void createBackupTestResourceLimitReached() throws ResourceAllocationException { Long vmId = 1L; Long zoneId = 2L; Long scheduleId = 3L; Long backupOfferingId = 4L; Long accountId = 5L; - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - when(vmInstanceDao.findById(vmId)).thenReturn(vm); - when(vm.getDataCenterId()).thenReturn(zoneId); - when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); - when(vm.getAccountId()).thenReturn(accountId); + when(vmInstanceDao.findById(vmId)).thenReturn(vmInstanceVOMock); + when(vmInstanceVOMock.getDataCenterId()).thenReturn(zoneId); + when(vmInstanceVOMock.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vmInstanceVOMock.getAccountId()).thenReturn(accountId); overrideBackupFrameworkConfigValue(); BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); when(offering.isUserDrivenBackupAllowed()).thenReturn(true); - BackupScheduleVO schedule = mock(BackupScheduleVO.class); - when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); - when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); + Mockito.doReturn(scheduleId).when(backupManager).getBackupScheduleId(asyncJobVOMock); Account account = Mockito.mock(Account.class); - when(account.getId()).thenReturn(accountId); when(accountManager.getAccount(accountId)).thenReturn(account); Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup_storage)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup_storage, 0L); - backupManager.createBackup(vmId, scheduleId); - - String msg = "Backup storage space resource limit exceeded for account id : " + accountId + ". Failed to create backup"; - Mockito.verify(alertManager, times(1)).sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + accountId - + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + backupManager.createBackup(vmId, asyncJobVOMock); } @Test @@ -766,4 +761,212 @@ public class BackupManagerTest { boolean success = backupManager.deleteBackupSchedule(deleteBackupScheduleCmdMock); assertTrue(success); } + + @Test + public void validateAndGetDefaultBackupRetentionIfRequiredTestReturnZeroAsDefaultValue() { + int retention = backupManager.validateAndGetDefaultBackupRetentionIfRequired(null, backupOfferingVOMock, null); + assertEquals(0, retention); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateAndGetDefaultBackupRetentionIfRequiredTestThrowExceptionWhenBackupOfferingProviderIsVeeam() { + Mockito.when(backupOfferingVOMock.getProvider()).thenReturn("veeam"); + backupManager.validateAndGetDefaultBackupRetentionIfRequired(1, backupOfferingVOMock, vmInstanceVOMock); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateAndGetDefaultBackupRetentionIfRequiredTestThrowExceptionWhenMaxBackupsIsLessThanZero() { + backupManager.validateAndGetDefaultBackupRetentionIfRequired(-1, backupOfferingVOMock, vmInstanceVOMock); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateAndGetDefaultBackupRetentionIfRequiredTestThrowExceptionWhenMaxBackupsExceedsAccountLimit() { + int maxBackups = 6; + long accountId = 1L; + long accountLimit = 5L; + long domainId = 10L; + long domainLimit = -1L; + + when(vmInstanceVOMock.getAccountId()).thenReturn(accountId); + when(accountManager.getAccount(accountId)).thenReturn(accountVOMock); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(accountVOMock, Resource.ResourceType.backup, null)).thenReturn(accountLimit); + when(accountVOMock.getDomainId()).thenReturn(domainId); + when(domainManager.getDomain(domainId)).thenReturn(domainMock); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domainMock, Resource.ResourceType.backup, null)).thenReturn(domainLimit); + when(accountVOMock.getId()).thenReturn(accountId); + when(accountManager.isRootAdmin(accountId)).thenReturn(false); + + backupManager.validateAndGetDefaultBackupRetentionIfRequired(maxBackups, backupOfferingVOMock, vmInstanceVOMock); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateAndGetDefaultBackupRetentionIfRequiredTestThrowExceptionWhenMaxBackupsExceedsDomainLimit() { + int maxBackups = 6; + long accountId = 1L; + long accountLimit = -1L; + long domainId = 10L; + long domainLimit = 5L; + + when(vmInstanceVOMock.getAccountId()).thenReturn(accountId); + when(accountManager.getAccount(accountId)).thenReturn(accountVOMock); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(accountVOMock, Resource.ResourceType.backup, null)).thenReturn(accountLimit); + when(accountVOMock.getDomainId()).thenReturn(domainId); + when(domainManager.getDomain(domainId)).thenReturn(domainMock); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domainMock, Resource.ResourceType.backup, null)).thenReturn(domainLimit); + when(accountVOMock.getId()).thenReturn(accountId); + when(accountManager.isRootAdmin(accountId)).thenReturn(false); + + backupManager.validateAndGetDefaultBackupRetentionIfRequired(maxBackups, backupOfferingVOMock, vmInstanceVOMock); + } + + @Test + public void validateAndGetDefaultBackupRetentionIfRequiredTestIgnoreLimitCheckWhenAccountIsRootAdmin() { + int maxBackups = 6; + long accountId = 1L; + long accountLimit = 5L; + long domainId = 10L; + long domainLimit = 5L; + + when(vmInstanceVOMock.getAccountId()).thenReturn(accountId); + when(accountManager.getAccount(accountId)).thenReturn(accountVOMock); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(accountVOMock, Resource.ResourceType.backup, null)).thenReturn(accountLimit); + when(accountVOMock.getDomainId()).thenReturn(domainId); + when(domainManager.getDomain(domainId)).thenReturn(domainMock); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domainMock, Resource.ResourceType.backup, null)).thenReturn(domainLimit); + when(accountVOMock.getId()).thenReturn(accountId); + when(accountManager.isRootAdmin(accountId)).thenReturn(true); + + int retention = backupManager.validateAndGetDefaultBackupRetentionIfRequired(maxBackups, backupOfferingVOMock, vmInstanceVOMock); + assertEquals(maxBackups, retention); + } + + @Test + public void getBackupScheduleTestReturnNullWhenBackupIsManual() { + String jobParams = "{}"; + when(asyncJobVOMock.getCmdInfo()).thenReturn(jobParams); + when(asyncJobVOMock.getId()).thenReturn(1L); + + Long backupScheduleId = backupManager.getBackupScheduleId(asyncJobVOMock); + assertNull(backupScheduleId); + } + + @Test + public void getBackupScheduleTestReturnBackupScheduleIdWhenBackupIsScheduled() { + Map params = Map.of( + ApiConstants.SCHEDULE_ID, "100" + ); + String jobParams = gson.toJson(params); + when(asyncJobVOMock.getCmdInfo()).thenReturn(jobParams); + when(asyncJobVOMock.getId()).thenReturn(1L); + + Long backupScheduleId = backupManager.getBackupScheduleId(asyncJobVOMock); + assertEquals(Long.valueOf("100"), backupScheduleId); + } + + @Test + public void getBackupScheduleTestReturnNullWhenSpecifiedBackupScheduleIdIsNotALongValue() { + Map params = Map.of( + ApiConstants.SCHEDULE_ID, "InvalidValue" + ); + String jobParams = gson.toJson(params); + when(asyncJobVOMock.getCmdInfo()).thenReturn(jobParams); + when(asyncJobVOMock.getId()).thenReturn(1L); + + Long backupScheduleId = backupManager.getBackupScheduleId(asyncJobVOMock); + assertNull(backupScheduleId); + } + + @Test + public void deleteOldestBackupFromScheduleIfRequiredTestSkipDeletionWhenBackupScheduleIsNotFound() { + backupManager.deleteOldestBackupFromScheduleIfRequired(1L, 1L); + Mockito.verify(backupManager, Mockito.never()).deleteExcessBackups(Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong()); + } + + @Test + public void deleteOldestBackupFromScheduleIfRequiredTestSkipDeletionWhenRetentionIsEqualToZero() { + Mockito.when(backupScheduleDao.findById(1L)).thenReturn(backupScheduleVOMock); + Mockito.when(backupScheduleVOMock.getMaxBackups()).thenReturn(0); + backupManager.deleteOldestBackupFromScheduleIfRequired(1L, 1L); + Mockito.verify(backupManager, Mockito.never()).deleteExcessBackups(Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong()); + } + + @Test + public void deleteOldestBackupFromScheduleIfRequiredTestSkipDeletionWhenAmountOfBackupsToBeDeletedIsLessThanOne() { + List backups = List.of(Mockito.mock(BackupVO.class), Mockito.mock(BackupVO.class)); + Mockito.when(backupScheduleDao.findById(1L)).thenReturn(backupScheduleVOMock); + Mockito.when(backupScheduleVOMock.getMaxBackups()).thenReturn(2); + Mockito.when(backupDao.listBySchedule(1L)).thenReturn(backups); + backupManager.deleteOldestBackupFromScheduleIfRequired(1L, 1L); + Mockito.verify(backupManager, Mockito.never()).deleteExcessBackups(Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong()); + } + + @Test + public void deleteOldestBackupFromScheduleIfRequiredTestDeleteBackupsWhenRequired() { + List backups = List.of(Mockito.mock(BackupVO.class), Mockito.mock(BackupVO.class)); + Mockito.when(backupScheduleDao.findById(1L)).thenReturn(backupScheduleVOMock); + Mockito.when(backupScheduleVOMock.getMaxBackups()).thenReturn(1); + Mockito.when(backupDao.listBySchedule(1L)).thenReturn(backups); + Mockito.doNothing().when(backupManager).deleteExcessBackups(Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong()); + backupManager.deleteOldestBackupFromScheduleIfRequired(1L, 1L); + Mockito.verify(backupManager).deleteExcessBackups(Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong()); + } + + @Test + public void deleteExcessBackupsTestEnsureBackupsAreDeletedWhenMethodIsCalled() { + try (MockedStatic actionEventUtils = Mockito.mockStatic(ActionEventUtils.class)) { + List backups = List.of(Mockito.mock(BackupVO.class), + Mockito.mock(BackupVO.class), + Mockito.mock(BackupVO.class)); + + Mockito.when(backups.get(0).getId()).thenReturn(1L); + Mockito.when(backups.get(1).getId()).thenReturn(2L); + Mockito.when(backups.get(0).getAccountId()).thenReturn(1L); + Mockito.when(backups.get(1).getAccountId()).thenReturn(2L); + Mockito.doReturn(true).when(backupManager).deleteBackup(Mockito.anyLong(), Mockito.eq(false)); + + actionEventUtils.when(() -> ActionEventUtils.onStartedActionEvent( + Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyLong(), Mockito.anyString(), + Mockito.anyBoolean(), Mockito.anyInt())).thenReturn(1L); + actionEventUtils.when(() -> ActionEventUtils.onCompletedActionEvent( + Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), Mockito.anyLong(), + Mockito.anyString(), Mockito.anyInt())).thenReturn(2L); + + backupManager.deleteExcessBackups(backups, 2, 1L); + Mockito.verify(backupManager, times(2)).deleteBackup(Mockito.anyLong(), Mockito.eq(false)); + } + } + + @Test + public void sendExceededBackupLimitAlertTestSendAlertForBackupResourceType() { + String accountUuid = UUID.randomUUID().toString(); + String expectedMessage = "Failed to create backup: backup resource limit exceeded for account with ID: " + accountUuid + "."; + String expectedAlertDetails = expectedMessage + " Please, use the 'updateResourceLimit' API to increase the backup limit."; + + backupManager.sendExceededBackupLimitAlert(accountUuid, Resource.ResourceType.backup); + verify(alertManagerMock).sendAlert( + AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, + 0L, + 0L, + expectedMessage, + expectedAlertDetails + ); + } + + @Test + public void sendExceededBackupLimitAlertTestSendAlertForBackupStorageResourceType() { + String accountUuid = UUID.randomUUID().toString(); + String expectedMessage = "Failed to create backup: backup storage space resource limit exceeded for account with ID: " + accountUuid + "."; + String expectedAlertDetails = expectedMessage + " Please, use the 'updateResourceLimit' API to increase the backup limit."; + + backupManager.sendExceededBackupLimitAlert(accountUuid, Resource.ResourceType.backup_storage); + verify(alertManagerMock).sendAlert( + AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, + 0L, + 0L, + expectedMessage, + expectedAlertDetails + ); + } } diff --git a/ui/src/views/compute/backup/FormSchedule.vue b/ui/src/views/compute/backup/FormSchedule.vue index 5a6928cca1c..01cae9d7d8f 100644 --- a/ui/src/views/compute/backup/FormSchedule.vue +++ b/ui/src/views/compute/backup/FormSchedule.vue @@ -112,7 +112,7 @@ + :min="0" /> From 712492230ae6149cbd06f7674ed0b7a4c9881e69 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 31 Jul 2025 10:00:29 +0530 Subject: [PATCH 09/53] Shutdown MS maintenance jobs when finished (#11330) --- agent/src/main/java/com/cloud/agent/Agent.java | 4 +++- .../maintenance/ManagementServerMaintenanceManagerImpl.java | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index b7c24e5126c..a1834b6827b 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -968,9 +968,11 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater if (CollectionUtils.isNotEmpty(cmd.getMsList())) { processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), false); } - Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")).schedule(() -> { + ScheduledExecutorService migrateAgentConnectionService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")); + migrateAgentConnectionService.schedule(() -> { migrateAgentConnection(cmd.getAvoidMsList()); }, 3, TimeUnit.SECONDS); + migrateAgentConnectionService.shutdown(); } catch (Exception e) { String errMsg = "Migrate agent connection failed, due to " + e.getMessage(); logger.debug(errMsg, e); diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java index 16cf14e1fb1..516ed40d48b 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java @@ -622,6 +622,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); if (msHost == null) { logger.warn("Unable to find the management server, invalid node id"); + managementServerMaintenanceManager.cancelWaitForPendingJobs(); return; } msHostDao.updateState(msHost.getId(), State.Maintenance); @@ -658,6 +659,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); if (msHost == null) { logger.warn("Unable to find the management server, invalid node id"); + managementServerMaintenanceManager.cancelWaitForPendingJobs(); return; } if (totalAgents == 0) { @@ -693,6 +695,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); if (msHost == null) { logger.warn("Unable to find the management server, invalid node id"); + managementServerMaintenanceManager.cancelWaitForPendingJobs(); return; } msHostDao.updateState(msHost.getId(), State.ReadyToShutDown); From 1f1e38f3a85f6bfe3e941c88671e36878a971fda Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 31 Jul 2025 10:08:35 +0530 Subject: [PATCH 10/53] Support to list templates in ready state (new API parameter 'isready', similar to list ISOs) (#11343) * Support to list templates in ready state (new API parameter 'isready', similar to list ISOs), and UI to display Templates/ISOs in ready state wherever applicable --- .../api/command/user/iso/ListIsosCmd.java | 2 +- .../command/user/template/ListTemplatesCmd.java | 14 ++++++++++++++ ui/src/views/compute/AttachIso.vue | 4 ++-- ui/src/views/compute/AutoScaleVmProfile.vue | 1 + ui/src/views/compute/CreateAutoScaleVmGroup.vue | 2 ++ ui/src/views/compute/CreateKubernetesCluster.vue | 3 ++- ui/src/views/compute/DeployVM.vue | 4 ++++ ui/src/views/compute/DeployVnfAppliance.vue | 1 + ui/src/views/compute/EditVM.vue | 1 + ui/src/views/compute/ReinstallVm.vue | 1 + ui/src/views/compute/ResetUserData.vue | 1 + ui/src/views/compute/ScaleVM.vue | 1 + ui/src/views/tools/ImportUnmanagedInstance.vue | 1 + 13 files changed, 32 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java index 760a531e899..346eca8cff0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java @@ -57,7 +57,7 @@ public class ListIsosCmd extends BaseListTaggedResourcesCmd implements UserCmd { @Parameter(name = ApiConstants.IS_PUBLIC, type = CommandType.BOOLEAN, description = "true if the ISO is publicly available to all users, false otherwise.") private Boolean publicIso; - @Parameter(name = ApiConstants.IS_READY, type = CommandType.BOOLEAN, description = "true if this ISO is ready to be deployed") + @Parameter(name = ApiConstants.IS_READY, type = CommandType.BOOLEAN, description = "list ISOs that are ready to be deployed") private Boolean ready; @Parameter(name = ApiConstants.ISO_FILTER, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java index 4727e395c41..223ac57b11f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java @@ -126,6 +126,9 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User since = "4.21.0") private Long extensionId; + @Parameter(name = ApiConstants.IS_READY, type = CommandType.BOOLEAN, description = "list templates that are ready to be deployed", since = "4.21.0") + private Boolean ready; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -195,6 +198,13 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User boolean onlyReady = (templateFilter == TemplateFilter.featured) || (templateFilter == TemplateFilter.selfexecutable) || (templateFilter == TemplateFilter.sharedexecutable) || (templateFilter == TemplateFilter.executable && isAccountSpecific) || (templateFilter == TemplateFilter.community); + + if (!onlyReady) { + if (isReady() != null && isReady().booleanValue() != onlyReady) { + onlyReady = isReady().booleanValue(); + } + } + return onlyReady; } @@ -230,6 +240,10 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User return extensionId; } + public Boolean isReady() { + return ready; + } + @Override public String getCommandName() { return s_name; diff --git a/ui/src/views/compute/AttachIso.vue b/ui/src/views/compute/AttachIso.vue index 9a1bcb50a42..aafed017a21 100644 --- a/ui/src/views/compute/AttachIso.vue +++ b/ui/src/views/compute/AttachIso.vue @@ -106,8 +106,8 @@ export default { const params = { listall: true, zoneid: this.resource.zoneid, - isready: true, - isofilter: isoFilter + isofilter: isoFilter, + isready: true } return new Promise((resolve, reject) => { getAPI('listIsos', params).then((response) => { diff --git a/ui/src/views/compute/AutoScaleVmProfile.vue b/ui/src/views/compute/AutoScaleVmProfile.vue index 92ff9c8777b..9d944c59898 100644 --- a/ui/src/views/compute/AutoScaleVmProfile.vue +++ b/ui/src/views/compute/AutoScaleVmProfile.vue @@ -424,6 +424,7 @@ export default { } if (isAdmin()) { params.templatefilter = 'all' + params.isready = true } else { params.templatefilter = 'executable' } diff --git a/ui/src/views/compute/CreateAutoScaleVmGroup.vue b/ui/src/views/compute/CreateAutoScaleVmGroup.vue index 292d3fc6a7f..3c509462958 100644 --- a/ui/src/views/compute/CreateAutoScaleVmGroup.vue +++ b/ui/src/views/compute/CreateAutoScaleVmGroup.vue @@ -1881,6 +1881,7 @@ export default { apiName = 'listTemplates' params.listall = true params.templatefilter = this.isNormalAndDomainUser ? 'executable' : 'all' + params.isready = true params.id = this.queryTemplateId this.dataPreFill.templateid = this.queryTemplateId } else if (this.queryNetworkId) { @@ -2989,6 +2990,7 @@ export default { args.arch = this.selectedArchitecture } args.templatefilter = templateFilter + args.isready = true args.details = 'all' args.showicon = 'true' args.id = this.queryTemplateId diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index 7a2bcec49b9..f945da73213 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -671,7 +671,8 @@ export default { for (const filtername of filters) { const params = { templatefilter: filtername, - forcks: true + forcks: true, + isready: true } this.templateLoading = true getAPI('listTemplates', params).then(json => { diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index cd248f123f1..e4b15e5bbe2 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -1796,12 +1796,14 @@ export default { apiName = 'listTemplates' params.listall = true params.templatefilter = this.isNormalAndDomainUser ? 'executable' : 'all' + params.isready = true params.id = this.queryTemplateId this.dataPreFill.templateid = this.queryTemplateId } else if (this.queryIsoId) { apiName = 'listIsos' params.listall = true params.isofilter = this.isNormalAndDomainUser ? 'executable' : 'all' + params.isready = true params.id = this.queryIsoId this.dataPreFill.isoid = this.queryIsoId } else if (this.queryNetworkId) { @@ -2617,6 +2619,7 @@ export default { args.domainid = store.getters.project?.id ? null : this.owner.domainid args.projectid = store.getters.project?.id || this.owner.projectid args.templatefilter = templateFilter + args.isready = true args.details = 'all' args.showicon = 'true' args.id = this.queryTemplateId @@ -2652,6 +2655,7 @@ export default { args.domainid = store.getters.project?.id ? null : this.owner.domainid args.projectid = store.getters.project?.id || this.owner.projectid args.isoFilter = isoFilter + args.isready = true args.bootable = true args.showicon = 'true' args.id = this.queryIsoId diff --git a/ui/src/views/compute/DeployVnfAppliance.vue b/ui/src/views/compute/DeployVnfAppliance.vue index cf8677a2c61..6a467c34cb6 100644 --- a/ui/src/views/compute/DeployVnfAppliance.vue +++ b/ui/src/views/compute/DeployVnfAppliance.vue @@ -2555,6 +2555,7 @@ export default { args.arch = this.selectedArchitecture } args.templatefilter = templateFilter + args.isready = true args.details = 'all' args.showicon = 'true' args.id = this.queryTemplateId diff --git a/ui/src/views/compute/EditVM.vue b/ui/src/views/compute/EditVM.vue index e35ca6dd49d..7489674b966 100644 --- a/ui/src/views/compute/EditVM.vue +++ b/ui/src/views/compute/EditVM.vue @@ -283,6 +283,7 @@ export default { params.id = this.resource.templateid params.isrecursive = true params.templatefilter = 'all' + params.isready = true var apiName = 'listTemplates' getAPI(apiName, params).then(json => { const templateResponses = json.listtemplatesresponse.template diff --git a/ui/src/views/compute/ReinstallVm.vue b/ui/src/views/compute/ReinstallVm.vue index 982c4543a37..4c7ad2191c0 100644 --- a/ui/src/views/compute/ReinstallVm.vue +++ b/ui/src/views/compute/ReinstallVm.vue @@ -367,6 +367,7 @@ export default { } args.zoneid = this.resource.zoneid args.templatefilter = templateFilter + args.isready = true if (this.resource.arch) { args.arch = this.resource.arch } diff --git a/ui/src/views/compute/ResetUserData.vue b/ui/src/views/compute/ResetUserData.vue index c05c9452b2d..2870af7dd1a 100644 --- a/ui/src/views/compute/ResetUserData.vue +++ b/ui/src/views/compute/ResetUserData.vue @@ -245,6 +245,7 @@ export default { params.id = this.resource.templateid params.isrecursive = true params.templatefilter = 'all' + params.isready = true var apiName = 'listTemplates' getAPI(apiName, params).then(json => { const templateResponses = json.listtemplatesresponse.template diff --git a/ui/src/views/compute/ScaleVM.vue b/ui/src/views/compute/ScaleVM.vue index 1a80f595f63..a040a81d32b 100644 --- a/ui/src/views/compute/ScaleVM.vue +++ b/ui/src/views/compute/ScaleVM.vue @@ -189,6 +189,7 @@ export default { return new Promise((resolve, reject) => { getAPI('listTemplates', { templatefilter: 'all', + isready: true, id: this.resource.templateid }).then(response => { var template = response?.listtemplatesresponse?.template?.[0] || null diff --git a/ui/src/views/tools/ImportUnmanagedInstance.vue b/ui/src/views/tools/ImportUnmanagedInstance.vue index aabd1df7572..4c1a3d53fbc 100644 --- a/ui/src/views/tools/ImportUnmanagedInstance.vue +++ b/ui/src/views/tools/ImportUnmanagedInstance.vue @@ -584,6 +584,7 @@ export default { isLoad: true, options: { templatefilter: 'all', + isready: true, hypervisor: this.cluster.hypervisortype, showicon: true }, From 2944bd1eda1fd587dc8301a15c0ea70a6919931a Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 31 Jul 2025 01:41:40 -0400 Subject: [PATCH 11/53] API: Set Object name when expunging VM (#11352) --- .../apache/cloudstack/api/command/user/vm/DestroyVMCmd.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index aa121162cb4..18a9d2058a6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -140,7 +140,8 @@ public class DestroyVMCmd extends BaseAsyncCmd implements UserCmd { if (responses != null && !responses.isEmpty()) { response = responses.get(0); } - response.setResponseName("virtualmachine"); + response.setResponseName(getCommandName()); + response.setObjectName("virtualmachine"); setResponseObject(response); } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to destroy vm"); From f58372e97b5f3f6db45f4b6ac4dfc48a6ca42bcf Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 31 Jul 2025 15:09:12 +0530 Subject: [PATCH 12/53] [UI] Use GET request method for list API calls (#11354) * [UI] Use GET request method for list API calls * Updated UI unit tests --- ui/src/api/index.js | 19 +++++++++++++++ ui/src/components/view/ListView.vue | 24 +++++++++---------- ui/src/components/view/TreeView.vue | 8 +++---- .../widgets/InfiniteScrollSelect.vue | 4 ++-- ui/src/views/AutogenView.vue | 6 ++--- .../views/compute/CreateAutoScaleVmGroup.vue | 4 ++-- ui/src/views/compute/DeployVM.vue | 6 ++--- ui/src/views/compute/DeployVnfAppliance.vue | 6 ++--- ui/src/views/iam/DomainView.vue | 4 ++-- .../infra/network/providers/ProviderItem.vue | 4 ++-- .../tungsten/TungstenFabricTableView.vue | 6 ++--- ui/src/views/tools/ManageInstances.vue | 4 ++-- ui/src/views/tools/ManageVolumes.vue | 2 +- ui/tests/unit/views/AutogenView.spec.js | 24 +++++++++---------- 14 files changed, 70 insertions(+), 51 deletions(-) diff --git a/ui/src/api/index.js b/ui/src/api/index.js index 1f532c36336..0c8e8e9696c 100644 --- a/ui/src/api/index.js +++ b/ui/src/api/index.js @@ -23,6 +23,19 @@ import { ACCESS_TOKEN } from '@/store/mutation-types' +const getAPICommandsRegex = /^(get|list|query|find)\w+$/i +const additionalGetAPICommandsList = [ + 'isaccountallowedtocreateofferingswithtags', + 'readyforshutdown', + 'cloudianisenabled', + 'quotabalance', + 'quotasummary', + 'quotatarifflist', + 'quotaisenabled', + 'quotastatement', + 'verifyoauthcodeandgetuser' +] + export function getAPI (command, args = {}) { args.command = command args.response = 'json' @@ -64,6 +77,12 @@ export function postAPI (command, data = {}) { }) } +export function callAPI (command, args = {}) { + const isGetAPICommand = getAPICommandsRegex.test(command) || additionalGetAPICommandsList.includes(command.toLowerCase()) + const call = isGetAPICommand ? getAPI : postAPI + return call(command, args) +} + export function login (arg) { if (!sourceToken.checkExistSource()) { sourceToken.init() diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 205e340652d..f1f9469a400 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -1230,45 +1230,45 @@ export default { this.editableValue = record.value }, getUpdateApi () { - let apiString = '' + let apiCommand = '' switch (this.$route.name) { case 'template': - apiString = 'updateTemplate' + apiCommand = 'updateTemplate' break case 'iso': - apiString = 'updateIso' + apiCommand = 'updateIso' break case 'zone': - apiString = 'updateZone' + apiCommand = 'updateZone' break case 'computeoffering': case 'systemoffering': - apiString = 'updateServiceOffering' + apiCommand = 'updateServiceOffering' break case 'diskoffering': - apiString = 'updateDiskOffering' + apiCommand = 'updateDiskOffering' break case 'networkoffering': - apiString = 'updateNetworkOffering' + apiCommand = 'updateNetworkOffering' break case 'vpcoffering': - apiString = 'updateVPCOffering' + apiCommand = 'updateVPCOffering' break case 'guestoscategory': - apiString = 'updateOsCategory' + apiCommand = 'updateOsCategory' break } - return apiString + return apiCommand }, isOrderUpdatable () { return this.getUpdateApi() in this.$store.getters.apis }, handleUpdateOrder (id, index) { this.parentToggleLoading() - const apiString = this.getUpdateApi() + const apiCommand = this.getUpdateApi() return new Promise((resolve, reject) => { - postAPI(apiString, { + postAPI(apiCommand, { id, sortKey: index }).then((response) => { diff --git a/ui/src/components/view/TreeView.vue b/ui/src/components/view/TreeView.vue index 0307b097475..f767b7adef2 100644 --- a/ui/src/components/view/TreeView.vue +++ b/ui/src/components/view/TreeView.vue @@ -92,7 +92,7 @@ diff --git a/ui/src/components/view/DeployVMFromBackup.vue b/ui/src/components/view/DeployVMFromBackup.vue new file mode 100644 index 00000000000..8d929a1fed0 --- /dev/null +++ b/ui/src/components/view/DeployVMFromBackup.vue @@ -0,0 +1,2663 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + + + diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 3a4d0fa043b..24c2043b90a 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -64,7 +64,8 @@
- {{ volume.type }} - {{ volume.path }} ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB) + {{ volume.type }} - {{ volume.path }} + {{ volume.type }} - {{ volume.path }} ({{ parseFloat(volume.size / (1024.0 * 1024.0 * 1024.0)).toFixed(1) }} GB)
diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index f1f9469a400..18ce8ed7912 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -342,7 +342,8 @@ }} + @@ -110,7 +116,7 @@ export default { }, computed: { columns () { - return [ + const cols = [ { key: 'icon', title: '', @@ -134,7 +140,17 @@ export default { key: 'keep', title: this.$t('label.keep'), dataIndex: 'maxbackups' - }, + } + ] + const hasQuiesce = this.dataSource.some(item => 'quiescevm' in item) + if (hasQuiesce) { + cols.push({ + key: 'quiescevm', + title: this.$t('label.quiescevm'), + dataIndex: 'quiescevm' + }) + } + cols.push( { key: 'timezone', title: this.$t('label.timezone'), @@ -146,7 +162,8 @@ export default { dataIndex: 'actions', width: 80 } - ] + ) + return cols } }, mounted () { diff --git a/ui/src/views/compute/backup/FormSchedule.vue b/ui/src/views/compute/backup/FormSchedule.vue index 01cae9d7d8f..643ae116916 100644 --- a/ui/src/views/compute/backup/FormSchedule.vue +++ b/ui/src/views/compute/backup/FormSchedule.vue @@ -132,6 +132,14 @@ + + + + + +
import { ref, reactive, toRaw } from 'vue' -import { postAPI } from '@/api' +import { getAPI, postAPI } from '@/api' import { timeZone } from '@/utils/timezone' import { mixinForm } from '@/utils/mixin' import debounce from 'lodash/debounce' +import TooltipLabel from '@/components/widgets/TooltipLabel' export default { name: 'FormSchedule', mixins: [mixinForm], + components: { + TooltipLabel + }, props: { loading: { type: Boolean, @@ -185,13 +197,18 @@ export default { dayOfMonth: [], timeZoneMap: [], fetching: false, + backupProvider: null, actionLoading: false, listDayOfWeek: ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'] } }, + beforeCreate () { + this.apiParams = this.$getApiParams('createBackupSchedule') + }, created () { this.initForm() this.fetchTimeZone() + this.fetchBackupOffering() }, inject: ['refreshSchedule', 'closeSchedule'], methods: { @@ -208,6 +225,16 @@ export default { timezone: [{ required: true, message: `${this.$t('message.error.select')}` }] }) }, + fetchBackupOffering () { + getAPI('listBackupOfferings', { id: this.resource.backupofferingid }).then(json => { + if (json.listbackupofferingsresponse && json.listbackupofferingsresponse.backupoffering) { + const backupoffering = json.listbackupofferingsresponse.backupoffering[0] + this.backupProvider = backupoffering.provider + } + }).catch(error => { + this.$notifyError(error) + }) + }, fetchTimeZone (value) { this.timeZoneMap = [] this.fetching = true @@ -261,6 +288,9 @@ export default { params.intervaltype = values.intervaltype params.maxbackups = values.maxbackups params.timezone = values.timezone + if (values.quiescevm) { + params.quiescevm = values.quiescevm + } switch (values.intervaltype) { case 'hourly': params.schedule = values.time diff --git a/ui/src/views/compute/wizard/NetworkConfiguration.vue b/ui/src/views/compute/wizard/NetworkConfiguration.vue index b2f05dbb6a5..f3192d278bd 100644 --- a/ui/src/views/compute/wizard/NetworkConfiguration.vue +++ b/ui/src/views/compute/wizard/NetworkConfiguration.vue @@ -77,6 +77,14 @@ +
+ + {{ $t('label.fetch.from.backup') }} + + + {{ $t('label.clear') }} + +
@@ -219,28 +227,31 @@ export default { }, updateNetworkData (name, key, value) { this.formRef.value.validate().then(() => { - this.$emit('handler-error', false) - const index = this.networks.findIndex(item => item.key === key) - if (index === -1) { - const networkItem = {} - networkItem.key = key - networkItem[name] = value - this.networks.push(networkItem) - this.$emit('update-network-config', this.networks) - return - } - - this.networks.filter((item, index) => { - if (item.key === key) { - this.networks[index][name] = value - } - }) + this.updateNetworkDataWithoutValidation(name, key, value) this.$emit('update-network-config', this.networks) }).catch((error) => { this.formRef.value.scrollToField(error.errorFields[0].name) this.$emit('handler-error', true) }) }, + updateNetworkDataWithoutValidation (name, key, value) { + this.$emit('handler-error', false) + const index = this.networks.findIndex(item => item.key === key) + if (index === -1) { + const networkItem = {} + networkItem.key = key + networkItem[name] = value + this.networks.push(networkItem) + this.$emit('update-network-config', this.networks) + return + } + + this.networks.filter((item, index) => { + if (item.key === key) { + this.networks[index][name] = value + } + }) + }, removeItem (id) { this.dataItems = this.dataItems.filter(item => item.id !== id) if (this.selectedRowKeys.includes(id)) { @@ -250,6 +261,59 @@ export default { } } }, + handleFetchIpAddresses () { + if (!this.preFillContent.networkids) { + return + } + if (!this.preFillContent.ipAddresses && !this.preFillContent.macAddresses) { + return + } + + const networkIds = this.dataItems.map(item => item.id) + this.dataItems.forEach(record => { + const ipAddressKey = 'ipAddress' + record.id + const macAddressKey = 'macAddress' + record.id + this.form[ipAddressKey] = '' + this.form[macAddressKey] = '' + }) + + networkIds.forEach((networkId) => { + const backupIndex = this.preFillContent.networkids.findIndex(id => id === networkId) + if (backupIndex !== -1) { + if (this.preFillContent.ipAddresses && backupIndex < this.preFillContent.ipAddresses.length) { + const ipAddress = this.preFillContent.ipAddresses[backupIndex] + if (ipAddress) { + const ipAddressKey = 'ipAddress' + networkId + this.form[ipAddressKey] = ipAddress + this.updateNetworkDataWithoutValidation('ipAddress', networkId, ipAddress) + } + } + + if (this.preFillContent.macAddresses && backupIndex < this.preFillContent.macAddresses.length) { + const macAddress = this.preFillContent.macAddresses[backupIndex] + if (macAddress) { + const macAddressKey = 'macAddress' + networkId + this.form[macAddressKey] = macAddress + this.updateNetworkDataWithoutValidation('macAddress', networkId, macAddress) + } + } + } + }) + }, + handleClearIpAddresses () { + this.dataItems.forEach(record => { + const ipAddressKey = 'ipAddress' + record.id + const macAddressKey = 'macAddress' + record.id + this.form[ipAddressKey] = '' + this.form[macAddressKey] = '' + + this.updateNetworkDataWithoutValidation('ipAddress', record.id, '') + this.updateNetworkDataWithoutValidation('macAddress', record.id, '') + }) + + this.networks = [] + this.$emit('update-network-config', this.networks) + }, async validatorMacAddress (rule, value) { if (!value || value === '') { return Promise.resolve() diff --git a/ui/src/views/compute/wizard/NetworkSelection.vue b/ui/src/views/compute/wizard/NetworkSelection.vue index 3a4fd8c88f9..fffcbf7e3e7 100644 --- a/ui/src/views/compute/wizard/NetworkSelection.vue +++ b/ui/src/views/compute/wizard/NetworkSelection.vue @@ -260,8 +260,11 @@ export default { }) if (!this.loading) { if (this.preFillContent.networkids) { - this.selectedRowKeys = this.preFillContent.networkids - this.$emit('select-network-item', this.preFillContent.networkids) + const validNetworkIds = this.preFillContent.networkids.filter(networkId => + this.items.some(item => item.id === networkId) + ) + this.selectedRowKeys = validNetworkIds + this.$emit('select-network-item', validNetworkIds) } else { if (this.items && this.items.length > 0) { if (this.oldZoneId === this.zoneId) { diff --git a/ui/src/views/compute/wizard/TemplateIsoSelection.vue b/ui/src/views/compute/wizard/TemplateIsoSelection.vue index 9393a7860de..4979068dac7 100644 --- a/ui/src/views/compute/wizard/TemplateIsoSelection.vue +++ b/ui/src/views/compute/wizard/TemplateIsoSelection.vue @@ -25,7 +25,7 @@ @@ -103,12 +103,19 @@ export default { deep: true, handler (items) { const key = this.inputDecorator.slice(0, -2) + if (this.pagination) { + return + } for (const filter of this.filterOpts) { - if (items[filter.id] && items[filter.id][key] && items[filter.id][key].length > 0) { - if (!this.pagination) { + if (this.preFillContent.templateid) { + if (items[filter.id]?.[key]?.some(item => item.id === this.preFillContent.templateid)) { this.filterType = filter.id this.checkedValue = items[filter.id][key][0].id + break } + } else if (items[filter.id]?.[key]?.length > 0) { + this.filterType = filter.id + this.checkedValue = items[filter.id][key][0].id break } } diff --git a/ui/src/views/compute/wizard/VolumeDiskOfferingSelectView.vue b/ui/src/views/compute/wizard/VolumeDiskOfferingSelectView.vue new file mode 100644 index 00000000000..4fc7c3fc972 --- /dev/null +++ b/ui/src/views/compute/wizard/VolumeDiskOfferingSelectView.vue @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/dashboard/CapacityDashboard.vue b/ui/src/views/dashboard/CapacityDashboard.vue index 7e0b8180ac8..9b374628886 100644 --- a/ui/src/views/dashboard/CapacityDashboard.vue +++ b/ui/src/views/dashboard/CapacityDashboard.vue @@ -210,14 +210,14 @@ - +
-
+
{{ $t(ts[ctype]) }} @@ -377,6 +377,8 @@ export default { MEMORY: 'label.memory', PRIVATE_IP: 'label.management.ips', SECONDARY_STORAGE: 'label.secondary.storage', + BACKUP_STORAGE: 'label.backup.storage', + OBJECT_STORAGE: 'label.object.storage', STORAGE: 'label.primary.storage.used', STORAGE_ALLOCATED: 'label.primary.storage.allocated', VIRTUAL_NETWORK_PUBLIC_IP: 'label.public.ips', @@ -438,6 +440,8 @@ export default { case 'STORAGE': case 'STORAGE_ALLOCATED': case 'SECONDARY_STORAGE': + case 'BACKUP_STORAGE': + case 'OBJECT_STORAGE': case 'LOCAL_STORAGE': value = parseFloat(value / (1024 * 1024 * 1024.0), 10).toFixed(2) if (value >= 1024.0) { @@ -667,6 +671,13 @@ export default { min-height: 370px; } +.dashboard-storage { + width: 100%; + overflow-x:hidden; + overflow-y: scroll; + max-height: 370px; +} + .dashboard-event { width: 100%; overflow-x:hidden; diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue index dca3b719b6c..5410a9b9502 100644 --- a/ui/src/views/infra/AddObjectStorage.vue +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -25,10 +25,16 @@ layout="vertical" @finish="handleSubmit" > - + + - + + - + + - + + + + +
-
{{ $t('label.cancel') }} {{ $t('label.ok') }} @@ -95,6 +109,7 @@ import { ref, reactive, toRaw } from 'vue' import { getAPI } from '@/api' import { mixinForm } from '@/utils/mixin' import ResourceIcon from '@/components/view/ResourceIcon' +import TooltipLabel from '@/components/widgets/TooltipLabel' export default { name: 'AddObjectStorage', @@ -106,7 +121,8 @@ export default { } }, components: { - ResourceIcon + ResourceIcon, + TooltipLabel }, inject: ['parentFetchData'], data () { @@ -116,6 +132,9 @@ export default { loading: false } }, + beforeCreate () { + this.apiParams = this.$getApiParams('addObjectStoragePool') + }, created () { this.initForm() this.fetchData() @@ -147,7 +166,8 @@ export default { const values = this.handleRemoveFields(formRaw) var data = { - name: values.name + name: values.name, + size: values.size } var provider = values.provider diff --git a/ui/src/views/storage/CreateVMFromBackup.vue b/ui/src/views/storage/CreateVMFromBackup.vue new file mode 100644 index 00000000000..f9dbf535d06 --- /dev/null +++ b/ui/src/views/storage/CreateVMFromBackup.vue @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 2e97e238f09..49d79999716 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1075,7 +1075,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna private boolean isBackupEvent(String eventType) { return eventType != null && ( eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN) || - eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE) || + eventType.equals(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED) || eventType.equals(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC)); } @@ -2030,10 +2030,10 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna if (EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN.equals(event.getType())) { final UsageBackupVO backupVO = new UsageBackupVO(zoneId, accountId, domainId, vmId, backupOfferingId, created); usageBackupDao.persist(backupVO); - } else if (EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE.equals(event.getType())) { - usageBackupDao.removeUsage(accountId, vmId, event.getCreateDate()); + } else if (EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED.equals(event.getType())) { + usageBackupDao.removeUsage(accountId, vmId, backupOfferingId, event.getCreateDate()); } else if (EventTypes.EVENT_VM_BACKUP_USAGE_METRIC.equals(event.getType())) { - usageBackupDao.updateMetrics(vmId, event.getSize(), event.getVirtualSize()); + usageBackupDao.updateMetrics(vmId, backupOfferingId, event.getSize(), event.getVirtualSize()); } } From f62b85dffee488f267ac363a3eb47dc956b41401 Mon Sep 17 00:00:00 2001 From: levindecaro <36956864+levindecaro@users.noreply.github.com> Date: Thu, 31 Jul 2025 18:36:54 +0800 Subject: [PATCH 16/53] fix fsvm-init.yml to detect virtio-scsi in kvm (#11070) * fix fsvm-init.yml to detect virtio-scsi in kvm * Update fsvm-init.yml to handle universal block device case. --- .../storagevm/src/main/resources/conf/fsvm-init.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/plugins/storage/sharedfs/storagevm/src/main/resources/conf/fsvm-init.yml b/plugins/storage/sharedfs/storagevm/src/main/resources/conf/fsvm-init.yml index 4d3572162c8..ceafa6c3cb1 100644 --- a/plugins/storage/sharedfs/storagevm/src/main/resources/conf/fsvm-init.yml +++ b/plugins/storage/sharedfs/storagevm/src/main/resources/conf/fsvm-init.yml @@ -30,14 +30,9 @@ write_files: } get_block_device() { - if [ "$HYPERVISOR" == "kvm" ]; then - BLOCK_DEVICE="vdb" - elif [ "$HYPERVISOR" == "xenserver" ]; then - BLOCK_DEVICE="xvdb" - elif [ "$HYPERVISOR" == "vmware" ]; then - BLOCK_DEVICE="sdb" - else - log "Unknown hypervisor" + BLOCK_DEVICE=$(lsblk -dn -o NAME,TYPE | awk '$2=="disk"{print $1}' | tail -n 1) + if [ -z "$BLOCK_DEVICE" ]; then + log "Unknown data disk" exit 1 fi echo "$BLOCK_DEVICE" From 5ea1ada59a43bccc699f09efb3a7a89df1562017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jandre?= <48719461+JoaoJandre@users.noreply.github.com> Date: Thu, 31 Jul 2025 07:42:17 -0300 Subject: [PATCH 17/53] Allow full clone volumes with thin provisioning in KVM (#11177) It adds a configuration called create.full.clone to the agent.properties file. When set to true, all QCOW2 volumes created will be full-clone. If false (default), the current behavior remains, where only FAT and SPARSE volumes are full-clone and THIN volumes are linked-clone. --- agent/conf/agent.properties | 3 +++ .../cloud/agent/properties/AgentProperties.java | 8 ++++++++ .../kvm/storage/LibvirtStorageAdaptor.java | 16 +++++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index e70acee229d..cd31b0db56d 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -447,3 +447,6 @@ iscsi.session.cleanup.enabled=false # Timeout (in seconds) to wait for the incremental snapshot to complete. # incremental.snapshot.timeout=10800 + +# If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise. +# create.full.clone=false diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 47255762a05..847d1bb2396 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -863,6 +863,14 @@ public class AgentProperties{ * */ public static final Property REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800); + /** + * If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise.
+ * Data type: Boolean.
+ * Default value: false + */ + public static final Property CREATE_FULL_CLONE = new Property<>("create.full.clone", false); + + public static class Property { private String name; private T defaultValue; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 7c66a91876f..bf851831cd0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -32,6 +32,8 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.utils.cryptsetup.KeyFile; import org.apache.cloudstack.utils.qemu.QemuImageOptions; @@ -1315,14 +1317,22 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { passphraseObjects.add(QemuObject.prepareSecretForQemuImg(format, QemuObject.EncryptFormat.LUKS, keyFile.toString(), "sec0", options)); disk.setQemuEncryptFormat(QemuObject.EncryptFormat.LUKS); } + + QemuImgFile srcFile = new QemuImgFile(template.getPath(), template.getFormat()); + Boolean createFullClone = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CREATE_FULL_CLONE); switch(provisioningType){ case THIN: - QemuImgFile backingFile = new QemuImgFile(template.getPath(), template.getFormat()); - qemu.create(destFile, backingFile, options, passphraseObjects); + logger.info("Creating volume [{}] {} backing file [{}] as the property [{}] is [{}].", destFile.getFileName(), createFullClone ? "without" : "with", + template.getPath(), AgentProperties.CREATE_FULL_CLONE.getName(), createFullClone); + if (createFullClone) { + qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); + } else { + qemu.create(destFile, srcFile, options, passphraseObjects); + } break; case SPARSE: case FAT: - QemuImgFile srcFile = new QemuImgFile(template.getPath(), template.getFormat()); + srcFile = new QemuImgFile(template.getPath(), template.getFormat()); qemu.convert(srcFile, destFile, options, passphraseObjects, null, false); break; } From ed0d606e983cd5ae2e4cd5b7428d054b9768b6ca Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 31 Jul 2025 08:12:47 -0300 Subject: [PATCH 18/53] Find system VM templates for CKS clusters and SharedFS honouring the preferred architecture (#10946) * Find system VM templates for CKS cluster honouring the preferred architecture * Fix unit tests * Fix checkstyle * Sort instead of filtering by preferred arch * Remove unnecesary stubs * Restore java version * Address review comments * Fail and display error message in case the CKS ISO arch doesnt match the selected template arch * Prefer CKS ISO arch instead of the system VM setting --- .../com/cloud/storage/dao/VMTemplateDao.java | 2 +- .../cloud/storage/dao/VMTemplateDaoImpl.java | 11 +++++- .../storage/dao/VMTemplateDaoImplTest.java | 21 ++++++++++ .../cluster/KubernetesClusterManagerImpl.java | 38 +++++++++++++++++-- .../KubernetesClusterActionWorker.java | 4 +- ...esClusterResourceModifierActionWorker.java | 2 +- .../KubernetesClusterManagerImplTest.java | 19 ++++++++++ .../lifecycle/StorageVmSharedFSLifeCycle.java | 3 +- .../StorageVmSharedFSLifeCycleTest.java | 3 +- 9 files changed, 93 insertions(+), 10 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 0b40366a866..c751f81f927 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -72,7 +72,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findSystemVMTemplate(long zoneId); - VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); + VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch); List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType, String preferredArch); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 12c00a3209a..b6796cf8f9d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -23,6 +23,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -578,11 +579,19 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } @Override - public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) { + public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch) { List templates = listAllReadySystemVMTemplates(zoneId); if (CollectionUtils.isEmpty(templates)) { return null; } + if (StringUtils.isNotBlank(preferredArch)) { + // Sort the templates by preferred architecture first + templates = templates.stream() + .sorted(Comparator.comparing( + x -> !x.getArch().getType().equalsIgnoreCase(preferredArch) + )) + .collect(Collectors.toList()); + } if (hypervisorType == HypervisorType.Any) { return templates.get(0); } diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java index df0b36ebdbf..e97a887cc47 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -186,4 +187,24 @@ public class VMTemplateDaoImplTest { VMTemplateVO result = templateDao.findLatestTemplateByTypeAndHypervisorAndArch(hypervisorType, arch, type); assertNull(result); } + + @Test + public void testFindSystemVMReadyTemplate() { + Long zoneId = 1L; + VMTemplateVO systemVmTemplate1 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate1.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate2 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate2.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate3 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate3.getArch()).thenReturn(CPU.CPUArch.arm64); + Mockito.when(systemVmTemplate3.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + List templates = Arrays.asList(systemVmTemplate1, systemVmTemplate2, systemVmTemplate3); + Mockito.when(hostDao.listDistinctHypervisorTypes(zoneId)).thenReturn(Arrays.asList(Hypervisor.HypervisorType.KVM)); + SearchBuilder sb = mock(SearchBuilder.class); + templateDao.readySystemTemplateSearch = sb; + when(sb.create()).thenReturn(mock(SearchCriteria.class)); + doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + VMTemplateVO readyTemplate = templateDao.findSystemVMReadyTemplate(zoneId, Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64.getType()); + Assert.assertEquals(CPU.CPUArch.arm64, readyTemplate.getArch()); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 411e6af883e..d1babb547f8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -433,8 +433,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return null; } - public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) { - VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType); + public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType, + KubernetesSupportedVersion clusterKubernetesVersion) { + String systemVMPreferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(dataCenter.getId()); + VMTemplateVO cksIso = clusterKubernetesVersion != null ? + templateDao.findById(clusterKubernetesVersion.getIsoId()) : + null; + String preferredArchitecture = getCksClusterPreferredArch(systemVMPreferredArchitecture, cksIso); + VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType, preferredArchitecture); if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) { logger.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter)); template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId())); @@ -445,6 +451,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return template; } + protected String getCksClusterPreferredArch(String systemVMPreferredArchitecture, VMTemplateVO cksIso) { + if (cksIso == null) { + return systemVMPreferredArchitecture; + } + String cksIsoArchName = cksIso.getArch().name(); + return cksIsoArchName.equals(systemVMPreferredArchitecture) ? systemVMPreferredArchitecture : cksIsoArchName; + } + protected void validateIsolatedNetworkIpRules(long ipId, FirewallRule.Purpose purpose, Network network, int clusterTotalNodeCount) { List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose); for (FirewallRuleVO rule : rules) { @@ -1302,7 +1316,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); - final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType()); + final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType(), clusterKubernetesVersion); + + compareKubernetesIsoArchToSelectedTemplateArch(clusterKubernetesVersion, finalTemplate); + final long cores = serviceOffering.getCpu() * (controlNodeCount + clusterSize); final long memory = serviceOffering.getRamSize() * (controlNodeCount + clusterSize); @@ -1331,6 +1348,21 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return cluster; } + private void compareKubernetesIsoArchToSelectedTemplateArch(KubernetesSupportedVersion clusterKubernetesVersion, VMTemplateVO finalTemplate) { + VMTemplateVO cksIso = templateDao.findById(clusterKubernetesVersion.getIsoId()); + if (cksIso == null) { + String err = String.format("Cannot find Kubernetes ISO associated to the Kubernetes version %s (id=%s)", + clusterKubernetesVersion.getName(), clusterKubernetesVersion.getUuid()); + throw new CloudRuntimeException(err); + } + if (!cksIso.getArch().equals(finalTemplate.getArch())) { + String err = String.format("The selected Kubernetes ISO %s arch (%s) doesn't match the template %s arch (%s) " + + "to deploy the Kubernetes cluster", + clusterKubernetesVersion.getName(), cksIso.getArch(), finalTemplate.getName(), finalTemplate.getArch()); + throw new CloudRuntimeException(err); + } + } + private SecurityGroup getOrCreateSecurityGroupForAccount(Account owner) { String securityGroupName = String.format("%s-%s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME, owner.getUuid()); String securityGroupDesc = String.format("%s and account %s", KubernetesClusterActionWorker.CKS_SECURITY_GROUP_DESCRIPTION, owner.getName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 29ecde477a4..99c4d4c9c96 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -29,6 +29,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -194,7 +195,8 @@ public class KubernetesClusterActionWorker { DataCenterVO dataCenterVO = dataCenterDao.findById(zoneId); VMTemplateVO template = templateDao.findById(templateId); Hypervisor.HypervisorType type = template.getHypervisorType(); - this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type); + KubernetesSupportedVersionVO kubernetesSupportedVersion = kubernetesSupportedVersionDao.findById(this.kubernetesCluster.getKubernetesVersionId()); + this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type, kubernetesSupportedVersion); this.sshKeyFile = getManagementServerSshPublicKeyFile(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 5dfab87dd71..d92d0692ca1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -250,7 +250,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); HostVO h = hp.first(); - if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType())) { + if (!h.getHypervisorType().equals(clusterTemplate.getHypervisorType()) || !h.getArch().equals(clusterTemplate.getArch())) { continue; } hostDao.loadHostTags(h); diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index a6d46ffc9aa..287e045a18e 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -21,6 +21,7 @@ package com.cloud.kubernetes.cluster; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; +import com.cloud.cpu.CPU; import com.cloud.dc.DataCenter; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; @@ -292,4 +293,22 @@ public class KubernetesClusterManagerImplTest { Mockito.when(kubernetesClusterDao.findById(Mockito.anyLong())).thenReturn(cluster); Assert.assertTrue(kubernetesClusterManager.removeVmsFromCluster(cmd).size() > 0); } + + @Test + public void testGetCksClusterPreferredArchDifferentArchsPreferCKSIsoArch() { + String systemVMArch = "x86_64"; + VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class); + Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.arm64); + String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); + Assert.assertEquals(CPU.CPUArch.arm64.name(), cksClusterPreferredArch); + } + + @Test + public void testGetCksClusterPreferredArchSameArch() { + String systemVMArch = "x86_64"; + VMTemplateVO cksIso = Mockito.mock(VMTemplateVO.class); + Mockito.when(cksIso.getArch()).thenReturn(CPU.CPUArch.amd64); + String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso); + Assert.assertEquals(CPU.CPUArch.amd64.name(), cksClusterPreferredArch); + } } diff --git a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java index ed26963cf81..1850c3c218b 100644 --- a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java +++ b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java @@ -179,10 +179,11 @@ public class StorageVmSharedFSLifeCycle implements SharedFSLifeCycle { customParameterMap.put("maxIopsDo", maxIops.toString()); } List keypairs = new ArrayList(); + String preferredArchitecture = ResourceManager.SystemVmPreferredArchitecture.valueIn(zoneId); for (final Iterator iter = hypervisors.iterator(); iter.hasNext();) { final Hypervisor.HypervisorType hypervisor = iter.next(); - VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor); + VMTemplateVO template = templateDao.findSystemVMReadyTemplate(zoneId, hypervisor, preferredArchitecture); if (template == null && !iter.hasNext()) { throw new CloudRuntimeException(String.format("Unable to find the systemvm template for %s or it was not downloaded in %s.", hypervisor.toString(), zone.toString())); } diff --git a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java index 2cc909ce0d7..bcd9d509a9a 100644 --- a/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java +++ b/plugins/storage/sharedfs/storagevm/src/test/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycleTest.java @@ -236,7 +236,7 @@ public class StorageVmSharedFSLifeCycleTest { when(serviceOfferingDao.findById(s_serviceOfferingId)).thenReturn(serviceOffering); VMTemplateVO template = mock(VMTemplateVO.class); - when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(template); + when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM, ResourceManager.SystemVmPreferredArchitecture.defaultValue())).thenReturn(template); when(template.getId()).thenReturn(s_templateId); return sharedFS; @@ -303,7 +303,6 @@ public class StorageVmSharedFSLifeCycleTest { when(dataCenterDao.findById(s_zoneId)).thenReturn(zone); when(resourceMgr.getSupportedHypervisorTypes(s_zoneId, false, null)).thenReturn(List.of(Hypervisor.HypervisorType.KVM)); - when(templateDao.findSystemVMReadyTemplate(s_zoneId, Hypervisor.HypervisorType.KVM)).thenReturn(null); lifeCycle.deploySharedFS(sharedFS, s_networkId, s_diskOfferingId, s_size, s_minIops, s_maxIops); } From 9f5828a02708ec62b4db9193cc399e0b95f5086b Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 31 Jul 2025 08:44:55 -0400 Subject: [PATCH 19/53] UI: Fix cpu & memory details on list view for unmanaged k8s clusters (CAPC) (#11353) --- ui/src/components/view/ListView.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 18ce8ed7912..f4acba595b9 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -479,12 +479,12 @@ -