From 2e113e5ed704ea6d24295ff765c30f66a1d0601b Mon Sep 17 00:00:00 2001 From: Vitor Hugo Homem Marzarotto <59698484+vits-hugs@users.noreply.github.com> Date: Wed, 10 Sep 2025 06:06:16 -0300 Subject: [PATCH 001/217] Change log level of AgentHandler#processRequest() (#10869) Co-authored-by: Vitor Hugo Homem Marzarotto --- .../main/java/com/cloud/agent/manager/AgentManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 4b2578d20c4..7fa4b31b15b 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -1454,7 +1454,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl } } } catch (final Throwable th) { - s_logger.warn("Caught: ", th); + s_logger.error("Caught: ", th); answer = new Answer(cmd, false, th.getMessage()); } answers[i] = answer; @@ -1471,7 +1471,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl try { link.send(response.toBytes()); } catch (final ClosedChannelException e) { - s_logger.warn("Unable to send response because connection is closed: " + response); + s_logger.error("Unable to send response because connection is closed: " + response); } } From 253ac0362146c4e97ca9a4ac527c6abfa2dc29c7 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Fri, 12 Sep 2025 14:58:50 +0530 Subject: [PATCH 002/217] Management server: fix qemu-img path in cloudstack sudoers (#11614) --- server/conf/cloudstack-sudoers.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/conf/cloudstack-sudoers.in b/server/conf/cloudstack-sudoers.in index 710241022f5..6e799297574 100644 --- a/server/conf/cloudstack-sudoers.in +++ b/server/conf/cloudstack-sudoers.in @@ -18,7 +18,7 @@ # The CloudStack management server needs sudo permissions # without a password. -Cmnd_Alias CLOUDSTACK = /bin/mkdir, /bin/mount, /bin/umount, /bin/cp, /bin/chmod, /usr/bin/keytool, /bin/keytool, /bin/touch, /bin/find, /bin/df, /bin/ls, /bin/qemu-img +Cmnd_Alias CLOUDSTACK = /bin/mkdir, /bin/mount, /bin/umount, /bin/cp, /bin/chmod, /usr/bin/keytool, /bin/keytool, /bin/touch, /bin/find, /bin/df, /bin/ls, /bin/qemu-img, /usr/bin/qemu-img Defaults:@MSUSER@ !requiretty From 6d16ac2113a8e8d7b02458c655329574519585d2 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 12 Sep 2025 19:47:20 +0530 Subject: [PATCH 003/217] ScaleIO/PowerFlex smoke tests improvements, and some fixes (#11554) * ScaleIO/PowerFlex smoke tests improvements, and some fixes * Fix test_volumes.py, encrypted volume size check (for powerflex volumes) * Fix test_over_provisioning.py (over provisioning supported for powerflex) * Update vm snapshot tests * Update volume size delta in primary storage resource count for user vm volumes only The VR volumes resource count for PowerFlex volumes is updated here, resulting in resource count discrepancy (which is re-calculated through ResourceCountCheckTask later, and skips the VR volumes) * Fix test_import_unmanage_volumes.py (unsupported for powerflex) * Fix test_sharedfs_lifecycle.py (volume size check for powerflex) * Update powerflex.connect.on.demand config default to true --- .../orchestration/VolumeOrchestrator.java | 25 +++++--- .../resource/LibvirtComputingResource.java | 14 ++++- .../metrics/MetricsServiceImpl.java | 2 +- .../driver/ScaleIOPrimaryDataStoreDriver.java | 6 +- .../manager/ScaleIOSDCManagerImpl.java | 2 +- .../ResourceLimitManagerImpl.java | 6 +- .../java/com/cloud/server/StatsCollector.java | 2 +- .../storage/snapshot/SnapshotManager.java | 2 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 2 +- .../smoke/test_deploy_vm_root_resize.py | 9 +++ .../smoke/test_import_unmanage_volumes.py | 22 ++++++- .../smoke/test_over_provisioning.py | 8 ++- test/integration/smoke/test_restore_vm.py | 40 ++++++++++-- .../smoke/test_sharedfs_lifecycle.py | 20 ++++-- test/integration/smoke/test_snapshots.py | 52 +++++++++++++--- test/integration/smoke/test_usage.py | 27 ++++++-- test/integration/smoke/test_vm_autoscaling.py | 36 +++++++++-- test/integration/smoke/test_vm_life_cycle.py | 4 +- .../integration/smoke/test_vm_snapshot_kvm.py | 61 +++++++++++-------- test/integration/smoke/test_vm_snapshots.py | 24 ++++++-- test/integration/smoke/test_volumes.py | 51 ++++++++++++++-- tools/marvin/marvin/lib/utils.py | 53 +++++++++++++++- 22 files changed, 375 insertions(+), 93 deletions(-) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index db0119febde..a6a43388665 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -177,6 +177,7 @@ import com.cloud.vm.dao.SecondaryStorageVmDao; import com.cloud.vm.dao.UserVmCloneSettingDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; +import com.cloud.vm.dao.VMInstanceDao; public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrationService, Configurable { @@ -257,6 +258,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati StoragePoolHostDao storagePoolHostDao; @Inject DiskOfferingDao diskOfferingDao; + @Inject + VMInstanceDao vmInstanceDao; @Inject protected SnapshotHelper snapshotHelper; @@ -933,9 +936,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati // Create event and update resource count for volumes if vm is a user vm if (vm.getType() == VirtualMachine.Type.User) { - Long offeringId = null; - if (!offering.isComputeOnly()) { offeringId = offering.getId(); } @@ -1868,14 +1869,18 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati if (newSize != vol.getSize()) { DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); - if (newSize > vol.getSize()) { - _resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()), - vol.isDisplay(), newSize - vol.getSize(), diskOffering); - _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), - newSize - vol.getSize(), diskOffering); - } else { - _resourceLimitMgr.decrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), - vol.getSize() - newSize, diskOffering); + VMInstanceVO vm = vol.getInstanceId() != null ? vmInstanceDao.findById(vol.getInstanceId()) : null; + if (vm == null || vm.getType() == VirtualMachine.Type.User) { + // Update resource count for user vm volumes when volume is attached + if (newSize > vol.getSize()) { + _resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()), + vol.isDisplay(), newSize - vol.getSize(), diskOffering); + _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), + newSize - vol.getSize(), diskOffering); + } else { + _resourceLimitMgr.decrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), + vol.getSize() - newSize, diskOffering); + } } vol.setSize(newSize); _volsDao.persist(vol); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 2d63a669661..a5aba34a031 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3101,7 +3101,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } if (vmSpec.getOs().toLowerCase().contains("window")) { - isWindowsTemplate =true; + isWindowsTemplate = true; } for (final DiskTO volume : disks) { KVMPhysicalDisk physicalDisk = null; @@ -3220,6 +3220,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv disk.defNetworkBasedDisk(physicalDisk.getPath().replace("rbd:", ""), pool.getSourceHost(), pool.getSourcePort(), pool.getAuthUserName(), pool.getUuid(), devId, diskBusType, DiskProtocol.RBD, DiskDef.DiskFmtType.RAW); } else if (pool.getType() == StoragePoolType.PowerFlex) { + if (isWindowsTemplate && isUefiEnabled) { + diskBusTypeData = DiskDef.DiskBus.SATA; + } disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData); if (physicalDisk.getFormat().equals(PhysicalDiskFormat.QCOW2)) { disk.setDiskFormatType(DiskDef.DiskFmtType.QCOW2); @@ -3250,7 +3253,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv disk.defFileBasedDisk(physicalDisk.getPath(), devId, diskBusType, DiskDef.DiskFmtType.QCOW2); } } - } pool.customizeLibvirtDiskDef(disk); } @@ -4527,6 +4529,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return token[1]; } } else if (token.length > 3) { + // for powerflex/scaleio, path = /dev/disk/by-id/emc-vol-2202eefc4692120f-540fd8fa00000003 + if (token.length > 4 && StringUtils.isNotBlank(token[4]) && token[4].startsWith("emc-vol-")) { + final String[] emcVolToken = token[4].split("-"); + if (emcVolToken.length == 4) { + return emcVolToken[3]; + } + } + // for example, path = /mnt/pool_uuid/disk_path/ return token[3]; } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index 3cd6bd33837..0ef094d3d4e 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -234,7 +234,7 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements @Override public ListResponse searchForVolumeMetricsStats(ListVolumesUsageHistoryCmd cmd) { Pair, Integer> volumeList = searchForVolumesInternal(cmd); - Map> volumeStatsList = searchForVolumeMetricsStatsInternal(cmd, volumeList.first()); + Map> volumeStatsList = searchForVolumeMetricsStatsInternal(cmd, volumeList.first()); return createVolumeMetricsStatsResponse(volumeList, volumeStatsList); } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java index 3d2ca5b1d09..7eb106ef9f8 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java @@ -571,8 +571,8 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { } } } else { - logger.debug("No encryption configured for data volume [id: {}, uuid: {}, name: {}]", - volumeInfo.getId(), volumeInfo.getUuid(), volumeInfo.getName()); + logger.debug("No encryption configured for volume [id: {}, uuid: {}, name: {}]", + volumeInfo.getId(), volumeInfo.getUuid(), volumeInfo.getName()); } return answer; @@ -1512,7 +1512,7 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver { * @return true if resize is required */ private boolean needsExpansionForEncryptionHeader(long srcSize, long dstSize) { - int headerSize = 32<<20; // ensure we have 32MiB for encryption header + int headerSize = 32 << 20; // ensure we have 32MiB for encryption header return srcSize + headerSize > dstSize; } diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java index 5f098badaa1..8ec64802ee2 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java @@ -61,7 +61,7 @@ public class ScaleIOSDCManagerImpl implements ScaleIOSDCManager, Configurable { static ConfigKey ConnectOnDemand = new ConfigKey<>("Storage", Boolean.class, "powerflex.connect.on.demand", - Boolean.FALSE.toString(), + Boolean.TRUE.toString(), "Connect PowerFlex client on Host when first Volume is mapped to SDC and disconnect when last Volume is unmapped from SDC," + " otherwise no action (that is connection remains in the same state whichever it is, connected or disconnected).", Boolean.TRUE, diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 85cca63546c..b890b72f758 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -511,7 +511,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim String convCurrentResourceReservation = String.valueOf(currentResourceReservation); String convNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage) { convDomainResourceLimit = toHumanReadableSize(domainResourceLimit); convCurrentDomainResourceCount = toHumanReadableSize(currentDomainResourceCount); convCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -554,7 +554,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim String convertedCurrentResourceReservation = String.valueOf(currentResourceReservation); String convertedNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage) { convertedAccountResourceLimit = toHumanReadableSize(accountResourceLimit); convertedCurrentResourceCount = toHumanReadableSize(currentResourceCount); convertedCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -1137,7 +1137,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } if (logger.isDebugEnabled()) { String convertedDelta = String.valueOf(delta); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage) { convertedDelta = toHumanReadableSize(delta); } String typeStr = StringUtils.isNotEmpty(tag) ? String.format("%s (tag: %s)", type, tag) : type.getName(); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 27ac0bb725d..a32dac398a8 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1459,7 +1459,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc for (VmDiskStats vmDiskStat : vmDiskStats) { VmDiskStatsEntry vmDiskStatEntry = (VmDiskStatsEntry)vmDiskStat; SearchCriteria sc_volume = _volsDao.createSearchCriteria(); - sc_volume.addAnd("path", SearchCriteria.Op.EQ, vmDiskStatEntry.getPath()); + sc_volume.addAnd("path", SearchCriteria.Op.LIKE, vmDiskStatEntry.getPath() + "%"); List volumes = _volsDao.search(sc_volume, null); if (CollectionUtils.isEmpty(volumes)) diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java index cce580d4106..329ed9bc710 100644 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManager.java @@ -57,7 +57,7 @@ public interface SnapshotManager extends Configurable { public static final ConfigKey BackupRetryInterval = new ConfigKey(Integer.class, "backup.retry.interval", "Advanced", "300", "Time in seconds between retries in backing up snapshot to secondary", false, ConfigKey.Scope.Global, null); - public static final ConfigKey VmStorageSnapshotKvm = new ConfigKey<>(Boolean.class, "kvm.vmstoragesnapshot.enabled", "Snapshots", "false", "For live snapshot of virtual machine instance on KVM hypervisor without memory. Requieres qemu version 1.6+ (on NFS or Local file system) and qemu-guest-agent installed on guest VM", true, ConfigKey.Scope.Global, null); + public static final ConfigKey VmStorageSnapshotKvm = new ConfigKey<>(Boolean.class, "kvm.vmstoragesnapshot.enabled", "Snapshots", "false", "For live snapshot of virtual machine instance on KVM hypervisor without memory. Requires qemu version 1.6+ (on NFS or Local file system) and qemu-guest-agent installed on guest VM", true, ConfigKey.Scope.Global, null); void deletePoliciesForVolume(Long volumeId); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 81b652169bf..b8710943164 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5860,7 +5860,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir for (VmDiskStatsEntry vmDiskStat : vmDiskStats) { SearchCriteria sc_volume = _volsDao.createSearchCriteria(); - sc_volume.addAnd("path", SearchCriteria.Op.EQ, vmDiskStat.getPath()); + sc_volume.addAnd("path", SearchCriteria.Op.LIKE, vmDiskStat.getPath() + "%"); List volumes = _volsDao.search(sc_volume, null); if ((volumes == null) || (volumes.size() == 0)) { break; diff --git a/test/integration/smoke/test_deploy_vm_root_resize.py b/test/integration/smoke/test_deploy_vm_root_resize.py index 1ef5d7d6ea6..b9d14e5bdca 100644 --- a/test/integration/smoke/test_deploy_vm_root_resize.py +++ b/test/integration/smoke/test_deploy_vm_root_resize.py @@ -32,6 +32,7 @@ from marvin.codes import FAILED, INVALID_INPUT, PASS,\ RESOURCE_PRIMARY_STORAGE from nose.plugins.attrib import attr from marvin.sshClient import SshClient +import math import time import re from marvin.cloudstackAPI import updateTemplate,registerTemplate @@ -276,6 +277,14 @@ class TestDeployVmRootSize(cloudstackTestCase): self.assertNotEqual(res[2], INVALID_INPUT, "Invalid list VM " "response") rootvolume = list_volume_response[0] + list_volume_pool_response = list_storage_pools( + self.apiclient, + id=rootvolume.storageid + ) + rootvolume_pool = list_volume_pool_response[0] + if rootvolume_pool.type.lower() == "powerflex": + newrootsize = (int(math.ceil(newrootsize / 8) * 8)) + success = False if rootvolume is not None and rootvolume.size == (newrootsize << 30): success = True diff --git a/test/integration/smoke/test_import_unmanage_volumes.py b/test/integration/smoke/test_import_unmanage_volumes.py index 9001e97a79e..fc1c558d70f 100644 --- a/test/integration/smoke/test_import_unmanage_volumes.py +++ b/test/integration/smoke/test_import_unmanage_volumes.py @@ -26,7 +26,11 @@ from marvin.lib.base import (Account, ServiceOffering, DiskOffering, VirtualMachine) -from marvin.lib.common import (get_domain, get_zone, get_suitable_test_template) +from marvin.lib.common import (get_domain, + get_zone, + get_suitable_test_template, + list_volumes, + list_storage_pools) # Import System modules from nose.plugins.attrib import attr @@ -107,6 +111,22 @@ class TestImportAndUnmanageVolumes(cloudstackTestCase): def test_01_detach_unmanage_import_volume(self): """Test attach/detach/unmanage/import volume """ + + volumes = list_volumes( + self.apiclient, + virtualmachineid=self.virtual_machine.id, + type='ROOT', + listall=True + ) + volume = volumes[0] + volume_pool_response = list_storage_pools( + self.apiclient, + id=volume.storageid + ) + volume_pool = volume_pool_response[0] + if volume_pool.type.lower() == "powerflex": + self.skipTest("This test is not supported for storage pool type %s on hypervisor KVM" % volume_pool.type) + # Create DATA volume volume = Volume.create( self.apiclient, diff --git a/test/integration/smoke/test_over_provisioning.py b/test/integration/smoke/test_over_provisioning.py index 94e4096b1ef..c2b1a5ac205 100644 --- a/test/integration/smoke/test_over_provisioning.py +++ b/test/integration/smoke/test_over_provisioning.py @@ -60,9 +60,10 @@ class TestUpdateOverProvision(cloudstackTestCase): "The environment don't have storage pools required for test") for pool in storage_pools: - if pool.type == "NetworkFilesystem" or pool.type == "VMFS": + if pool.type == "NetworkFilesystem" or pool.type == "VMFS" or pool.type == "PowerFlex": break - if pool.type != "NetworkFilesystem" and pool.type != "VMFS": + + if pool.type != "NetworkFilesystem" and pool.type != "VMFS" and pool.type != "PowerFlex": raise self.skipTest("Storage overprovisioning currently not supported on " + pool.type + " pools") self.poolId = pool.id @@ -101,6 +102,9 @@ class TestUpdateOverProvision(cloudstackTestCase): """Reset the storage.overprovisioning.factor back to its original value @return: """ + if not hasattr(self, 'poolId'): + return + storage_pools = StoragePool.list( self.apiClient, id = self.poolId diff --git a/test/integration/smoke/test_restore_vm.py b/test/integration/smoke/test_restore_vm.py index 3798bef852a..b961bee39f2 100644 --- a/test/integration/smoke/test_restore_vm.py +++ b/test/integration/smoke/test_restore_vm.py @@ -16,10 +16,13 @@ # under the License. """ P1 tests for Scaling up Vm """ + +import math + # Import Local Modules from marvin.cloudstackTestCase import cloudstackTestCase from marvin.lib.base import (VirtualMachine, Volume, DiskOffering, ServiceOffering, Template) -from marvin.lib.common import (get_zone, get_domain) +from marvin.lib.common import (get_zone, get_domain, list_storage_pools) from nose.plugins.attrib import attr _multiprocess_shared_ = True @@ -78,8 +81,13 @@ class TestRestoreVM(cloudstackTestCase): self._cleanup.append(virtual_machine) old_root_vol = Volume.list(self.apiclient, virtualmachineid=virtual_machine.id)[0] + old_root_vol_pool_res = list_storage_pools(self.apiclient, id=old_root_vol.storageid) + old_root_vol_pool = old_root_vol_pool_res[0] + expected_old_root_vol_size = self.template_t1.size + if old_root_vol_pool.type.lower() == "powerflex": + expected_old_root_vol_size = (int(math.ceil((expected_old_root_vol_size / (1024 ** 3)) / 8) * 8)) * (1024 ** 3) self.assertEqual(old_root_vol.state, 'Ready', "Volume should be in Ready state") - self.assertEqual(old_root_vol.size, self.template_t1.size, "Size of volume and template should match") + self.assertEqual(old_root_vol.size, expected_old_root_vol_size, "Size of volume and template should match") virtual_machine.restore(self.apiclient, self.template_t2.id, expunge=True) @@ -88,8 +96,13 @@ class TestRestoreVM(cloudstackTestCase): self.assertEqual(restored_vm.templateid, self.template_t2.id, "VM's template after restore is incorrect") root_vol = Volume.list(self.apiclient, virtualmachineid=restored_vm.id)[0] + root_vol_pool_res = list_storage_pools(self.apiclient, id=root_vol.storageid) + root_vol_pool = root_vol_pool_res[0] + expected_root_vol_size = self.template_t2.size + if root_vol_pool.type.lower() == "powerflex": + expected_root_vol_size = (int(math.ceil((expected_root_vol_size / (1024 ** 3)) / 8) * 8)) * (1024 ** 3) self.assertEqual(root_vol.state, 'Ready', "Volume should be in Ready state") - self.assertEqual(root_vol.size, self.template_t2.size, "Size of volume and template should match") + self.assertEqual(root_vol.size, expected_root_vol_size, "Size of volume and template should match") old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id) self.assertEqual(old_root_vol, None, "Old volume should be deleted") @@ -105,8 +118,13 @@ class TestRestoreVM(cloudstackTestCase): self._cleanup.append(virtual_machine) old_root_vol = Volume.list(self.apiclient, virtualmachineid=virtual_machine.id)[0] + old_root_vol_pool_res = list_storage_pools(self.apiclient, id=old_root_vol.storageid) + old_root_vol_pool = old_root_vol_pool_res[0] + expected_old_root_vol_size = self.template_t1.size + if old_root_vol_pool.type.lower() == "powerflex": + expected_old_root_vol_size = (int(math.ceil((expected_old_root_vol_size / (1024 ** 3)) / 8) * 8)) * (1024 ** 3) self.assertEqual(old_root_vol.state, 'Ready', "Volume should be in Ready state") - self.assertEqual(old_root_vol.size, self.template_t1.size, "Size of volume and template should match") + self.assertEqual(old_root_vol.size, expected_old_root_vol_size, "Size of volume and template should match") virtual_machine.restore(self.apiclient, self.template_t2.id, self.disk_offering.id, expunge=True) @@ -115,9 +133,14 @@ class TestRestoreVM(cloudstackTestCase): self.assertEqual(restored_vm.templateid, self.template_t2.id, "VM's template after restore is incorrect") root_vol = Volume.list(self.apiclient, virtualmachineid=restored_vm.id)[0] + root_vol_pool_res = list_storage_pools(self.apiclient, id=root_vol.storageid) + root_vol_pool = root_vol_pool_res[0] + expected_root_vol_size = self.disk_offering.disksize + if root_vol_pool.type.lower() == "powerflex": + expected_root_vol_size = (int(math.ceil(expected_root_vol_size / 8) * 8)) self.assertEqual(root_vol.diskofferingid, self.disk_offering.id, "Disk offering id should match") self.assertEqual(root_vol.state, 'Ready', "Volume should be in Ready state") - self.assertEqual(root_vol.size, self.disk_offering.disksize * 1024 * 1024 * 1024, + self.assertEqual(root_vol.size, expected_root_vol_size * 1024 * 1024 * 1024, "Size of volume and disk offering should match") old_root_vol = Volume.list(self.apiclient, id=old_root_vol.id) @@ -134,8 +157,13 @@ class TestRestoreVM(cloudstackTestCase): self._cleanup.append(virtual_machine) old_root_vol = Volume.list(self.apiclient, virtualmachineid=virtual_machine.id)[0] + old_root_vol_pool_res = list_storage_pools(self.apiclient, id=old_root_vol.storageid) + old_root_vol_pool = old_root_vol_pool_res[0] + expected_old_root_vol_size = self.template_t1.size + if old_root_vol_pool.type.lower() == "powerflex": + expected_old_root_vol_size = (int(math.ceil((expected_old_root_vol_size / (1024 ** 3)) / 8) * 8)) * (1024 ** 3) self.assertEqual(old_root_vol.state, 'Ready', "Volume should be in Ready state") - self.assertEqual(old_root_vol.size, self.template_t1.size, "Size of volume and template should match") + self.assertEqual(old_root_vol.size, expected_old_root_vol_size, "Size of volume and template should match") virtual_machine.restore(self.apiclient, self.template_t2.id, self.disk_offering.id, rootdisksize=16) diff --git a/test/integration/smoke/test_sharedfs_lifecycle.py b/test/integration/smoke/test_sharedfs_lifecycle.py index f4b2c2fc593..4daf0d7696a 100644 --- a/test/integration/smoke/test_sharedfs_lifecycle.py +++ b/test/integration/smoke/test_sharedfs_lifecycle.py @@ -38,7 +38,8 @@ from marvin.lib.base import (Account, ) from marvin.lib.common import (get_domain, get_zone, - get_template) + get_template, + list_storage_pools) from marvin.codes import FAILED from marvin.lib.decoratorGenerators import skipTestIf @@ -258,15 +259,23 @@ class TestSharedFSLifecycle(cloudstackTestCase): def test_resize_shared_fs(self): """Resize the shared filesystem by changing the disk offering and validate """ + sharedfs_pool_response = list_storage_pools(self.apiclient, id=self.sharedfs.storageid) + sharedfs_pool = sharedfs_pool_response[0] + self.mountSharedFSOnVM(self.vm1_ssh_client, self.sharedfs) result = self.vm1_ssh_client.execute("df -Th /mnt/fs1 | grep nfs")[0] self.debug(result) size = result.split()[-5] self.debug("Size of the filesystem is " + size) - self.assertEqual(size, "2.0G", "SharedFS size should be 2.0G") + if sharedfs_pool.type.lower() == "powerflex": + self.assertEqual(size, "8.0G", "SharedFS size should be 8.0G") + new_size = 9 + else: + self.assertEqual(size, "2.0G", "SharedFS size should be 2.0G") + new_size = 3 response = SharedFS.stop(self.sharedfs, self.apiclient) - response = SharedFS.changediskoffering(self.sharedfs, self.apiclient, self.disk_offering.id, 3) + response = SharedFS.changediskoffering(self.sharedfs, self.apiclient, self.disk_offering.id, new_size) self.debug(response) response = SharedFS.start(self.sharedfs, self.apiclient) time.sleep(10) @@ -274,4 +283,7 @@ class TestSharedFSLifecycle(cloudstackTestCase): result = self.vm1_ssh_client.execute("df -Th /mnt/fs1 | grep nfs")[0] size = result.split()[-5] self.debug("Size of the filesystem is " + size) - self.assertEqual(size, "3.0G", "SharedFS size should be 3.0G") + if sharedfs_pool.type.lower() == "powerflex": + self.assertEqual(size, "16G", "SharedFS size should be 16G") + else: + self.assertEqual(size, "3.0G", "SharedFS size should be 3.0G") diff --git a/test/integration/smoke/test_snapshots.py b/test/integration/smoke/test_snapshots.py index f8346093c64..b1a2569d969 100644 --- a/test/integration/smoke/test_snapshots.py +++ b/test/integration/smoke/test_snapshots.py @@ -18,8 +18,10 @@ from marvin.codes import FAILED from nose.plugins.attrib import attr from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackException import CloudstackAPIException from marvin.lib.utils import (cleanup_resources, is_snapshot_on_nfs, + is_snapshot_on_powerflex, validateList) from marvin.lib.base import (VirtualMachine, Account, @@ -146,10 +148,16 @@ class TestSnapshotRootDisk(cloudstackTestCase): type='ROOT', listall=True ) + volume = volumes[0] + volume_pool_response = list_storage_pools( + self.apiclient, + id=volume.storageid + ) + volume_pool = volume_pool_response[0] snapshot = Snapshot.create( self.apiclient, - volumes[0].id, + volume.id, account=self.account.name, domainid=self.account.domainid ) @@ -209,6 +217,11 @@ class TestSnapshotRootDisk(cloudstackTestCase): "Check if backup_snap_id is not null" ) + if volume_pool.type.lower() == "powerflex": + self.assertTrue(is_snapshot_on_powerflex( + self.apiclient, self.dbclient, self.config, self.zone.id, snapshot.id)) + return + self.assertTrue(is_snapshot_on_nfs( self.apiclient, self.dbclient, self.config, self.zone.id, snapshot.id)) return @@ -246,6 +259,11 @@ class TestSnapshotRootDisk(cloudstackTestCase): PASS, "Invalid response returned for list volumes") vol_uuid = vol_res[0].id + volume_pool_response = list_storage_pools(self.apiclient, + id=vol_res[0].storageid) + volume_pool = volume_pool_response[0] + if volume_pool.type.lower() != 'networkfilesystem': + self.skipTest("This test is not supported for volume created on storage pool type %s" % volume_pool.type) clusters = list_clusters( self.apiclient, zoneid=self.zone.id @@ -437,15 +455,16 @@ class TestSnapshotStandaloneBackup(cloudstackTestCase): ) cls._cleanup.append(cls.virtual_machine) - volumes =Volume.list( + volumes = Volume.list( cls.userapiclient, virtualmachineid=cls.virtual_machine.id, type='ROOT', listall=True ) + cls.volume = volumes[0] cls.snapshot = Snapshot.create( cls.userapiclient, - volumes[0].id, + cls.volume.id, account=cls.account.name, domainid=cls.account.domainid ) @@ -475,13 +494,28 @@ class TestSnapshotStandaloneBackup(cloudstackTestCase): """Test creating volume from snapshot """ self.services['volume_from_snapshot']['zoneid'] = self.zone.id - self.volume_from_snap = Volume.create_from_snapshot( - self.userapiclient, - snapshot_id=self.snapshot.id, - services=self.services["volume_from_snapshot"], - account=self.account.name, - domainid=self.account.domainid + snapshot_volume_pool_response = list_storage_pools( + self.apiclient, + id=self.volume.storageid ) + snapshot_volume_pool = snapshot_volume_pool_response[0] + try: + self.volume_from_snap = Volume.create_from_snapshot( + self.userapiclient, + snapshot_id=self.snapshot.id, + services=self.services["volume_from_snapshot"], + account=self.account.name, + domainid=self.account.domainid + ) + except CloudstackAPIException as cs: + self.debug(cs.errorMsg) + if snapshot_volume_pool.type.lower() == "powerflex": + self.assertTrue( + cs.errorMsg.find("Create volume from snapshot is not supported for PowerFlex volume snapshots") > 0, + msg="Other than unsupported error while creating volume from snapshot for volume on PowerFlex pool") + return + self.fail("Failed to create volume from snapshot: %s" % cs) + self.cleanup.append(self.volume_from_snap) self.assertEqual( diff --git a/test/integration/smoke/test_usage.py b/test/integration/smoke/test_usage.py index 1a6ff37cedb..9ec5205403e 100644 --- a/test/integration/smoke/test_usage.py +++ b/test/integration/smoke/test_usage.py @@ -40,6 +40,7 @@ from marvin.lib.base import (Account, from marvin.lib.common import (get_zone, get_domain, get_suitable_test_template, + list_storage_pools, find_storage_pool_type) @@ -611,17 +612,17 @@ class TestVolumeUsage(cloudstackTestCase): except Exception as e: self.fail("Failed to stop instance: %s" % e) - volume_response = Volume.list( + data_volume_response = Volume.list( self.apiclient, virtualmachineid=self.virtual_machine.id, type='DATADISK', listall=True) self.assertEqual( - isinstance(volume_response, list), + isinstance(data_volume_response, list), True, "Check for valid list volumes response" ) - data_volume = volume_response[0] + data_volume = data_volume_response[0] # Detach data Disk self.debug("Detaching volume ID: %s VM with ID: %s" % ( @@ -769,7 +770,25 @@ class TestVolumeUsage(cloudstackTestCase): "Running", "VM state should be running after deployment" ) - self.virtual_machine.attach_volume(self.apiclient,volume_uploaded) + root_volume_response = Volume.list( + self.apiclient, + virtualmachineid=self.virtual_machine.id, + type='ROOT', + listall=True) + root_volume = root_volume_response[0] + rool_volume_pool_response = list_storage_pools( + self.apiclient, + id=root_volume.storageid + ) + rool_volume_pool = rool_volume_pool_response[0] + try: + self.virtual_machine.attach_volume(self.apiclient,volume_uploaded) + except Exception as e: + self.debug("Exception %s: " % e) + if rool_volume_pool.type.lower() == "powerflex" and "this operation is unsupported on storage pool type PowerFlex" in str(e): + return + self.fail(e) + self.debug("select type from usage_event where offering_id = 6 and volume_id = '%s';" % volume_id) diff --git a/test/integration/smoke/test_vm_autoscaling.py b/test/integration/smoke/test_vm_autoscaling.py index 7ae61ce57da..782d2bce3ad 100644 --- a/test/integration/smoke/test_vm_autoscaling.py +++ b/test/integration/smoke/test_vm_autoscaling.py @@ -22,6 +22,7 @@ Tests of VM Autoscaling import logging import time import datetime +import math from nose.plugins.attrib import attr from marvin.cloudstackTestCase import cloudstackTestCase @@ -53,7 +54,8 @@ from marvin.lib.base import (Account, from marvin.lib.common import (get_domain, get_zone, - get_template) + get_template, + list_storage_pools) from marvin.lib.utils import wait_until MIN_MEMBER = 1 @@ -466,8 +468,10 @@ class TestVmAutoScaling(cloudstackTestCase): def verifyVmProfile(self, vm, autoscalevmprofileid, networkid=None, projectid=None): self.message("Verifying profiles of new VM %s (%s)" % (vm.name, vm.id)) datadisksizeInBytes = None + datadiskpoolid = None diskofferingid = None rootdisksizeInBytes = None + rootdiskpoolid = None sshkeypairs = None affinitygroupIdsArray = [] @@ -496,10 +500,24 @@ class TestVmAutoScaling(cloudstackTestCase): for volume in volumes: if volume.type == 'ROOT': rootdisksizeInBytes = volume.size + rootdiskpoolid = volume.storageid elif volume.type == 'DATADISK': datadisksizeInBytes = volume.size + datadiskpoolid = volume.storageid diskofferingid = volume.diskofferingid + rootdisk_pool_response = list_storage_pools( + self.apiclient, + id=rootdiskpoolid + ) + rootdisk_pool = rootdisk_pool_response[0] + + datadisk_pool_response = list_storage_pools( + self.apiclient, + id=datadiskpoolid + ) + datadisk_pool = datadisk_pool_response[0] + vmprofiles_list = AutoScaleVmProfile.list( self.regular_user_apiclient, listall=True, @@ -522,18 +540,26 @@ class TestVmAutoScaling(cloudstackTestCase): self.assertEquals(templateid, vmprofile.templateid) self.assertEquals(serviceofferingid, vmprofile.serviceofferingid) + rootdisksize = None if vmprofile_otherdeployparams.rootdisksize: - self.assertEquals(int(rootdisksizeInBytes), int(vmprofile_otherdeployparams.rootdisksize) * (1024 ** 3)) + rootdisksize = int(vmprofile_otherdeployparams.rootdisksize) elif vmprofile_otherdeployparams.overridediskofferingid: self.assertEquals(vmprofile_otherdeployparams.overridediskofferingid, self.disk_offering_override.id) - self.assertEquals(int(rootdisksizeInBytes), int(self.disk_offering_override.disksize) * (1024 ** 3)) + rootdisksize = int(self.disk_offering_override.disksize) else: - self.assertEquals(int(rootdisksizeInBytes), int(self.templatesize) * (1024 ** 3)) + rootdisksize = int(self.templatesize) + + if rootdisk_pool.type.lower() == "powerflex": + rootdisksize = (int(math.ceil(rootdisksize / 8) * 8)) + self.assertEquals(int(rootdisksizeInBytes), rootdisksize * (1024 ** 3)) if vmprofile_otherdeployparams.diskofferingid: self.assertEquals(diskofferingid, vmprofile_otherdeployparams.diskofferingid) if vmprofile_otherdeployparams.disksize: - self.assertEquals(int(datadisksizeInBytes), int(vmprofile_otherdeployparams.disksize) * (1024 ** 3)) + datadisksize = int(vmprofile_otherdeployparams.disksize) + if datadisk_pool.type.lower() == "powerflex": + datadisksize = (int(math.ceil(datadisksize / 8) * 8)) + self.assertEquals(int(datadisksizeInBytes), datadisksize * (1024 ** 3)) if vmprofile_otherdeployparams.keypairs: self.assertEquals(sshkeypairs, vmprofile_otherdeployparams.keypairs) diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index c7c9a01bd32..8df0b994a55 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -1710,8 +1710,8 @@ class TestKVMLiveMigration(cloudstackTestCase): def get_target_pool(self, volid): target_pools = StoragePool.listForMigration(self.apiclient, id=volid) - if len(target_pools) < 1: - self.skipTest("Not enough storage pools found") + if target_pools is None or len(target_pools) == 0: + self.skipTest("Not enough storage pools found for migration") return target_pools[0] diff --git a/test/integration/smoke/test_vm_snapshot_kvm.py b/test/integration/smoke/test_vm_snapshot_kvm.py index 5c133f6e762..9dd7c529de5 100644 --- a/test/integration/smoke/test_vm_snapshot_kvm.py +++ b/test/integration/smoke/test_vm_snapshot_kvm.py @@ -77,6 +77,18 @@ class TestVmSnapshot(cloudstackTestCase): Configurations.update(cls.apiclient, name = "kvm.vmstoragesnapshot.enabled", value = "true") + + cls.services["domainid"] = cls.domain.id + cls.services["small"]["zoneid"] = cls.zone.id + cls.services["zoneid"] = cls.zone.id + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) + #The version of CentOS has to be supported templ = { "name": "CentOS8", @@ -91,36 +103,33 @@ class TestVmSnapshot(cloudstackTestCase): "directdownload": True, } - template = Template.register(cls.apiclient, templ, zoneid=cls.zone.id, hypervisor=cls.hypervisor) + template = Template.register( + cls.apiclient, + templ, + zoneid=cls.zone.id, + account=cls.account.name, + domainid=cls.account.domainid, + hypervisor=cls.hypervisor + ) if template == FAILED: assert False, "get_template() failed to return template\ with description %s" % cls.services["ostype"] - cls.services["domainid"] = cls.domain.id - cls.services["small"]["zoneid"] = cls.zone.id cls.services["templates"]["ostypeid"] = template.ostypeid - cls.services["zoneid"] = cls.zone.id - cls.account = Account.create( - cls.apiclient, - cls.services["account"], - domainid=cls.domain.id - ) - cls._cleanup.append(cls.account) - - service_offerings_nfs = { + service_offering_nfs = { "name": "nfs", - "displaytext": "nfs", - "cpunumber": 1, - "cpuspeed": 500, - "memory": 512, - "storagetype": "shared", - "customizediops": False, - } + "displaytext": "nfs", + "cpunumber": 1, + "cpuspeed": 500, + "memory": 512, + "storagetype": "shared", + "customizediops": False, + } cls.service_offering = ServiceOffering.create( cls.apiclient, - service_offerings_nfs, + service_offering_nfs, ) cls._cleanup.append(cls.service_offering) @@ -138,7 +147,7 @@ class TestVmSnapshot(cloudstackTestCase): rootdisksize=20, ) cls.random_data_0 = random_gen(size=100) - cls.test_dir = "/tmp" + cls.test_dir = "$HOME" cls.random_data = "random.data" return @@ -201,8 +210,8 @@ class TestVmSnapshot(cloudstackTestCase): self.apiclient, self.virtual_machine.id, MemorySnapshot, - "TestSnapshot", - "Display Text" + "TestVmSnapshot", + "Test VM Snapshot" ) self.assertEqual( vm_snapshot.state, @@ -269,6 +278,8 @@ class TestVmSnapshot(cloudstackTestCase): self.virtual_machine.start(self.apiclient) + time.sleep(30) + try: ssh_client = self.virtual_machine.get_ssh_client(reconnect=True) @@ -288,7 +299,7 @@ class TestVmSnapshot(cloudstackTestCase): self.assertEqual( self.random_data_0, result[0], - "Check the random data is equal with the ramdom file!" + "Check the random data is equal with the random file!" ) @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") @@ -320,7 +331,7 @@ class TestVmSnapshot(cloudstackTestCase): list_snapshot_response = VmSnapshot.list( self.apiclient, virtualmachineid=self.virtual_machine.id, - listall=False) + listall=True) self.debug('list_snapshot_response -------------------- %s' % list_snapshot_response) self.assertIsNone(list_snapshot_response, "snapshot is already deleted") diff --git a/test/integration/smoke/test_vm_snapshots.py b/test/integration/smoke/test_vm_snapshots.py index 07779e78c58..8c106f05a9f 100644 --- a/test/integration/smoke/test_vm_snapshots.py +++ b/test/integration/smoke/test_vm_snapshots.py @@ -27,7 +27,9 @@ from marvin.lib.base import (Account, from marvin.lib.common import (get_zone, get_domain, get_suitable_test_template, + list_volumes, list_snapshots, + list_storage_pools, list_virtual_machines) import time @@ -87,6 +89,18 @@ class TestVmSnapshot(cloudstackTestCase): serviceofferingid=cls.service_offering.id, mode=cls.zone.networktype ) + volumes = list_volumes( + cls.apiclient, + virtualmachineid=cls.virtual_machine.id, + type='ROOT', + listall=True + ) + volume = volumes[0] + volume_pool_response = list_storage_pools( + cls.apiclient, + id=volume.storageid + ) + cls.volume_pool = volume_pool_response[0] cls.random_data_0 = random_gen(size=100) cls.test_dir = "$HOME" cls.random_data = "random.data" @@ -146,15 +160,15 @@ class TestVmSnapshot(cloudstackTestCase): #KVM VM Snapshot needs to set snapshot with memory MemorySnapshot = False - if self.hypervisor.lower() in (KVM.lower()): + if self.hypervisor.lower() in (KVM.lower()) and self.volume_pool.type.lower() != "powerflex": MemorySnapshot = True vm_snapshot = VmSnapshot.create( self.apiclient, self.virtual_machine.id, MemorySnapshot, - "TestSnapshot", - "Display Text" + "TestVmSnapshot", + "Test VM Snapshot" ) self.assertEqual( vm_snapshot.state, @@ -214,7 +228,7 @@ class TestVmSnapshot(cloudstackTestCase): ) #We don't need to stop the VM when taking a VM Snapshot on KVM - if self.hypervisor.lower() in (KVM.lower()): + if self.hypervisor.lower() in (KVM.lower()) and self.volume_pool.type.lower() != "powerflex": pass else: self.virtual_machine.stop(self.apiclient) @@ -224,7 +238,7 @@ class TestVmSnapshot(cloudstackTestCase): list_snapshot_response[0].id) #We don't need to start the VM when taking a VM Snapshot on KVM - if self.hypervisor.lower() in (KVM.lower()): + if self.hypervisor.lower() in (KVM.lower()) and self.volume_pool.type.lower() != "powerflex": pass else: self.virtual_machine.start(self.apiclient) diff --git a/test/integration/smoke/test_volumes.py b/test/integration/smoke/test_volumes.py index 28a029adf70..6cf3f082bc2 100644 --- a/test/integration/smoke/test_volumes.py +++ b/test/integration/smoke/test_volumes.py @@ -19,6 +19,7 @@ import os import tempfile import time +import math import unittest import urllib.error import urllib.parse @@ -42,6 +43,7 @@ from marvin.lib.common import (get_domain, get_zone, find_storage_pool_type, get_pod, + list_storage_pools, list_disk_offering) from marvin.lib.utils import (cleanup_resources, checkVolumeSize) from marvin.lib.utils import (format_volume_to_ext3, @@ -235,7 +237,6 @@ class TestCreateVolume(cloudstackTestCase): "Failed to start VM (ID: %s) " % vm.id) timeout = timeout - 1 - vol_sz = str(list_volume_response[0].size) ssh = self.virtual_machine.get_ssh_client( reconnect=True ) @@ -243,6 +244,7 @@ class TestCreateVolume(cloudstackTestCase): list_volume_response = Volume.list( self.apiClient, id=volume.id) + vol_sz = str(list_volume_response[0].size) if list_volume_response[0].hypervisor.lower() == XEN_SERVER.lower(): volume_name = "/dev/xvd" + chr(ord('a') + int(list_volume_response[0].deviceid)) self.debug(" Using XenServer volume_name: %s" % (volume_name)) @@ -533,6 +535,17 @@ class TestVolumes(cloudstackTestCase): # Sleep to ensure the current state will reflected in other calls time.sleep(self.services["sleep"]) + list_volume_response = Volume.list( + self.apiClient, + id=self.volume.id + ) + volume = list_volume_response[0] + + list_volume_pool_response = list_storage_pools(self.apiClient, id=volume.storageid) + volume_pool = list_volume_pool_response[0] + if volume_pool.type.lower() == "powerflex": + self.skipTest("Extract volume operation is unsupported for volumes on storage pool type %s" % volume_pool.type) + cmd = extractVolume.extractVolumeCmd() cmd.id = self.volume.id cmd.mode = "HTTP_DOWNLOAD" @@ -658,7 +671,15 @@ class TestVolumes(cloudstackTestCase): type='DATADISK' ) for vol in list_volume_response: - if vol.id == self.volume.id and int(vol.size) == (int(disk_offering_20_GB.disksize) * (1024 ** 3)) and vol.state == 'Ready': + list_volume_pool_response = list_storage_pools( + self.apiClient, + id=vol.storageid + ) + volume_pool = list_volume_pool_response[0] + disksize = (int(disk_offering_20_GB.disksize)) + if volume_pool.type.lower() == "powerflex": + disksize = (int(math.ceil(disksize / 8) * 8)) + if vol.id == self.volume.id and int(vol.size) == disksize * (1024 ** 3) and vol.state == 'Ready': success = True if success: break @@ -925,7 +946,15 @@ class TestVolumes(cloudstackTestCase): type='DATADISK' ) for vol in list_volume_response: - if vol.id == self.volume.id and int(vol.size) == (20 * (1024 ** 3)) and vol.state == 'Ready': + list_volume_pool_response = list_storage_pools( + self.apiClient, + id=vol.storageid + ) + volume_pool = list_volume_pool_response[0] + disksize = 20 + if volume_pool.type.lower() == "powerflex": + disksize = (int(math.ceil(disksize / 8) * 8)) + if vol.id == self.volume.id and int(vol.size) == disksize * (1024 ** 3) and vol.state == 'Ready': success = True if success: break @@ -1283,7 +1312,6 @@ class TestVolumeEncryption(cloudstackTestCase): "Failed to start VM (ID: %s) " % vm.id) timeout = timeout - 1 - vol_sz = str(list_volume_response[0].size) ssh = virtual_machine.get_ssh_client( reconnect=True ) @@ -1292,6 +1320,7 @@ class TestVolumeEncryption(cloudstackTestCase): list_volume_response = Volume.list( self.apiclient, id=volume.id) + vol_sz = str(list_volume_response[0].size) volume_name = "/dev/vd" + chr(ord('a') + int(list_volume_response[0].deviceid)) self.debug(" Using KVM volume_name: %s" % (volume_name)) @@ -1410,7 +1439,6 @@ class TestVolumeEncryption(cloudstackTestCase): "Failed to start VM (ID: %s) " % vm.id) timeout = timeout - 1 - vol_sz = str(list_volume_response[0].size) ssh = virtual_machine.get_ssh_client( reconnect=True ) @@ -1419,6 +1447,12 @@ class TestVolumeEncryption(cloudstackTestCase): list_volume_response = Volume.list( self.apiclient, id=volume.id) + vol_sz = str(list_volume_response[0].size) + list_volume_pool_response = list_storage_pools(self.apiclient, id=list_volume_response[0].storageid) + volume_pool = list_volume_pool_response[0] + if volume_pool.type.lower() == "powerflex": + vol_sz = int(vol_sz) + vol_sz = str(vol_sz - (128 << 20) - ((vol_sz >> 30) * 200704)) volume_name = "/dev/vd" + chr(ord('a') + int(list_volume_response[0].deviceid)) self.debug(" Using KVM volume_name: %s" % (volume_name)) @@ -1543,7 +1577,6 @@ class TestVolumeEncryption(cloudstackTestCase): "Failed to start VM (ID: %s) " % vm.id) timeout = timeout - 1 - vol_sz = str(list_volume_response[0].size) ssh = virtual_machine.get_ssh_client( reconnect=True ) @@ -1552,6 +1585,12 @@ class TestVolumeEncryption(cloudstackTestCase): list_volume_response = Volume.list( self.apiclient, id=volume.id) + vol_sz = str(list_volume_response[0].size) + list_volume_pool_response = list_storage_pools(self.apiclient, id=list_volume_response[0].storageid) + volume_pool = list_volume_pool_response[0] + if volume_pool.type.lower() == "powerflex": + vol_sz = int(vol_sz) + vol_sz = str(vol_sz - (128 << 20) - ((vol_sz >> 30) * 200704)) volume_name = "/dev/vd" + chr(ord('a') + int(list_volume_response[0].deviceid)) self.debug(" Using KVM volume_name: %s" % (volume_name)) diff --git a/tools/marvin/marvin/lib/utils.py b/tools/marvin/marvin/lib/utils.py index f80eccf1159..c822a587dfc 100644 --- a/tools/marvin/marvin/lib/utils.py +++ b/tools/marvin/marvin/lib/utils.py @@ -300,12 +300,63 @@ def get_hypervisor_version(apiclient): assert hosts_list_validation_result[0] == PASS, "host list validation failed" return hosts_list_validation_result[1].hypervisorversion +def is_snapshot_on_powerflex(apiclient, dbconn, config, zoneid, snapshotid): + """ + Checks whether a snapshot with id (not UUID) `snapshotid` is present on the powerflex storage + + @param apiclient: api client connection + @param dbconn: connection to the cloudstack db + @param config: marvin configuration file + @param zoneid: uuid of the zone on which the secondary nfs storage pool is mounted + @param snapshotid: uuid of the snapshot + @return: True if snapshot is found, False otherwise + """ + + qresultset = dbconn.execute( + "SELECT id FROM snapshots WHERE uuid = '%s';" \ + % str(snapshotid) + ) + if len(qresultset) == 0: + raise Exception( + "No snapshot found in cloudstack with id %s" % snapshotid) + + + snapshotid = qresultset[0][0] + qresultset = dbconn.execute( + "SELECT install_path, store_id FROM snapshot_store_ref WHERE snapshot_id='%s' AND store_role='Primary';" % snapshotid + ) + + assert isinstance(qresultset, list), "Invalid db query response for snapshot %s" % snapshotid + + if len(qresultset) == 0: + #Snapshot does not exist + return False + + from .base import StoragePool + #pass store_id to get the exact storage pool where snapshot is stored + primaryStores = StoragePool.list(apiclient, zoneid=zoneid, id=int(qresultset[0][1])) + + assert isinstance(primaryStores, list), "Not a valid response for listStoragePools" + assert len(primaryStores) != 0, "No storage pools found in zone %s" % zoneid + + primaryStore = primaryStores[0] + + if str(primaryStore.provider).lower() != "powerflex": + raise Exception( + "is_snapshot_on_powerflex works only against powerflex storage pool. found %s" % str(primaryStore.provider)) + + snapshotPath = str(qresultset[0][0]) + if not snapshotPath: + return False + + return True + def is_snapshot_on_nfs(apiclient, dbconn, config, zoneid, snapshotid): """ Checks whether a snapshot with id (not UUID) `snapshotid` is present on the nfs storage @param apiclient: api client connection - @param @dbconn: connection to the cloudstack db + @param dbconn: connection to the cloudstack db @param config: marvin configuration file @param zoneid: uuid of the zone on which the secondary nfs storage pool is mounted @param snapshotid: uuid of the snapshot From 2cd7d8a315e2917d90cf6994977937d3752a6785 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 15 Sep 2025 12:41:26 +0530 Subject: [PATCH 004/217] server: check limit on correct store during snapshot allocation (#11558) Fixes #11551 Signed-off-by: Abhishek Kumar --- .../com/cloud/storage/snapshot/SnapshotManagerImpl.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index a3fb8eda2fe..b0070890de2 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -1696,9 +1696,14 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement Type snapshotType = getSnapshotType(policyId); Account owner = _accountMgr.getAccount(volume.getAccountId()); + ResourceType storeResourceType = ResourceType.secondary_storage; + if (!isBackupSnapshotToSecondaryForZone(volume.getDataCenterId()) || + Snapshot.LocationType.PRIMARY.equals(locationType)) { + storeResourceType = ResourceType.primary_storage; + } try { _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.secondary_storage, new Long(volume.getSize()).longValue()); + _resourceLimitMgr.checkResourceLimit(owner, storeResourceType, volume.getSize()); } catch (ResourceAllocationException e) { if (snapshotType != Type.MANUAL) { String msg = String.format("Snapshot resource limit exceeded for account %s. Failed to create recurring snapshots", owner); @@ -1749,7 +1754,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement } CallContext.current().putContextParameter(Snapshot.class, snapshot.getUuid()); _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.snapshot); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), storeResourceType, volume.getSize()); return snapshot; } From 7c76a3c12a84db92be46712c69b10cd9aacc3055 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 15 Sep 2025 13:31:02 +0530 Subject: [PATCH 005/217] ui: searchview change should only remove related query params (#11576) --- ui/src/views/AutogenView.vue | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index 6b57e0f6ea0..b1f26a17f8f 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -1821,9 +1821,8 @@ export default { }, onSearch (opts) { const query = Object.assign({}, this.$route.query) - for (const key in this.searchParams) { - delete query[key] - } + const searchFilters = this.$route?.meta?.searchFilters || [] + searchFilters.forEach(key => delete query[key]) delete query.name delete query.templatetype delete query.keyword From cd69f2ce16d4524e2068c3be508dbf5352221316 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 15 Sep 2025 13:44:06 +0530 Subject: [PATCH 006/217] server: Fix NPE during VM IP fetch for shared networks (#11389) * Fix NPE during VM IP fetch for shared networks * PR 11389: add missing import org.apache.commons.lang3.ObjectUtils --------- Co-authored-by: Wei Zhou --- .../main/java/com/cloud/vm/UserVmManagerImpl.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index b8710943164..794d28c7adc 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -139,6 +139,7 @@ import org.apache.cloudstack.vm.UnmanagedVMsManager; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -658,7 +659,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir "Wait Interval (in seconds) for shared network vm dhcp ip addr fetch for next iteration ", true); private static final ConfigKey VmIpFetchTrialMax = new ConfigKey("Advanced", Integer.class, "externaldhcp.vmip.max.retry", "10", - "The max number of retrieval times for shared entwork vm dhcp ip fetch, in case of failures", true); + "The max number of retrieval times for shared network vm dhcp ip fetch, in case of failures", true); private static final ConfigKey VmIpFetchThreadPoolMax = new ConfigKey("Advanced", Integer.class, "externaldhcp.vmipFetch.threadPool.max", "10", "number of threads for fetching vms ip address", true); @@ -2674,7 +2675,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (vmIdAndCount.getRetrievalCount() <= 0) { vmIdCountMap.remove(nicId); - logger.debug("Vm " + vmId +" nic "+nicId + " count is zero .. removing vm nic from map "); + logger.debug("Vm {} nic {} count is zero .. removing vm nic from map ", vmId, nicId); ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, Domain.ROOT_DOMAIN, EventTypes.EVENT_NETWORK_EXTERNAL_DHCP_VM_IPFETCH, @@ -2683,12 +2684,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir continue; } - UserVm userVm = _vmDao.findById(vmId); VMInstanceVO vmInstance = _vmInstanceDao.findById(vmId); NicVO nicVo = _nicDao.findById(nicId); - NetworkVO network = _networkDao.findById(nicVo.getNetworkId()); + if (ObjectUtils.anyNull(userVm, vmInstance, nicVo)) { + logger.warn("Couldn't fetch ip addr, Vm {} or nic {} doesn't exists", vmId, nicId); + continue; + } + NetworkVO network = _networkDao.findById(nicVo.getNetworkId()); VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(userVm); VirtualMachine vm = vmProfile.getVirtualMachine(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); From 6a145358a92c16a7a371349a763375d63b66fc21 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 15 Sep 2025 15:02:10 +0530 Subject: [PATCH 007/217] ui: fix tab name in query params (#11590) --- ui/src/components/view/ResourceView.vue | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ui/src/components/view/ResourceView.vue b/ui/src/components/view/ResourceView.vue index 2c1764da143..af27a3d1b95 100644 --- a/ui/src/components/view/ResourceView.vue +++ b/ui/src/components/view/ResourceView.vue @@ -33,17 +33,17 @@ :is="tabs[0].component" :resource="resource" :loading="loading" - :tab="tabs[0].name" /> + :tab="tabName(tabs[0])" /> - + + + @@ -901,7 +920,7 @@ From a749206eb8e992f8cbc0d82495c7bbc713ff470e Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 24 Sep 2025 11:52:49 +0530 Subject: [PATCH 033/217] storage: Mount disabled pools by default when host is booted (#11666) --- .../src/main/java/com/cloud/storage/StorageManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 7b31ec6a81b..529e506e8a0 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -182,7 +182,7 @@ public interface StorageManager extends StorageService { ConfigKey MountDisabledStoragePool = new ConfigKey<>(Boolean.class, "mount.disabled.storage.pool", "Storage", - "false", + Boolean.TRUE.toString(), "Mount all zone-wide or cluster-wide disabled storage pools after node reboot", true, ConfigKey.Scope.Cluster, From a18b5514e67a248f57f6e212822a4a44e0b6195c Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Wed, 24 Sep 2025 12:04:18 +0530 Subject: [PATCH 034/217] kvm: honor templateId passed in importVM API (#11640) --- .../apache/cloudstack/vm/UnmanagedVMsManagerImpl.java | 11 ++--------- .../cloudstack/vm/UnmanagedVMsManagerImplTest.java | 4 ---- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 981f83936d2..30cf4ad76a7 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -1493,7 +1493,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { if (templateId == null) { template = templateDao.findByName(VM_IMPORT_DEFAULT_TEMPLATE_NAME); if (template == null) { - template = createDefaultDummyVmImportTemplate(false); + template = createDefaultDummyVmImportTemplate(Hypervisor.HypervisorType.KVM == hypervisorType); if (template == null) { throw new InvalidParameterValueException(String.format("Default VM import template with unique name: %s for hypervisor: %s cannot be created. Please use templateid parameter for import", VM_IMPORT_DEFAULT_TEMPLATE_NAME, hypervisorType.toString())); } @@ -2331,14 +2331,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { if (CollectionUtils.isNotEmpty(userVOs)) { userId = userVOs.get(0).getId(); } - VMTemplateVO template = templateDao.findByName(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME); - if (template == null) { - template = createDefaultDummyVmImportTemplate(true); - if (template == null) { - throw new InvalidParameterValueException("Error while creating default Import Vm Template"); - } - } - + VMTemplateVO template = getTemplateForImportInstance(cmd.getTemplateId(), Hypervisor.HypervisorType.KVM); final Long serviceOfferingId = cmd.getServiceOfferingId(); if (serviceOfferingId == null) { throw new InvalidParameterValueException(String.format("Service offering ID cannot be null")); diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index 4aa5df9d411..09f62f7a049 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -504,8 +504,6 @@ public class UnmanagedVMsManagerImplTest { when(cmd.getPassword()).thenReturn("pass"); when(cmd.getImportSource()).thenReturn("external"); when(cmd.getDomainId()).thenReturn(null); - VMTemplateVO template = Mockito.mock(VMTemplateVO.class); - when(templateDao.findByName(anyString())).thenReturn(template); HostVO host = Mockito.mock(HostVO.class); DeployDestination mockDest = Mockito.mock(DeployDestination.class); when(deploymentPlanningManager.planDeployment(any(), any(), any(), any())).thenReturn(mockDest); @@ -736,8 +734,6 @@ public class UnmanagedVMsManagerImplTest { when(cmd.getImportSource()).thenReturn(source); when(cmd.getDiskPath()).thenReturn("/var/lib/libvirt/images/test.qcow2"); when(cmd.getDomainId()).thenReturn(null); - VMTemplateVO template = Mockito.mock(VMTemplateVO.class); - when(templateDao.findByName(anyString())).thenReturn(template); HostVO host = Mockito.mock(HostVO.class); when(hostDao.findById(anyLong())).thenReturn(host); NetworkOffering netOffering = Mockito.mock(NetworkOffering.class); From 98b9af29040533089a7f8153c74f95f9c324d66b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 24 Sep 2025 12:51:40 +0530 Subject: [PATCH 035/217] server: set VirtualMachineTO arch from template if present (#11530) * server: set VirtualMachineTO arch from template if present Fixes #11529 Signed-off-by: Abhishek Kumar * refactor Signed-off-by: Abhishek Kumar --------- Signed-off-by: Abhishek Kumar --- .../com/cloud/hypervisor/HypervisorGuruBase.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java index c510502f5f9..4f14fe20dac 100644 --- a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java @@ -23,6 +23,7 @@ import java.util.UUID; import javax.inject.Inject; +import com.cloud.cpu.CPU; import com.cloud.dc.DataCenter; import com.cloud.dc.dao.DataCenterDao; import com.cloud.domain.Domain; @@ -307,10 +308,15 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis to.setNics(nics); to.setDisks(vmProfile.getDisks().toArray(new DiskTO[vmProfile.getDisks().size()])); - if (vmProfile.getTemplate().getBits() == 32) { - to.setArch("i686"); + CPU.CPUArch templateArch = vmProfile.getTemplate().getArch(); + if (templateArch != null) { + to.setArch(templateArch.getType()); } else { - to.setArch("x86_64"); + if (vmProfile.getTemplate().getBits() == 32) { + to.setArch(CPU.CPUArch.x86.getType()); + } else { + to.setArch(CPU.CPUArch.amd64.getType()); + } } Map detailsInVm = _userVmDetailsDao.listDetailsKeyPairs(vm.getId()); From 96992d3d640967b62ca5e323890ca4052d05b072 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Wed, 24 Sep 2025 13:58:24 +0530 Subject: [PATCH 036/217] server: Fix vpclimit count for listAcccount API response (#11686) --- .../org/apache/cloudstack/api/response/AccountResponse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java index 6fc098295f6..d761d7c0394 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java @@ -464,7 +464,7 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Override public void setVpcLimit(String vpcLimit) { - this.vpcLimit = networkLimit; + this.vpcLimit = vpcLimit; } @Override From c24d2b88f6bff2a13b21cad8fc25ebf367c726be Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 24 Sep 2025 11:30:04 +0200 Subject: [PATCH 037/217] LDAP: honour nested groups for MSAD (#11696) --- .../org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java | 8 ++++++-- .../org/apache/cloudstack/ldap/LdapConfiguration.java | 5 ++++- .../apache/cloudstack/ldap/OpenLdapUserManagerImpl.java | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java index 552d5969a9e..e96606dca2f 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java @@ -93,10 +93,14 @@ public class ADLdapUserManagerImpl extends OpenLdapUserManagerImpl implements Ld } protected String getMemberOfAttribute(final Long domainId) { + String rc; if(_ldapConfiguration.isNestedGroupsEnabled(domainId)) { - return MICROSOFT_AD_NESTED_MEMBERS_FILTER; + rc = MICROSOFT_AD_NESTED_MEMBERS_FILTER; } else { - return MICROSOFT_AD_MEMBERS_FILTER; + rc = MICROSOFT_AD_MEMBERS_FILTER; } + logger.trace("using memberOf filter = {} for domain with id {}", rc, domainId); + + return rc; } } diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java index 6a62ad8d99d..87ff2d0a2ac 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java @@ -27,9 +27,12 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import com.cloud.utils.Pair; import org.apache.cloudstack.ldap.dao.LdapConfigurationDao; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class LdapConfiguration implements Configurable{ private final static String factory = "com.sun.jndi.ldap.LdapCtxFactory"; + protected Logger logger = LogManager.getLogger(getClass()); private static final ConfigKey ldapReadTimeout = new ConfigKey( Long.class, @@ -325,7 +328,7 @@ public class LdapConfiguration implements Configurable{ try { provider = LdapUserManager.Provider.valueOf(ldapProvider.valueIn(domainId).toUpperCase()); } catch (IllegalArgumentException ex) { - //openldap is the default + logger.warn("no LDAP provider found for domain {}, using openldap as default", domainId); provider = LdapUserManager.Provider.OPENLDAP; } return provider; diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java index 4c125af2ea6..d0b6bc4bd34 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java @@ -63,7 +63,7 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { final String firstname = LdapUtils.getAttributeValue(attributes, _ldapConfiguration.getFirstnameAttribute(domainId)); final String lastname = LdapUtils.getAttributeValue(attributes, _ldapConfiguration.getLastnameAttribute(domainId)); final String principal = result.getNameInNamespace(); - final List memberships = LdapUtils.getAttributeValues(attributes, _ldapConfiguration.getUserMemberOfAttribute(domainId)); + final List memberships = LdapUtils.getAttributeValues(attributes, getMemberOfAttribute(domainId)); String domain = principal.replace("cn=" + LdapUtils.getAttributeValue(attributes, _ldapConfiguration.getCommonNameAttribute()) + ",", ""); domain = domain.replace("," + _ldapConfiguration.getBaseDn(domainId), ""); @@ -87,7 +87,7 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { usernameFilter.append((username == null ? "*" : LdapUtils.escapeLDAPSearchFilter(username))); usernameFilter.append(")"); - String memberOfAttribute = _ldapConfiguration.getUserMemberOfAttribute(domainId); + String memberOfAttribute = getMemberOfAttribute(domainId); StringBuilder ldapGroupsFilter = new StringBuilder(); // this should get the trustmaps for this domain List ldapGroups = getMappedLdapGroups(domainId); From 36cfd76ce199ce159f496b27f9ed080488b96427 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 24 Sep 2025 13:53:27 +0200 Subject: [PATCH 038/217] KVM: fix delete vm snapshot if it does not exist with a Stopped vm (#11687) * KVM: fix delete vm snapshot if it does not exist with a Stopped vm * update 11687 --- .../wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java index 58a74d6e0f6..9f1b264ff71 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java @@ -40,6 +40,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Volume; +import com.cloud.utils.StringUtils; import com.cloud.utils.script.Script; @ResourceWrapper(handles = DeleteVMSnapshotCommand.class) @@ -104,7 +105,7 @@ public final class LibvirtDeleteVMSnapshotCommandWrapper extends CommandWrapper< commands.add(new String[]{Script.getExecutableAbsolutePath("awk"), "-F", " ", "{print $2}"}); commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), "^" + sanitizeBashCommandArgument(cmd.getTarget().getSnapshotName()) + "$"}); String qemu_img_snapshot = Script.executePipedCommands(commands, 0).second(); - if (qemu_img_snapshot == null) { + if (StringUtils.isEmpty(qemu_img_snapshot)) { logger.info("Cannot find snapshot " + cmd.getTarget().getSnapshotName() + " in file " + rootDisk.getPath() + ", return true"); return new DeleteVMSnapshotAnswer(cmd, cmd.getVolumeTOs()); } From b0c77190066fd74336ea831ccf9e5427e793ba7f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 24 Sep 2025 21:12:28 +0530 Subject: [PATCH 039/217] ui: do not show admin only options to users while registering template (#11702) --- ui/src/views/image/RegisterOrUploadTemplate.vue | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index 4cb6aa10df8..188f8c45ae9 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -685,6 +685,9 @@ export default { }) }, fetchCustomHypervisorName () { + if (!('listConfigurations' in store.getters.apis)) { + return + } const params = { name: 'hypervisor.custom.display.name' } @@ -702,6 +705,9 @@ export default { }) }, fetchExtensionsList () { + if (!this.isAdminRole) { + return + } this.loading = true getAPI('listExtensions', { }).then(response => { @@ -758,6 +764,9 @@ export default { name: 'Simulator' }) } + if (!this.isAdminRole) { + listhyperVisors = listhyperVisors.filter(hv => hv.name !== 'External') + } this.hyperVisor.opts = listhyperVisors }).finally(() => { this.hyperVisor.loading = false From 23c9e830473ef9a45a28439ea7ec18b1f05205af Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 25 Sep 2025 13:28:29 +0530 Subject: [PATCH 040/217] Create Instance from backup on another Zone (DRaaS use case) (#11560) * draas initial changes * Added option to enable disaster recovery on a backup respository. Added UpdateBackupRepositoryCmd api. * Added timeout for mount operation in backup restore configurable via global setting * Addressed review comments * fix for simulator test failures * Added UT for coverage * Fix create instance from backup ui for other providers * Added events to add/update backup repository * Fix race in fetchZones * One more fix in fetchZones in DeployVMFromBackup.vue * Fix zone selection in createNetwork via Create Instance from backup form. * Allow template/iso selection in create instance from backup ui * rename draasenabled to crosszoneinstancecreation * Added Cross-zone instance creation in test_backup_recovery_nas.py * Added UT in BackupManagerTest and UserVmManagerImplTest * Integration test added for Cross-zone instance creation in test_backup_recovery_nas.py --- .../main/java/com/cloud/event/EventTypes.java | 9 + .../com/cloud/vm/VirtualMachineProfile.java | 1 + .../apache/cloudstack/api/ApiConstants.java | 1 + .../repository/AddBackupRepositoryCmd.java | 8 +- .../repository/UpdateBackupRepositoryCmd.java | 116 ++++ .../response/BackupRepositoryResponse.java | 12 + .../cloudstack/backup/BackupManager.java | 2 + .../cloudstack/backup/BackupProvider.java | 4 +- .../cloudstack/backup/BackupRepository.java | 3 + .../backup/BackupRepositoryService.java | 2 + .../backup/RestoreBackupCommand.java | 9 + .../cloud/vm/VirtualMachineManagerImpl.java | 15 + .../cloudstack/backup/BackupRepositoryVO.java | 21 +- .../META-INF/db/schema-42100to42200.sql | 3 + .../backup/DummyBackupProvider.java | 9 +- .../cloudstack/backup/NASBackupProvider.java | 73 ++- .../backup/NASBackupProviderTest.java | 138 ++++- .../backup/NetworkerBackupProvider.java | 9 +- .../backup/VeeamBackupProvider.java | 9 +- .../LibvirtRestoreBackupCommandWrapper.java | 103 ++-- ...ibvirtRestoreBackupCommandWrapperTest.java | 499 ++++++++++++++++++ .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../java/com/cloud/vm/UserVmManagerImpl.java | 49 +- .../cloudstack/backup/BackupManagerImpl.java | 50 +- .../backup/BackupRepositoryServiceImpl.java | 53 +- .../com/cloud/vm/UserVmManagerImplTest.java | 126 ++++- .../cloudstack/backup/BackupManagerTest.java | 274 +++++++++- .../BackupRepositoryServiceImplTest.java | 243 +++++++++ .../smoke/test_backup_recovery_nas.py | 124 +++-- tools/marvin/marvin/lib/base.py | 14 +- ui/public/locales/en.json | 4 + ui/src/components/view/DeployVMFromBackup.vue | 111 +++- ui/src/config/section/config.js | 13 +- .../compute/wizard/TemplateIsoRadioGroup.vue | 10 +- .../network/CreateIsolatedNetworkForm.vue | 2 +- ui/src/views/network/CreateL2NetworkForm.vue | 2 +- ui/src/views/network/CreateNetwork.vue | 2 +- .../views/network/CreateSharedNetworkForm.vue | 2 +- ui/src/views/storage/CreateVMFromBackup.vue | 41 +- 39 files changed, 1961 insertions(+), 206 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/backup/BackupRepositoryServiceImplTest.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index be21f13267b..38e601c790a 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.BackupRepositoryService; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.datacenter.DataCenterIpv4GuestSubnet; import org.apache.cloudstack.extension.Extension; @@ -852,6 +853,10 @@ public class EventTypes { // Custom Action public static final String EVENT_CUSTOM_ACTION = "CUSTOM.ACTION"; + // Backup Repository + public static final String EVENT_BACKUP_REPOSITORY_ADD = "BACKUP.REPOSITORY.ADD"; + public static final String EVENT_BACKUP_REPOSITORY_UPDATE = "BACKUP.REPOSITORY.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1385,6 +1390,10 @@ public class EventTypes { entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_UPDATE, ExtensionCustomAction.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_DELETE, ExtensionCustomAction.class); + + // Backup Repository + entityEventDetails.put(EVENT_BACKUP_REPOSITORY_ADD, BackupRepositoryService.class); + entityEventDetails.put(EVENT_BACKUP_REPOSITORY_UPDATE, BackupRepositoryService.class); } public static boolean isNetworkEvent(String eventType) { diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index c67ee4eabc2..5c78d6bedd6 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -78,6 +78,7 @@ public interface VirtualMachineProfile { public static final Param BootIntoSetup = new Param("enterHardwareSetup"); public static final Param PreserveNics = new Param("PreserveNics"); public static final Param ConsiderLastHost = new Param("ConsiderLastHost"); + public static final Param ReturnAfterVolumePrepare = new Param("ReturnAfterVolumePrepare"); private String name; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 489d737b5bb..6c84e54b2d1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -139,6 +139,7 @@ public class ApiConstants { public static final String CPU_SPEED = "cpuspeed"; public static final String CPU_LOAD_AVERAGE = "cpuloadaverage"; public static final String CREATED = "created"; + public static final String CROSS_ZONE_INSTANCE_CREATION = "crosszoneinstancecreation"; public static final String CTX_ACCOUNT_ID = "ctxaccountid"; public static final String CTX_DETAILS = "ctxDetails"; public static final String CTX_USER_ID = "ctxuserid"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java index 5d0c838bc37..64998a74954 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java @@ -63,12 +63,14 @@ public class AddBackupRepositoryCmd extends BaseCmd { type = CommandType.UUID, entityType = ZoneResponse.class, required = true, - description = "ID of the zone where the backup repository is to be added") + description = "ID of the zone where the backup repository is to be added for taking backups") private Long zoneId; @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, description = "capacity of this backup repository") private Long capacityBytes; + @Parameter(name = ApiConstants.CROSS_ZONE_INSTANCE_CREATION, type = CommandType.BOOLEAN, description = "backups on this repository can be used to create Instances on all Zones", since = "4.22.0") + private Boolean crossZoneInstanceCreation; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -109,6 +111,10 @@ public class AddBackupRepositoryCmd extends BaseCmd { return capacityBytes; } + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java new file mode 100644 index 00000000000..5ffd79e497e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup.repository; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryService; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; + +@APICommand(name = "updateBackupRepository", + description = "Update a backup repository", + responseObject = BackupRepositoryResponse.class, since = "4.22.0", + authorized = {RoleType.Admin}) +public class UpdateBackupRepositoryCmd extends BaseCmd { + + @Inject + private BackupRepositoryService backupRepositoryService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, entityType = BackupRepositoryResponse.class, description = "ID of the backup repository") + private Long id; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the backup repository") + private String name; + + @Parameter(name = ApiConstants.ADDRESS, type = CommandType.STRING, description = "address of the backup repository") + private String address; + + @Parameter(name = ApiConstants.MOUNT_OPTIONS, type = CommandType.STRING, description = "shared storage mount options") + private String mountOptions; + + @Parameter(name = ApiConstants.CROSS_ZONE_INSTANCE_CREATION, type = CommandType.BOOLEAN, description = "backups in this repository can be used to create Instances on all Zones") + private Boolean crossZoneInstanceCreation; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public BackupRepositoryService getBackupRepositoryService() { + return backupRepositoryService; + } + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getAddress() { + return address; + } + + public String getMountOptions() { + return mountOptions == null ? "" : mountOptions; + } + + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + + ///////////////////////////////////////////////////// + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + try { + BackupRepository result = backupRepositoryService.updateBackupRepository(this); + if (result != null) { + BackupRepositoryResponse response = _responseGenerator.createBackupRepositoryResponse(result); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the backup repository"); + } + } catch (Exception ex4) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex4.getMessage()); + } + + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java index 0db51f04034..327bbae0051 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java @@ -61,6 +61,10 @@ public class BackupRepositoryResponse extends BaseResponse { @Param(description = "capacity of the backup repository") private Long capacityBytes; + @SerializedName(ApiConstants.CROSS_ZONE_INSTANCE_CREATION) + @Param(description = "the backups in this repository can be used to create Instances on all Zones") + private Boolean crossZoneInstanceCreation; + @SerializedName("created") @Param(description = "the date and time the backup repository was added") private Date created; @@ -132,6 +136,14 @@ public class BackupRepositoryResponse extends BaseResponse { this.capacityBytes = capacityBytes; } + public Boolean getCrossZoneInstanceCreation() { + return crossZoneInstanceCreation; + } + + public void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation) { + this.crossZoneInstanceCreation = crossZoneInstanceCreation; + } + public Date getCreated() { return created; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index c4b92fc9e05..37d21613c3d 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -205,6 +205,8 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer Boolean canCreateInstanceFromBackup(Long backupId); + Boolean canCreateInstanceFromBackupAcrossZones(Long backupId); + /** * Restore a backup to a new Instance */ diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 1eb36f89556..32a714370df 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -23,6 +23,8 @@ import com.cloud.vm.VirtualMachine; public interface BackupProvider { + Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering); + /** * Returns the unique name of the provider * @return returns provider name @@ -85,7 +87,7 @@ public interface BackupProvider { */ boolean deleteBackup(Backup backup, boolean forced); - boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); + Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); /** * Restore VM from backup diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java index be539a0eb04..886d13c13f9 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java @@ -28,9 +28,12 @@ public interface BackupRepository extends InternalIdentity, Identity { String getType(); String getAddress(); String getMountOptions(); + void setMountOptions(String mountOptions); void setUsedBytes(Long usedBytes); Long getCapacityBytes(); Long getUsedBytes(); void setCapacityBytes(Long capacityBytes); + Boolean crossZoneInstanceCreationEnabled(); + void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation); Date getCreated(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java index ae71053e400..875fc3b3d90 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java @@ -23,11 +23,13 @@ import com.cloud.utils.Pair; import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; +import org.apache.cloudstack.api.command.user.backup.repository.UpdateBackupRepositoryCmd; import java.util.List; public interface BackupRepositoryService { BackupRepository addBackupRepository(AddBackupRepositoryCmd cmd); + BackupRepository updateBackupRepository(UpdateBackupRepositoryCmd cmd); boolean deleteBackupRepository(DeleteBackupRepositoryCmd cmd); Pair, Integer> listBackupRepositories(ListBackupRepositoriesCmd cmd); diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index f447fbe3d00..453b236df6b 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -36,6 +36,7 @@ public class RestoreBackupCommand extends Command { private Boolean vmExists; private String restoreVolumeUUID; private VirtualMachine.State vmState; + private Integer mountTimeout; protected RestoreBackupCommand() { super(); @@ -136,4 +137,12 @@ public class RestoreBackupCommand extends Command { public void setBackupVolumesUUIDs(List backupVolumesUUIDs) { this.backupVolumesUUIDs = backupVolumesUUIDs; } + + public Integer getMountTimeout() { + return this.mountTimeout; + } + + public void setMountTimeout(Integer mountTimeout) { + this.mountTimeout = mountTimeout; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 3a6e1b62277..b5597280364 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1482,6 +1482,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac volumeMgr.prepare(vmProfile, dest); } + if (params != null) { + Boolean returnAfterVolumePrepare = (Boolean) params.get(VirtualMachineProfile.Param.ReturnAfterVolumePrepare); + if (Boolean.TRUE.equals(returnAfterVolumePrepare)) { + logger.info("Returning from VM start command execution for VM {} as requested. Volumes are prepared and ready.", vm.getUuid()); + + if (!changeState(vm, Event.AgentReportStopped, destHostId, work, Step.Done)) { + logger.error("Unable to transition to a new state. VM uuid: {}, VM oldstate: {}, Event: {}", vm, vm.getState(), Event.AgentReportStopped); + throw new ConcurrentOperationException(String.format("Failed to deploy VM %s", vm)); + } + + logger.debug("Volume preparation completed for VM {} (VM state set to Stopped)", vm); + return; + } + } + if (!reuseVolume) { reuseVolume = true; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java index 98efa94ceca..1764496a6c0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java @@ -67,6 +67,9 @@ public class BackupRepositoryVO implements BackupRepository { @Column(name = "capacity_bytes", nullable = true) private Long capacityBytes; + @Column(name = "cross_zone_instance_creation") + private Boolean crossZoneInstanceCreation; + @Column(name = "created") @Temporal(value = TemporalType.TIMESTAMP) private Date created; @@ -79,7 +82,7 @@ public class BackupRepositoryVO implements BackupRepository { this.uuid = UUID.randomUUID().toString(); } - public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes) { + public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes, final Boolean crossZoneInstanceCreation) { this(); this.zoneId = zoneId; this.provider = provider; @@ -88,6 +91,7 @@ public class BackupRepositoryVO implements BackupRepository { this.address = address; this.mountOptions = mountOptions; this.capacityBytes = capacityBytes; + this.crossZoneInstanceCreation = crossZoneInstanceCreation; this.created = new Date(); } @@ -139,6 +143,11 @@ public class BackupRepositoryVO implements BackupRepository { return mountOptions; } + @Override + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + @Override public Long getUsedBytes() { return usedBytes; @@ -154,6 +163,16 @@ public class BackupRepositoryVO implements BackupRepository { return capacityBytes; } + @Override + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + + @Override + public void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation) { + this.crossZoneInstanceCreation = crossZoneInstanceCreation; + } + @Override public void setCapacityBytes(Long capacityBytes) { this.capacityBytes = capacityBytes; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 0538e45d1dc..4fcb2b75de5 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -25,3 +25,6 @@ CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', ' -- Increase length of scripts_version column to 128 due to md5sum to sha512sum change CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.domain_router', 'scripts_version', 'scripts_version', 'VARCHAR(128)'); + +-- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository. +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone'''); diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index 48082c1c8a4..b228a9f8ce0 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -53,6 +53,11 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { @Inject private DiskOfferingDao diskOfferingDao; + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return true; + } + @Override public String getName() { return "dummy"; @@ -199,7 +204,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return true; + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return new Pair<>(true, null); } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index e5f98ad291b..9cd2f20e386 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -70,9 +70,18 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; + public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); + ConfigKey NASBackupRestoreMountTimeout = new ConfigKey<>("Advanced", Integer.class, + "nas.backup.restore.mount.timeout", + "30", + "Timeout in seconds after which backup repository mount for restore fails.", + true, + BackupFrameworkEnabled.key()); + @Inject private BackupDao backupDao; @@ -115,30 +124,45 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private DiskOfferingDao diskOfferingDao; - protected Host getLastVMHypervisorHost(VirtualMachine vm) { - Long hostId = vm.getLastHostId(); - if (hostId == null) { - LOG.debug("Cannot find last host for vm. This should never happen, please check your database."); + private Long getClusterIdFromRootVolume(VirtualMachine vm) { + VolumeVO rootVolume = volumeDao.getInstanceRootVolume(vm.getId()); + StoragePoolVO rootDiskPool = primaryDataStoreDao.findById(rootVolume.getPoolId()); + if (rootDiskPool == null) { return null; } - Host host = hostDao.findById(hostId); + return rootDiskPool.getClusterId(); + } - if (host.getStatus() == Status.Up) { - return host; - } else { + protected Host getVMHypervisorHost(VirtualMachine vm) { + Long hostId = vm.getLastHostId(); + Long clusterId = null; + + if (hostId != null) { + Host host = hostDao.findById(hostId); + if (host.getStatus() == Status.Up) { + return host; + } // Try to find any Up host in the same cluster - for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(host.getClusterId())) { + clusterId = host.getClusterId(); + } else { + // Try to find any Up host in the same cluster as the root volume + clusterId = getClusterIdFromRootVolume(vm); + } + + if (clusterId != null) { + for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(clusterId)) { if (hostInCluster.getStatus() == Status.Up) { - LOG.debug("Found Host {} in cluster {}", hostInCluster, host.getClusterId()); + LOG.debug("Found Host {} in cluster {}", hostInCluster, clusterId); return hostInCluster; } } } + // Try to find any Host in the zone - return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, host.getDataCenterId()); + return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, vm.getDataCenterId()); } - protected Host getVMHypervisorHost(VirtualMachine vm) { + protected Host getVMHypervisorHostForBackup(VirtualMachine vm) { Long hostId = vm.getHostId(); if (hostId == null && VirtualMachine.State.Running.equals(vm.getState())) { throw new CloudRuntimeException(String.format("Unable to find the hypervisor host for %s. Make sure the virtual machine is running", vm.getName())); @@ -158,7 +182,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public Pair takeBackup(final VirtualMachine vm, Boolean quiesceVM) { - final Host host = getVMHypervisorHost(vm); + final Host host = getVMHypervisorHostForBackup(vm); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); if (backupRepository == null) { @@ -249,16 +273,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return restoreVMBackup(vm, backup); } @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { - return restoreVMBackup(vm, backup); + return restoreVMBackup(vm, backup).first(); } - private boolean restoreVMBackup(VirtualMachine vm, Backup backup) { + private Pair restoreVMBackup(VirtualMachine vm, Backup backup) { List backedVolumesUUIDs = backup.getBackedUpVolumes().stream() .sorted(Comparator.comparingLong(Backup.VolumeInfo::getDeviceId)) .map(Backup.VolumeInfo::getUuid) @@ -271,7 +295,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm, backup); BackupRepository backupRepository = getBackupRepository(backup); - final Host host = getLastVMHypervisorHost(vm); + final Host host = getVMHypervisorHost(vm); RestoreBackupCommand restoreCommand = new RestoreBackupCommand(); restoreCommand.setBackupPath(backup.getExternalId()); restoreCommand.setBackupRepoType(backupRepository.getType()); @@ -282,6 +306,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoreCommand.setRestoreVolumePaths(getVolumePaths(restoreVolumes)); restoreCommand.setVmExists(vm.getRemoved() == null); restoreCommand.setVmState(vm.getState()); + restoreCommand.setMountTimeout(NASBackupRestoreMountTimeout.value()); BackupAnswer answer; try { @@ -291,7 +316,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); } - return answer.getResult(); + return new Pair<>(answer.getResult(), answer.getDetails()); } private List getVolumePaths(List volumes) { @@ -398,7 +423,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co final Host host; final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); if (vm != null) { - host = getLastVMHypervisorHost(vm); + host = getVMHypervisorHost(vm); } else { host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); } @@ -513,9 +538,19 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return true; } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(backupOffering.getId()); + if (backupRepository == null) { + throw new CloudRuntimeException("Backup repository not found for the backup offering" + backupOffering.getName()); + } + return Boolean.TRUE.equals(backupRepository.crossZoneInstanceCreationEnabled()); + } + @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ + NASBackupRestoreMountTimeout }; } diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index d6f29dc1aac..7540cabbbf5 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -21,11 +21,9 @@ import static org.mockito.Mockito.mock; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; -import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupRepositoryDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -39,6 +37,7 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.agent.AgentManager; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -51,6 +50,12 @@ import com.cloud.utils.Pair; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + @RunWith(MockitoJUnitRunner.class) public class NASBackupProviderTest { @Spy @@ -84,6 +89,9 @@ public class NASBackupProviderTest { @Mock private ResourceManager resourceManager; + @Mock + private PrimaryDataStoreDao storagePoolDao; + @Test public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { Long hostId = 1L; @@ -94,7 +102,7 @@ public class NASBackupProviderTest { ReflectionTestUtils.setField(backup, "id", 1L); BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); VMInstanceVO vm = mock(VMInstanceVO.class); Mockito.when(vm.getLastHostId()).thenReturn(hostId); @@ -113,7 +121,7 @@ public class NASBackupProviderTest { @Test public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); HostVO host = mock(HostVO.class); Mockito.when(resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, 1L)).thenReturn(host); @@ -132,7 +140,7 @@ public class NASBackupProviderTest { @Test public void testListBackupOfferings() { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); ReflectionTestUtils.setField(backupRepository, "uuid", "uuid"); Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")).thenReturn(Collections.singletonList(backupRepository)); @@ -146,11 +154,11 @@ public class NASBackupProviderTest { @Test public void testGetBackupStorageStats() { BackupRepositoryVO backupRepository1 = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1000L); + "nfs", "address", "sync", 1000L, null); backupRepository1.setUsedBytes(500L); BackupRepositoryVO backupRepository2 = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 2000L); + "nfs", "address", "sync", 2000L, null); backupRepository2.setUsedBytes(600L); Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")) @@ -227,4 +235,118 @@ public class NASBackupProviderTest { Mockito.verify(backupDao).update(Mockito.anyLong(), Mockito.any(BackupVO.class)); Mockito.verify(agentManager).send(anyLong(), Mockito.any(TakeBackupCommand.class)); } + + @Test + public void testGetVMHypervisorHost() { + Long hostId = 1L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + + HostVO host = mock(HostVO.class); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(hostId, result.getId())); + Mockito.verify(hostDao).findById(hostId); + } + + @Test + public void testGetVMHypervisorHostWithHostDown() { + Long hostId = 1L; + Long clusterId = 2L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + + HostVO downHost = mock(HostVO.class); + Mockito.when(downHost.getStatus()).thenReturn(Status.Down); + Mockito.when(downHost.getClusterId()).thenReturn(clusterId); + Mockito.when(hostDao.findById(hostId)).thenReturn(downHost); + + HostVO upHostInCluster = mock(HostVO.class); + Mockito.when(upHostInCluster.getId()).thenReturn(3L); + Mockito.when(upHostInCluster.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(List.of(upHostInCluster)); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(3L), result.getId())); + Mockito.verify(hostDao).findById(hostId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + } + + @Test + public void testGetVMHypervisorHostWithUpHostViaRootVolumeCluster() { + Long vmId = 1L; + Long zoneId = 1L; + Long clusterId = 2L; + Long poolId = 3L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(null); + Mockito.when(vm.getId()).thenReturn(vmId); + + VolumeVO rootVolume = mock(VolumeVO.class); + Mockito.when(rootVolume.getPoolId()).thenReturn(poolId); + Mockito.when(volumeDao.getInstanceRootVolume(vmId)).thenReturn(rootVolume); + + StoragePoolVO storagePool = mock(StoragePoolVO.class); + Mockito.when(storagePool.getClusterId()).thenReturn(clusterId); + Mockito.when(storagePoolDao.findById(poolId)).thenReturn(storagePool); + + HostVO upHostInCluster = mock(HostVO.class); + Mockito.when(upHostInCluster.getId()).thenReturn(4L); + Mockito.when(upHostInCluster.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(List.of(upHostInCluster)); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(4L), result.getId())); + Mockito.verify(volumeDao).getInstanceRootVolume(vmId); + Mockito.verify(storagePoolDao).findById(poolId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + } + + @Test + public void testGetVMHypervisorHostFallbackToZoneWideKVMHost() { + Long hostId = 1L; + Long clusterId = 2L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + Mockito.when(vm.getDataCenterId()).thenReturn(zoneId); + + HostVO downHost = mock(HostVO.class); + Mockito.when(downHost.getStatus()).thenReturn(Status.Down); + Mockito.when(downHost.getClusterId()).thenReturn(clusterId); + Mockito.when(hostDao.findById(hostId)).thenReturn(downHost); + + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(Collections.emptyList()); + + HostVO fallbackHost = mock(HostVO.class); + Mockito.when(fallbackHost.getId()).thenReturn(5L); + Mockito.when(resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId)) + .thenReturn(fallbackHost); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(5L), result.getId())); + Mockito.verify(hostDao).findById(hostId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + Mockito.verify(resourceManager).findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + } } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index f39aedb55f2..66b633e11a9 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -158,6 +158,11 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid }; } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return false; + } + @Override public String getName() { return "networker"; @@ -630,7 +635,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid public boolean willDeleteBackupsOnOfferingRemoval() { return false; } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return true; + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return new Pair<>(true, null); } } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index c81c5d34ea2..39970dab342 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -337,11 +337,11 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { final Long zoneId = backup.getZoneId(); final String restorePointId = backup.getExternalId(); final String restoreLocation = vm.getInstanceName(); - return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid).first(); + return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid); } @Override @@ -358,6 +358,11 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, public void syncBackupStorageStats(Long zoneId) { } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return false; + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 0e5091ebcf4..243cf2efa03 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -61,42 +61,49 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper backedVolumeUUIDs = command.getBackupVolumesUUIDs(); List restoreVolumePaths = command.getRestoreVolumePaths(); String restoreVolumeUuid = command.getRestoreVolumeUUID(); + Integer mountTimeout = command.getMountTimeout() * 1000; String newVolumeId = null; try { + String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions, mountTimeout); if (Objects.isNull(vmExists)) { String volumePath = restoreVolumePaths.get(0); int lastIndex = volumePath.lastIndexOf("/"); newVolumeId = volumePath.substring(lastIndex + 1); - restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid, - new Pair<>(vmName, command.getVmState()), mountOptions); + restoreVolume(backupPath, volumePath, diskType, restoreVolumeUuid, + new Pair<>(vmName, command.getVmState()), mountDirectory); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, mountDirectory); } else { - restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, mountDirectory); } } catch (CloudRuntimeException e) { - String errorMessage = "Failed to restore backup for VM: " + vmName + "."; - if (e.getMessage() != null && !e.getMessage().isEmpty()) { - errorMessage += " Details: " + e.getMessage(); - } - logger.error(errorMessage); + String errorMessage = e.getMessage() != null ? e.getMessage() : ""; return new BackupAnswer(command, false, errorMessage); } return new BackupAnswer(command, true, newVolumeId); } - private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { + private void verifyBackupFile(String backupPath, String volUuid) { + if (!checkBackupPathExists(backupPath)) { + throw new CloudRuntimeException(String.format("Backup file for the volume [%s] does not exist.", volUuid)); + } + if (!checkBackupFileImage(backupPath)) { + throw new CloudRuntimeException(String.format("Backup qcow2 file for the volume [%s] is corrupt.", volUuid)); + } + } + + private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, + String backupPath, String mountDirectory) { String diskType = "root"; - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); try { for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { String restoreVolumePath = restoreVolumePaths.get(idx); String backupVolumeUuid = backedVolumesUUIDs.get(idx); Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, null, backupPath, diskType, backupVolumeUuid); diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(restoreVolumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -107,15 +114,14 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper volumePaths, String vmName, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); + private void restoreVolumesOfDestroyedVMs(List volumePaths, String vmName, String backupPath, String mountDirectory) { String diskType = "root"; try { for (int i = 0; i < volumePaths.size(); i++) { String volumePath = volumePaths.get(i); Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -126,12 +132,12 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper vmNameAndState, String mountOptions) { - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); + private void restoreVolume(String backupPath, String volumePath, String diskType, String volumeUUID, + Pair vmNameAndState, String mountDirectory) { Pair bkpPathAndVolUuid; try { bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID); + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -140,8 +146,6 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(bkpPath, volUuid); } + private boolean checkBackupFileImage(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("qemu-img check %s", backupPath)); + return exitValue == 0; + } + + private boolean checkBackupPathExists(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("ls %s", backupPath)); + return exitValue == 0; + } + private boolean replaceVolumeWithBackup(String volumePath, String backupPath) { int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath)); return exitValue == 0; diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java new file mode 100644 index 00000000000..d120abd0a1b --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java @@ -0,0 +1,499 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.script.Script; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.RestoreBackupCommand; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtRestoreBackupCommandWrapperTest { + + private LibvirtRestoreBackupCommandWrapper wrapper; + private LibvirtComputingResource libvirtComputingResource; + private RestoreBackupCommand command; + + @Before + public void setUp() { + wrapper = new LibvirtRestoreBackupCommandWrapper(); + libvirtComputingResource = Mockito.mock(LibvirtComputingResource.class); + command = Mockito.mock(RestoreBackupCommand.class); + } + + @Test + public void testExecuteWithVmExistsNull() throws Exception { + when(command.getVmName()).thenReturn("test-vm"); + when(command.getBackupPath()).thenReturn("backup/path"); + when(command.getBackupRepoAddress()).thenReturn("192.168.1.100:/backup"); + when(command.getBackupRepoType()).thenReturn("nfs"); + when(command.getMountOptions()).thenReturn("rw"); + when(command.isVmExists()).thenReturn(null); + when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); + when(command.getRestoreVolumeUUID()).thenReturn("volume-123"); + when(command.getVmState()).thenReturn(VirtualMachine.State.Running); + when(command.getMountTimeout()).thenReturn(30); + + try (MockedStatic filesMock = mockStatic(Files.class)) { + Path tempPath = Mockito.mock(Path.class); + when(tempPath.toString()).thenReturn("/tmp/csbackup.abc123"); + filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath); + + try (MockedStatic + + diff --git a/ui/src/views/storage/RecurringSnapshotVolume.vue b/ui/src/views/storage/RecurringSnapshotVolume.vue index 1cac35c3eba..5eed42a736d 100644 --- a/ui/src/views/storage/RecurringSnapshotVolume.vue +++ b/ui/src/views/storage/RecurringSnapshotVolume.vue @@ -17,11 +17,33 @@ @@ -53,44 +80,150 @@ export default { props: { resource: { type: Object, - required: true + required: false, + default: () => null } }, + inject: ['parentFetchData'], data () { return { loading: false, - dataSource: [] + dataSource: [], + volumes: [], + volumesLoading: false, + selectedVolumeId: null, + selectedVolume: null + } + }, + computed: { + resourceType () { + if (!this.resource) return 'none' + if (this.resource.type === 'ROOT' || this.resource.type === 'DATADISK' || + this.resource.state === 'Ready' || this.resource.state === 'Allocated' || + this.resource.sizegb !== undefined) { + return 'volume' + } + if (this.resource.intervaltype !== undefined || this.resource.schedule !== undefined) { + return 'snapshotpolicy' + } + + return 'unknown' + }, + + isVolumeResource () { + return this.resourceType === 'volume' + }, + + currentVolumeResource () { + if (this.isVolumeResource) { + return this.resource + } else { + return this.selectedVolume + } } }, created () { - this.fetchData() + if (this.isVolumeResource) { + this.fetchData() + } else { + this.fetchVolumes() + } }, methods: { + async fetchVolumes () { + this.volumesLoading = true + try { + const response = await getAPI('listVolumes', { listAll: true }) + const volumes = response.listvolumesresponse.volume || [] + this.volumes = volumes.filter(volume => { + return volume.state === 'Ready' && + (volume.hypervisor !== 'KVM' || + (['Stopped', 'Destroyed'].includes(volume.vmstate)) || + (this.$store.getters.features.kvmsnapshotenabled)) + }) + } catch (error) { + this.$message.error(this.$t('message.error.fetch.volumes')) + console.error('Error fetching volumes:', error) + } finally { + this.volumesLoading = false + } + }, + onVolumeChange (volumeId) { + const volume = this.volumes.find(v => v.id === volumeId) + if (volume) { + this.selectedVolume = volume + this.selectedVolumeId = volumeId + this.dataSource = [] + this.fetchData() + } + }, fetchData () { - const params = {} + const volumeResource = this.currentVolumeResource + if (!volumeResource || !volumeResource.id) { + return + } + + const params = { + volumeid: volumeResource.id, + listAll: true + } + this.dataSource = [] this.loading = true - params.volumeid = this.resource.id + getAPI('listSnapshotPolicies', params).then(json => { this.loading = false const listSnapshotPolicies = json.listsnapshotpoliciesresponse.snapshotpolicy if (listSnapshotPolicies && listSnapshotPolicies.length > 0) { this.dataSource = listSnapshotPolicies } + }).catch(error => { + this.loading = false + this.$message.error(this.$t('message.error.fetch.snapshot.policies')) + console.error('Error fetching snapshot policies:', error) }) }, handleRefresh () { this.fetchData() + this.parentFetchData() }, closeAction () { + this.fetchData() this.$emit('close-action') + }, + filterOption (input, option) { + return option.children[0].children.toLowerCase().indexOf(input.toLowerCase()) >= 0 } } } From 2b1f0bbbdbd6909076e7d7a5cb7fdc0a954450cc Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 10 Oct 2025 12:35:41 +0530 Subject: [PATCH 080/217] UI: Fix for cluster addition in VMware (#11812) --- ui/src/views/infra/ClusterAdd.vue | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ui/src/views/infra/ClusterAdd.vue b/ui/src/views/infra/ClusterAdd.vue index 7fee1f11bf6..be561832f31 100644 --- a/ui/src/views/infra/ClusterAdd.vue +++ b/ui/src/views/infra/ClusterAdd.vue @@ -155,7 +155,7 @@ - + @@ -266,6 +266,7 @@ export default { this.loading = true getAPI('listZones', { showicon: true }).then(response => { this.zonesList = response.listzonesresponse.zone || [] + this.form.zoneid = this.zonesList?.[0]?.id || null this.fetchPods() }).catch(error => { this.$notifyError(error) @@ -288,7 +289,7 @@ export default { fetchPods () { this.loading = true getAPI('listPods', { - zoneid: this.zoneId + zoneid: this.form.zoneid }).then(response => { this.podsList = response.listpodsresponse.pod || [] }).catch(error => { @@ -314,12 +315,12 @@ export default { this.loading = true this.clustertype = 'ExternalManaged' getAPI('listVmwareDcs', { - zoneid: this.form.zoneId + zoneid: this.form.zoneid }).then(response => { var vmwaredcs = response.listvmwaredcsresponse.VMwareDC if (vmwaredcs !== null) { this.form.host = vmwaredcs[0].vcenter - this.form.dataCenter = vmwaredcs[0].name + this.form.datacenter = vmwaredcs[0].name } }).catch(error => { this.$notification.error({ @@ -352,7 +353,7 @@ export default { var clustername = values.clustername var url = '' if (values.hypervisor === 'VMware') { - clustername = `${this.host}/${this.dataCenter}/${clustername}` + clustername = `${this.form.host}/${this.form.datacenter}/${clustername}` url = `http://${clustername}` } this.loading = true From 67250d99d4494b888270e76202906566bc614673 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Oct 2025 13:09:25 +0530 Subject: [PATCH 081/217] ui: fix add host form state on submit (#11815) --- ui/src/views/infra/HostAdd.vue | 458 +++++++++++++++++---------------- 1 file changed, 230 insertions(+), 228 deletions(-) diff --git a/ui/src/views/infra/HostAdd.vue b/ui/src/views/infra/HostAdd.vue index 4cc179ac467..6879fa89f8b 100644 --- a/ui/src/views/infra/HostAdd.vue +++ b/ui/src/views/infra/HostAdd.vue @@ -17,236 +17,238 @@