diff --git a/agent/pom.xml b/agent/pom.xml index 9caa6d992c8..dc72b50fa19 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/api/pom.xml b/api/pom.xml index 32897725e0c..89ca1c4c3bb 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 63f5455cfd0..60db7abb734 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -116,6 +116,8 @@ public interface AccountService { void checkAccess(Account account, AccessType accessType, boolean sameOwner, String apiName, ControlledEntity... entities) throws PermissionDeniedException; + void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource); + Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly); /** diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java index a7610717435..fa6e3ea5d45 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java @@ -82,7 +82,7 @@ public class ListBackupScheduleCmd extends BaseCmd { List schedules = backupManager.listBackupSchedule(getVmId()); ListResponse response = new ListResponse<>(); List scheduleResponses = new ArrayList<>(); - if (CollectionUtils.isNullOrEmpty(schedules)) { + if (!CollectionUtils.isNullOrEmpty(schedules)) { for (BackupSchedule schedule : schedules) { scheduleResponses.add(_responseGenerator.createBackupScheduleResponse(schedule)); } diff --git a/client/pom.xml b/client/pom.xml index d89dbbbbe9d..eddf49c040f 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/core/pom.xml b/core/pom.xml index 83cdee8cf4f..86f48b1a663 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/debian/changelog b/debian/changelog index cbc4fcfdb2d..1da019b0293 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +cloudstack (4.20.0.0) unstable; urgency=low + + * Update the version to 4.20.0.0 + + -- the Apache CloudStack project Tue, 19 Nov 2024 08:54:07 -0300 + cloudstack (4.20.0.0-SNAPSHOT) unstable; urgency=low * Update the version to 4.20.0.0-SNAPSHOT diff --git a/debian/control b/debian/control index ed4f95d8e49..c0cb95af035 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker +Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/developer/pom.xml b/developer/pom.xml index b44446ec0a5..2dedab75a5e 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/engine/api/pom.xml b/engine/api/pom.xml index 1112e6eff8b..8e990b19620 100644 --- a/engine/api/pom.xml +++ b/engine/api/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/engine/components-api/pom.xml b/engine/components-api/pom.xml index b06b644d67f..56e136d3e46 100644 --- a/engine/components-api/pom.xml +++ b/engine/components-api/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/engine/orchestration/pom.xml b/engine/orchestration/pom.xml index e4953fc0cbe..77a4c0929e9 100755 --- a/engine/orchestration/pom.xml +++ b/engine/orchestration/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/engine/pom.xml b/engine/pom.xml index 5e52544aeca..6ad7c69264f 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 82120ae70cc..97ff8d80e5d 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java index d1149e47408..e4c19fd1666 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java @@ -126,14 +126,7 @@ public class VolumeStatsDaoImpl extends GenericDaoBase impl logger.debug(String.format("Starting to remove all volume_stats rows older than [%s].", limitDate)); - long totalRemoved = 0; - long removed; - - do { - removed = expunge(sc, limitPerQuery); - totalRemoved += removed; - logger.trace(String.format("Removed [%s] volume_stats rows on the last update and a sum of [%s] volume_stats rows older than [%s] until now.", removed, totalRemoved, limitDate)); - } while (limitPerQuery > 0 && removed >= limitPerQuery); + long totalRemoved = batchExpunge(sc, limitPerQuery); logger.info(String.format("Removed a total of [%s] volume_stats rows older than [%s].", totalRemoved, limitDate)); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java index aa58e489364..327acec0c17 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java @@ -123,14 +123,7 @@ public class VmStatsDaoImpl extends GenericDaoBase implements V logger.debug(String.format("Starting to remove all vm_stats rows older than [%s].", limitDate)); - long totalRemoved = 0; - long removed; - - do { - removed = expunge(sc, limitPerQuery); - totalRemoved += removed; - logger.trace(String.format("Removed [%s] vm_stats rows on the last update and a sum of [%s] vm_stats rows older than [%s] until now.", removed, totalRemoved, limitDate)); - } while (limitPerQuery > 0 && removed >= limitPerQuery); + long totalRemoved = batchExpunge(sc, limitPerQuery); logger.info(String.format("Removed a total of [%s] vm_stats rows older than [%s].", totalRemoved, limitDate)); } diff --git a/engine/service/pom.xml b/engine/service/pom.xml index 34221e1001d..f4cfda50449 100644 --- a/engine/service/pom.xml +++ b/engine/service/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 cloud-engine-service war diff --git a/engine/storage/cache/pom.xml b/engine/storage/cache/pom.xml index a1b7aff7afd..d6739b60992 100644 --- a/engine/storage/cache/pom.xml +++ b/engine/storage/cache/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/storage/configdrive/pom.xml b/engine/storage/configdrive/pom.xml index b14acf10138..19f7ea05608 100644 --- a/engine/storage/configdrive/pom.xml +++ b/engine/storage/configdrive/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/storage/datamotion/pom.xml b/engine/storage/datamotion/pom.xml index 5620ca8ef70..2f303277a74 100644 --- a/engine/storage/datamotion/pom.xml +++ b/engine/storage/datamotion/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 07b53842640..9755dcbb0df 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -1958,25 +1958,26 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { * - Full clones (no backing file): Take snapshot of the VM prior disk creation * Return this information */ - protected void setVolumeMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, - VirtualMachineTO vmTO, Host srcHost, StoragePoolVO destStoragePool) { - if (!destStoragePool.isManaged()) { - String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo); - - String srcPoolUuid = srcVolumeInfo.getDataStore().getUuid(); - StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId()); - Storage.StoragePoolType srcPoolType = srcPool.getPoolType(); - - MigrationOptions migrationOptions; - if (StringUtils.isNotBlank(srcVolumeBackingFile)) { - migrationOptions = createLinkedCloneMigrationOptions(srcVolumeInfo, destVolumeInfo, - srcVolumeBackingFile, srcPoolUuid, srcPoolType); - } else { - migrationOptions = createFullCloneMigrationOptions(srcVolumeInfo, vmTO, srcHost, srcPoolUuid, srcPoolType); - } - migrationOptions.setTimeout(StorageManager.KvmStorageOnlineMigrationWait.value()); - destVolumeInfo.setMigrationOptions(migrationOptions); + protected void setVolumeMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, VirtualMachineTO vmTO, Host srcHost, StoragePoolVO destStoragePool, + MigrationOptions.Type migrationType) { + if (destStoragePool.isManaged()) { + return; } + + String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo); + + String srcPoolUuid = srcVolumeInfo.getDataStore().getUuid(); + StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId()); + Storage.StoragePoolType srcPoolType = srcPool.getPoolType(); + + MigrationOptions migrationOptions; + if (MigrationOptions.Type.LinkedClone.equals(migrationType)) { + migrationOptions = createLinkedCloneMigrationOptions(srcVolumeInfo, destVolumeInfo, srcVolumeBackingFile, srcPoolUuid, srcPoolType); + } else { + migrationOptions = createFullCloneMigrationOptions(srcVolumeInfo, vmTO, srcHost, srcPoolUuid, srcPoolType); + } + migrationOptions.setTimeout(StorageManager.KvmStorageOnlineMigrationWait.value()); + destVolumeInfo.setMigrationOptions(migrationOptions); } /** @@ -2007,6 +2008,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { Map srcVolumeInfoToDestVolumeInfo = new HashMap<>(); boolean managedStorageDestination = false; + boolean migrateNonSharedInc = false; for (Map.Entry entry : volumeDataStoreMap.entrySet()) { VolumeInfo srcVolumeInfo = entry.getKey(); DataStore destDataStore = entry.getValue(); @@ -2024,15 +2026,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { continue; } - VMTemplateVO vmTemplate = _vmTemplateDao.findById(vmInstance.getTemplateId()); - if (srcVolumeInfo.getTemplateId() != null && - Objects.nonNull(vmTemplate) && - !Arrays.asList(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME, VM_IMPORT_DEFAULT_TEMPLATE_NAME).contains(vmTemplate.getName())) { - logger.debug(String.format("Copying template [%s] of volume [%s] from source storage pool [%s] to target storage pool [%s].", srcVolumeInfo.getTemplateId(), srcVolumeInfo.getId(), sourceStoragePool.getId(), destStoragePool.getId())); - copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, sourceStoragePool, destDataStore, destStoragePool, destHost); - } else { - logger.debug(String.format("Skipping copy template from source storage pool [%s] to target storage pool [%s] before migration due to volume [%s] does not have a template.", sourceStoragePool.getId(), destStoragePool.getId(), srcVolumeInfo.getId())); - } + MigrationOptions.Type migrationType = decideMigrationTypeAndCopyTemplateIfNeeded(destHost, vmInstance, srcVolumeInfo, sourceStoragePool, destStoragePool, destDataStore); + migrateNonSharedInc = migrateNonSharedInc || MigrationOptions.Type.LinkedClone.equals(migrationType); VolumeVO destVolume = duplicateVolumeOnAnotherStorage(srcVolume, destStoragePool); VolumeInfo destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); @@ -2044,7 +2039,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { // move the volume from Ready to Migrating destVolumeInfo.processEvent(Event.MigrationRequested); - setVolumeMigrationOptions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost, destStoragePool); + setVolumeMigrationOptions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost, destStoragePool, migrationType); // create a volume on the destination storage destDataStore.getDriver().createAsync(destDataStore, destVolumeInfo, null); @@ -2059,7 +2054,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { _volumeDao.update(destVolume.getId(), destVolume); - postVolumeCreationActions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost); + postVolumeCreationActions(srcVolumeInfo, destVolumeInfo); destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); @@ -2083,8 +2078,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath, backingPath); migrateDiskInfo.setSourceDiskOnStorageFileSystem(isStoragePoolTypeOfFile(sourceStoragePool)); migrateDiskInfoList.add(migrateDiskInfo); - prepareDiskWithSecretConsumerDetail(vmTO, srcVolumeInfo, destVolumeInfo.getPath()); } + prepareDiskWithSecretConsumerDetail(vmTO, srcVolumeInfo, destVolumeInfo.getPath()); migrateStorage.put(srcVolumeInfo.getPath(), migrateDiskInfo); @@ -2110,8 +2105,6 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { VMInstanceVO vm = _vmDao.findById(vmTO.getId()); boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - boolean migrateNonSharedInc = isSourceAndDestinationPoolTypeOfNfs(volumeDataStoreMap); - MigrateCommand migrateCommand = new MigrateCommand(vmTO.getName(), destHost.getPrivateIpAddress(), isWindows, vmTO, true); migrateCommand.setWait(StorageManager.KvmStorageOnlineMigrationWait.value()); migrateCommand.setMigrateStorage(migrateStorage); @@ -2161,6 +2154,22 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } } + private MigrationOptions.Type decideMigrationTypeAndCopyTemplateIfNeeded(Host destHost, VMInstanceVO vmInstance, VolumeInfo srcVolumeInfo, StoragePoolVO sourceStoragePool, StoragePoolVO destStoragePool, DataStore destDataStore) { + VMTemplateVO vmTemplate = _vmTemplateDao.findById(vmInstance.getTemplateId()); + String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo); + if (StringUtils.isNotBlank(srcVolumeBackingFile) && supportStoragePoolType(destStoragePool.getPoolType(), StoragePoolType.Filesystem) && + srcVolumeInfo.getTemplateId() != null && + Objects.nonNull(vmTemplate) && + !Arrays.asList(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME, VM_IMPORT_DEFAULT_TEMPLATE_NAME).contains(vmTemplate.getName())) { + logger.debug(String.format("Copying template [%s] of volume [%s] from source storage pool [%s] to target storage pool [%s].", srcVolumeInfo.getTemplateId(), srcVolumeInfo.getId(), sourceStoragePool.getId(), destStoragePool.getId())); + copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, sourceStoragePool, destDataStore, destStoragePool, destHost); + return MigrationOptions.Type.LinkedClone; + } + logger.debug(String.format("Skipping copy template from source storage pool [%s] to target storage pool [%s] before migration due to volume [%s] does not have a " + + "template or we are doing full clone migration.", sourceStoragePool.getId(), destStoragePool.getId(), srcVolumeInfo.getId())); + return MigrationOptions.Type.FullClone; + } + protected String formatMigrationElementsAsJsonToDisplayOnLog(String objectName, Object object, Object from, Object to){ return String.format("{%s: \"%s\", from: \"%s\", to:\"%s\"}", objectName, object, from, to); } @@ -2422,7 +2431,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { /** * Handle post destination volume creation actions depending on the migrating volume type: full clone or linked clone */ - protected void postVolumeCreationActions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, VirtualMachineTO vmTO, Host srcHost) { + protected void postVolumeCreationActions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { MigrationOptions migrationOptions = destVolumeInfo.getMigrationOptions(); if (migrationOptions != null) { if (migrationOptions.getType() == MigrationOptions.Type.LinkedClone && migrationOptions.isCopySrcTemplate()) { @@ -2470,7 +2479,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException("Destination storage pool with ID " + dataStore.getId() + " was not located."); } - if (srcStoragePoolVO.isManaged() && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) { + boolean isSrcAndDestPoolPowerFlexStorage = srcStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex) && destStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex); + if (srcStoragePoolVO.isManaged() && !isSrcAndDestPoolPowerFlexStorage && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) { throw new CloudRuntimeException("Migrating a volume online with KVM from managed storage is not currently supported."); } diff --git a/engine/storage/image/pom.xml b/engine/storage/image/pom.xml index 278b3672b2f..dc4eaa42229 100644 --- a/engine/storage/image/pom.xml +++ b/engine/storage/image/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 5e21f37f4d5..abc955c2e49 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -533,6 +533,11 @@ public class TemplateServiceImpl implements TemplateService { logger.info("Skip downloading template " + tmplt.getUniqueName() + " since no url is specified."); continue; } + // if this is private template, skip sync to a new image store + if (isSkipTemplateStoreDownload(tmplt, zoneId)) { + logger.info("Skip sync downloading private template " + tmplt.getUniqueName() + " to a new image store"); + continue; + } // if this is a region store, and there is already an DOWNLOADED entry there without install_path information, which // means that this is a duplicate entry from migration of previous NFS to staging. diff --git a/engine/storage/integration-test/pom.xml b/engine/storage/integration-test/pom.xml index a5bc225f4f6..779981f4c31 100644 --- a/engine/storage/integration-test/pom.xml +++ b/engine/storage/integration-test/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/storage/object/pom.xml b/engine/storage/object/pom.xml index 7159a646fbb..fcc73e40968 100644 --- a/engine/storage/object/pom.xml +++ b/engine/storage/object/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/storage/pom.xml b/engine/storage/pom.xml index e16e88e235d..1289a5f74a6 100644 --- a/engine/storage/pom.xml +++ b/engine/storage/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/engine/storage/snapshot/pom.xml b/engine/storage/snapshot/pom.xml index f29b43d8de0..e6474b5a6d4 100644 --- a/engine/storage/snapshot/pom.xml +++ b/engine/storage/snapshot/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/storage/volume/pom.xml b/engine/storage/volume/pom.xml index a00c3314126..549cf7b8ef4 100644 --- a/engine/storage/volume/pom.xml +++ b/engine/storage/volume/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/userdata/cloud-init/pom.xml b/engine/userdata/cloud-init/pom.xml index d4396ba382a..6b9db2f287b 100644 --- a/engine/userdata/cloud-init/pom.xml +++ b/engine/userdata/cloud-init/pom.xml @@ -23,7 +23,7 @@ cloud-engine org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/engine/userdata/pom.xml b/engine/userdata/pom.xml index 038aa18f290..e742b20e2e2 100644 --- a/engine/userdata/pom.xml +++ b/engine/userdata/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/agent-lb/pom.xml b/framework/agent-lb/pom.xml index 50e0bd47b90..b1fa2c8c67c 100644 --- a/framework/agent-lb/pom.xml +++ b/framework/agent-lb/pom.xml @@ -24,7 +24,7 @@ cloudstack-framework org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/ca/pom.xml b/framework/ca/pom.xml index d82389cd008..f1bd1dc6461 100644 --- a/framework/ca/pom.xml +++ b/framework/ca/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/cluster/pom.xml b/framework/cluster/pom.xml index ef511584ae6..67d46c1464c 100644 --- a/framework/cluster/pom.xml +++ b/framework/cluster/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/config/pom.xml b/framework/config/pom.xml index fc3b14642f1..f44bc5c1fce 100644 --- a/framework/config/pom.xml +++ b/framework/config/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/db/pom.xml b/framework/db/pom.xml index 586d72f34f3..03e1101628b 100644 --- a/framework/db/pom.xml +++ b/framework/db/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java index 84750c2068c..de8838b0999 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java @@ -247,14 +247,6 @@ public interface GenericDao { int expungeList(List ids); - /** - * Delete the entity beans specified by the search criteria with a given limit - * @param sc Search criteria - * @param limit Maximum number of rows that will be affected - * @return Number of rows deleted - */ - int expunge(SearchCriteria sc, long limit); - /** * expunge the removed rows. */ diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index 52a6b204ee8..82fea9749ff 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -1244,13 +1244,6 @@ public abstract class GenericDaoBase extends Compone } } - // FIXME: Does not work for joins. - @Override - public int expunge(final SearchCriteria sc, long limit) { - Filter filter = new Filter(limit); - return expunge(sc, filter); - } - @Override public int expunge(final SearchCriteria sc, final Filter filter) { if (sc == null) { @@ -1400,22 +1393,39 @@ public abstract class GenericDaoBase extends Compone onClause.append("?"); joinAttrList.add(join.getFirstAttributes()[i]); } else { - onClause.append(joinedTableNames.getOrDefault(join.getFirstAttributes()[i].table, join.getFirstAttributes()[i].table)) - .append(".") - .append(join.getFirstAttributes()[i].columnName); - } - onClause.append("="); - if (join.getSecondAttribute()[i].getValue() != null) { - onClause.append("?"); - joinAttrList.add(join.getSecondAttribute()[i]); - } else { - if(!joinTableAlias.equals(joinTableName)) { - onClause.append(joinTableAlias); + if ((join.getFirstAttributes()[i].table == null && join.getFirstAttributes()[i].value == null) || + (join.getSecondAttribute()[i].table == null && join.getSecondAttribute()[i].value == null)) { + onClause.append(joinedTableNames.getOrDefault(join.getSecondAttribute()[i].table, join.getFirstAttributes()[i].table)) + .append("."); + if (join.getFirstAttributes()[i].table == null && join.getFirstAttributes()[i].value == null) { + onClause.append(join.getSecondAttribute()[i].columnName); + } else { + onClause.append(join.getFirstAttributes()[i].columnName); + } + } else { - onClause.append(joinTableName); + onClause.append(joinedTableNames.getOrDefault(join.getFirstAttributes()[i].table, join.getFirstAttributes()[i].table)) + .append(".") + .append(join.getFirstAttributes()[i].columnName); + } + } + if ((join.getFirstAttributes()[i].table == null && join.getFirstAttributes()[i].value == null) || + (join.getSecondAttribute()[i].table == null && join.getSecondAttribute()[i].value == null)) { + onClause.append(" IS NULL"); + } else { + onClause.append("="); + if (join.getSecondAttribute()[i].getValue() != null) { + onClause.append("?"); + joinAttrList.add(join.getSecondAttribute()[i]); + } else { + if (!joinTableAlias.equals(joinTableName)) { + onClause.append(joinTableAlias); + } else { + onClause.append(joinTableName); + } + onClause.append(".") + .append(join.getSecondAttribute()[i].columnName); } - onClause.append(".") - .append(join.getSecondAttribute()[i].columnName); } } onClause.append(" "); diff --git a/framework/direct-download/pom.xml b/framework/direct-download/pom.xml index 1915377f222..0ff8b6b6610 100644 --- a/framework/direct-download/pom.xml +++ b/framework/direct-download/pom.xml @@ -32,7 +32,7 @@ cloudstack-framework org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/events/pom.xml b/framework/events/pom.xml index 3f457920cc9..5a707390c92 100644 --- a/framework/events/pom.xml +++ b/framework/events/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/ipc/pom.xml b/framework/ipc/pom.xml index 3c03ed04e28..fe3211420fc 100644 --- a/framework/ipc/pom.xml +++ b/framework/ipc/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/jobs/pom.xml b/framework/jobs/pom.xml index a82f514635f..cd46a1bff9d 100644 --- a/framework/jobs/pom.xml +++ b/framework/jobs/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/managed-context/pom.xml b/framework/managed-context/pom.xml index bc7fa17940b..ff9d6f15306 100644 --- a/framework/managed-context/pom.xml +++ b/framework/managed-context/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/framework/pom.xml b/framework/pom.xml index 79b1036eae7..49ff575ac1b 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/framework/quota/pom.xml b/framework/quota/pom.xml index 2e608d7a248..ca8b6f64afd 100644 --- a/framework/quota/pom.xml +++ b/framework/quota/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/rest/pom.xml b/framework/rest/pom.xml index d1ffff3c7bd..a61fe7d9a95 100644 --- a/framework/rest/pom.xml +++ b/framework/rest/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml cloud-framework-rest diff --git a/framework/security/pom.xml b/framework/security/pom.xml index f41d5460bb7..74251bc6edc 100644 --- a/framework/security/pom.xml +++ b/framework/security/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/framework/spring/lifecycle/pom.xml b/framework/spring/lifecycle/pom.xml index fbdb2e60dc6..662ea2476a6 100644 --- a/framework/spring/lifecycle/pom.xml +++ b/framework/spring/lifecycle/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/framework/spring/module/pom.xml b/framework/spring/module/pom.xml index ea39e3a6141..03d3d0d5503 100644 --- a/framework/spring/module/pom.xml +++ b/framework/spring/module/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index a88d4b1cbbf..f9d6fd1ce87 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -102,6 +102,7 @@ Requires: java-17-openjdk Requires: tzdata-java Requires: %{name}-common = %{_ver} Requires: libvirt +Requires: libvirt-daemon-driver-storage-rbd Requires: ebtables Requires: iptables Requires: ethtool diff --git a/plugins/acl/dynamic-role-based/pom.xml b/plugins/acl/dynamic-role-based/pom.xml index b1972a54ba5..5179cb0af1c 100644 --- a/plugins/acl/dynamic-role-based/pom.xml +++ b/plugins/acl/dynamic-role-based/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/acl/project-role-based/pom.xml b/plugins/acl/project-role-based/pom.xml index 3f5d64d29a7..f121080a20d 100644 --- a/plugins/acl/project-role-based/pom.xml +++ b/plugins/acl/project-role-based/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/acl/static-role-based/pom.xml b/plugins/acl/static-role-based/pom.xml index 62fb60395e0..0c4cf96264c 100644 --- a/plugins/acl/static-role-based/pom.xml +++ b/plugins/acl/static-role-based/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/affinity-group-processors/explicit-dedication/pom.xml b/plugins/affinity-group-processors/explicit-dedication/pom.xml index d6827ee13b6..0cb9db8fdf9 100644 --- a/plugins/affinity-group-processors/explicit-dedication/pom.xml +++ b/plugins/affinity-group-processors/explicit-dedication/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/affinity-group-processors/host-affinity/pom.xml b/plugins/affinity-group-processors/host-affinity/pom.xml index bd999288717..e51159f310b 100644 --- a/plugins/affinity-group-processors/host-affinity/pom.xml +++ b/plugins/affinity-group-processors/host-affinity/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/affinity-group-processors/host-anti-affinity/pom.xml b/plugins/affinity-group-processors/host-anti-affinity/pom.xml index b224bbaf34a..e81c9822952 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/pom.xml +++ b/plugins/affinity-group-processors/host-anti-affinity/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml b/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml index bf751ca4141..8a919f9a32e 100644 --- a/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml +++ b/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml b/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml index 445acfc11d6..91f56279686 100644 --- a/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml +++ b/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml @@ -32,7 +32,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/alert-handlers/snmp-alerts/pom.xml b/plugins/alert-handlers/snmp-alerts/pom.xml index fad47d426f2..5bfe59071b1 100644 --- a/plugins/alert-handlers/snmp-alerts/pom.xml +++ b/plugins/alert-handlers/snmp-alerts/pom.xml @@ -24,7 +24,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/alert-handlers/syslog-alerts/pom.xml b/plugins/alert-handlers/syslog-alerts/pom.xml index 54641bd8a8a..b890831fcac 100644 --- a/plugins/alert-handlers/syslog-alerts/pom.xml +++ b/plugins/alert-handlers/syslog-alerts/pom.xml @@ -24,7 +24,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/api/discovery/pom.xml b/plugins/api/discovery/pom.xml index 6426dcd70a5..12a74db045e 100644 --- a/plugins/api/discovery/pom.xml +++ b/plugins/api/discovery/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/api/rate-limit/pom.xml b/plugins/api/rate-limit/pom.xml index 2449a23f2d0..862e91197bf 100644 --- a/plugins/api/rate-limit/pom.xml +++ b/plugins/api/rate-limit/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/api/solidfire-intg-test/pom.xml b/plugins/api/solidfire-intg-test/pom.xml index 907c5f2968d..490f9e5fdaf 100644 --- a/plugins/api/solidfire-intg-test/pom.xml +++ b/plugins/api/solidfire-intg-test/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/api/vmware-sioc/pom.xml b/plugins/api/vmware-sioc/pom.xml index b3c04e603cb..0cc21482fa5 100644 --- a/plugins/api/vmware-sioc/pom.xml +++ b/plugins/api/vmware-sioc/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/backup/dummy/pom.xml b/plugins/backup/dummy/pom.xml index 52fbd085eb2..b47bdf6f5ef 100644 --- a/plugins/backup/dummy/pom.xml +++ b/plugins/backup/dummy/pom.xml @@ -23,7 +23,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/backup/nas/pom.xml b/plugins/backup/nas/pom.xml index 096bf45c67e..cfc4ff3e8fc 100644 --- a/plugins/backup/nas/pom.xml +++ b/plugins/backup/nas/pom.xml @@ -25,7 +25,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/backup/networker/pom.xml b/plugins/backup/networker/pom.xml index 1124d281d2e..430867f7acf 100644 --- a/plugins/backup/networker/pom.xml +++ b/plugins/backup/networker/pom.xml @@ -25,7 +25,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/backup/veeam/pom.xml b/plugins/backup/veeam/pom.xml index a317f90388a..5144989cace 100644 --- a/plugins/backup/veeam/pom.xml +++ b/plugins/backup/veeam/pom.xml @@ -23,7 +23,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/ca/root-ca/pom.xml b/plugins/ca/root-ca/pom.xml index fe1fb006302..dd3df448125 100644 --- a/plugins/ca/root-ca/pom.xml +++ b/plugins/ca/root-ca/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/database/mysql-ha/pom.xml b/plugins/database/mysql-ha/pom.xml index 37a07784341..928eb1756d6 100644 --- a/plugins/database/mysql-ha/pom.xml +++ b/plugins/database/mysql-ha/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/database/quota/pom.xml b/plugins/database/quota/pom.xml index b574b263020..63b4e45932d 100644 --- a/plugins/database/quota/pom.xml +++ b/plugins/database/quota/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index 218e3c2b2f9..628eca56c2d 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@ -21,6 +21,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.user.Account; + +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; @@ -40,6 +43,7 @@ public class QuotaBalanceCmd extends BaseCmd { @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which statement needs to be generated") private String accountName; + @ACL @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.") private Long domainId; @@ -51,6 +55,7 @@ public class QuotaBalanceCmd extends BaseCmd { ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; + @ACL @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account") private Long accountId; @@ -104,7 +109,14 @@ public class QuotaBalanceCmd extends BaseCmd { @Override public long getEntityOwnerId() { - return _accountService.getActiveAccountByName(accountName, domainId).getAccountId(); + if (accountId != null) { + return accountId; + } + Account account = _accountService.getActiveAccountByName(accountName, domainId); + if (account != null) { + return account.getAccountId(); + } + return Account.ACCOUNT_ID_SYSTEM; } @Override diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaCreditsCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaCreditsCmd.java index 8ca29f275dd..cfa391291c8 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaCreditsCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaCreditsCmd.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command; import com.cloud.user.Account; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -46,6 +47,7 @@ public class QuotaCreditsCmd extends BaseCmd { @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which quota credits need to be added") private String accountName; + @ACL @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Domain for which quota credits need to be added") private Long domainId; @@ -130,6 +132,10 @@ public class QuotaCreditsCmd extends BaseCmd { @Override public long getEntityOwnerId() { + Account account = _accountService.getActiveAccountByName(accountName, domainId); + if (account != null) { + return account.getAccountId(); + } return Account.ACCOUNT_ID_SYSTEM; } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index 4fb33f79672..78fa0f7df84 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -21,6 +21,7 @@ import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; @@ -42,6 +43,7 @@ public class QuotaStatementCmd extends BaseCmd { @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Optional, Account Id for which statement needs to be generated") private String accountName; + @ACL @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") private Long domainId; @@ -56,6 +58,7 @@ public class QuotaStatementCmd extends BaseCmd { @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type") private Integer usageType; + @ACL @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified account") private Long accountId; @@ -112,6 +115,9 @@ public class QuotaStatementCmd extends BaseCmd { @Override public long getEntityOwnerId() { + if (accountId != null) { + return accountId; + } Account activeAccountByName = _accountService.getActiveAccountByName(accountName, domainId); if (activeAccountByName != null) { return activeAccountByName.getAccountId(); diff --git a/plugins/dedicated-resources/pom.xml b/plugins/dedicated-resources/pom.xml index 5aeecec818d..afb3b937350 100644 --- a/plugins/dedicated-resources/pom.xml +++ b/plugins/dedicated-resources/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/plugins/deployment-planners/implicit-dedication/pom.xml b/plugins/deployment-planners/implicit-dedication/pom.xml index b9f8be5d08a..dc0ff5f6d28 100644 --- a/plugins/deployment-planners/implicit-dedication/pom.xml +++ b/plugins/deployment-planners/implicit-dedication/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/deployment-planners/user-concentrated-pod/pom.xml b/plugins/deployment-planners/user-concentrated-pod/pom.xml index 0dcbe3506c6..495aec4405a 100644 --- a/plugins/deployment-planners/user-concentrated-pod/pom.xml +++ b/plugins/deployment-planners/user-concentrated-pod/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/deployment-planners/user-dispersing/pom.xml b/plugins/deployment-planners/user-dispersing/pom.xml index bbd74bf1a7a..2a514cdd66a 100644 --- a/plugins/deployment-planners/user-dispersing/pom.xml +++ b/plugins/deployment-planners/user-dispersing/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/drs/cluster/balanced/pom.xml b/plugins/drs/cluster/balanced/pom.xml index 743a5f2bc98..b7ea52d976d 100644 --- a/plugins/drs/cluster/balanced/pom.xml +++ b/plugins/drs/cluster/balanced/pom.xml @@ -27,7 +27,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/drs/cluster/condensed/pom.xml b/plugins/drs/cluster/condensed/pom.xml index 60b472bccb5..b15509fdcbb 100644 --- a/plugins/drs/cluster/condensed/pom.xml +++ b/plugins/drs/cluster/condensed/pom.xml @@ -27,7 +27,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/event-bus/inmemory/pom.xml b/plugins/event-bus/inmemory/pom.xml index be85e8afd8d..8e74ffefd5c 100644 --- a/plugins/event-bus/inmemory/pom.xml +++ b/plugins/event-bus/inmemory/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/event-bus/kafka/pom.xml b/plugins/event-bus/kafka/pom.xml index 44014847548..6c9f191a9db 100644 --- a/plugins/event-bus/kafka/pom.xml +++ b/plugins/event-bus/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/event-bus/rabbitmq/pom.xml b/plugins/event-bus/rabbitmq/pom.xml index 1e04caf026f..f1e155a6000 100644 --- a/plugins/event-bus/rabbitmq/pom.xml +++ b/plugins/event-bus/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/event-bus/webhook/pom.xml b/plugins/event-bus/webhook/pom.xml index 278f4dc0ec5..39e3cc3c098 100644 --- a/plugins/event-bus/webhook/pom.xml +++ b/plugins/event-bus/webhook/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/ha-planners/skip-heurestics/pom.xml b/plugins/ha-planners/skip-heurestics/pom.xml index 3dc3989da2c..b716f00ff18 100644 --- a/plugins/ha-planners/skip-heurestics/pom.xml +++ b/plugins/ha-planners/skip-heurestics/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml index f01494914ce..2ef9740000e 100644 --- a/plugins/host-allocators/random/pom.xml +++ b/plugins/host-allocators/random/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/hypervisors/baremetal/pom.xml b/plugins/hypervisors/baremetal/pom.xml index d866c9b47e2..398735f1bdd 100755 --- a/plugins/hypervisors/baremetal/pom.xml +++ b/plugins/hypervisors/baremetal/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml cloud-plugin-hypervisor-baremetal diff --git a/plugins/hypervisors/hyperv/pom.xml b/plugins/hypervisors/hyperv/pom.xml index 56b2b6d1503..fa1238229dd 100644 --- a/plugins/hypervisors/hyperv/pom.xml +++ b/plugins/hypervisors/hyperv/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/hypervisors/kvm/pom.xml b/plugins/hypervisors/kvm/pom.xml index f1eb4ea8c4f..31ba66da992 100644 --- a/plugins/hypervisors/kvm/pom.xml +++ b/plugins/hypervisors/kvm/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index abd6e24a532..0b0416a8049 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -381,6 +381,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected static final String DEFAULT_TUNGSTEN_VIF_DRIVER_CLASS_NAME = "com.cloud.hypervisor.kvm.resource.VRouterVifDriver"; private final static long HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IO_URING = 6003000; private final static long HYPERVISOR_QEMU_VERSION_SUPPORTS_IO_URING = 5000000; + private final static long HYPERVISOR_QEMU_VERSION_IDE_DISCARD_FIXED = 7000000; protected HypervisorType hypervisorType; protected String hypervisorURI; @@ -3050,7 +3051,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } else if (volume.getType() != Volume.Type.ISO) { final PrimaryDataStoreTO store = (PrimaryDataStoreTO)data.getDataStore(); - physicalDisk = storagePoolManager.getPhysicalDisk(store.getPoolType(), store.getUuid(), data.getPath()); + physicalDisk = getStoragePoolMgr().getPhysicalDisk(store.getPoolType(), store.getUuid(), data.getPath()); pool = physicalDisk.getPool(); } @@ -3152,6 +3153,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv else { disk.defBlockBasedDisk(physicalDisk.getPath(), devId, diskBusType); } + if (pool.getType() == StoragePoolType.Linstor && isQemuDiscardBugFree(diskBusType)) { + disk.setDiscard(DiscardType.UNMAP); + } } else { if (volume.getType() == Volume.Type.DATADISK && !(isWindowsTemplate && isUefiEnabled)) { disk.defFileBasedDisk(physicalDisk.getPath(), devId, diskBusTypeData, DiskDef.DiskFmtType.QCOW2); @@ -3296,6 +3300,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return isUbuntuHost() || isIoUringSupportedByQemu(); } + /** + * Qemu has a bug with discard enabled on IDE bus devices if qemu version < 7.0. + * redhat bug entry + * @param diskBus used for the disk + * @return true if it is safe to enable discard, otherwise false. + */ + public boolean isQemuDiscardBugFree(DiskDef.DiskBus diskBus) { + return diskBus != DiskDef.DiskBus.IDE || getHypervisorQemuVersion() >= HYPERVISOR_QEMU_VERSION_IDE_DISCARD_FIXED; + } + public boolean isUbuntuHost() { Map versionString = getVersionStrings(); String hostKey = "Host.OS"; @@ -3498,6 +3512,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, busT, DiskDef.DiskFmtType.QCOW2); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) { diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, busT); + if (attachingPool.getType() == StoragePoolType.Linstor) { + diskdef.setDiscard(DiscardType.UNMAP); + } } if (bytesReadRate != null && bytesReadRate > 0) { diskdef.setBytesReadRate(bytesReadRate); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java index 09ee45d5908..4f8b2843ec2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java @@ -24,7 +24,7 @@ import org.apache.commons.collections.CollectionUtils; public class LibvirtStoragePoolDef { public enum PoolType { - ISCSI("iscsi"), NETFS("netfs"), loggerICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"), POWERFLEX("powerflex"); + ISCSI("iscsi"), NETFS("netfs"), LOGICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"), POWERFLEX("powerflex"); String _poolType; PoolType(String poolType) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java index ae8d06a1986..8f027e01ca4 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java @@ -123,8 +123,10 @@ public class MigrateKVMAsync implements Callable { if (migrateNonSharedInc) { flags |= VIR_MIGRATE_PERSIST_DEST; flags |= VIR_MIGRATE_NON_SHARED_INC; + logger.debug("Setting VIR_MIGRATE_NON_SHARED_INC for linked clone migration."); } else { flags |= VIR_MIGRATE_NON_SHARED_DISK; + logger.debug("Setting VIR_MIGRATE_NON_SHARED_DISK for full clone migration."); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java index 923c44f4828..9ecdfddc2ad 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapper.java @@ -89,7 +89,7 @@ public final class LibvirtOvsFetchInterfaceCommandWrapper extends CommandWrapper @Override public Answer execute(final OvsFetchInterfaceCommand command, final LibvirtComputingResource libvirtComputingResource) { - final String label = "'" + command.getLabel() + "'"; + final String label = command.getLabel(); logger.debug("Will look for network with name-label:" + label); try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java index f023457461e..09cdad64ace 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java @@ -43,7 +43,7 @@ public class IscsiAdmStorageAdaptor implements StorageAdaptor { private static final Map MapStorageUuidToStoragePool = new HashMap<>(); @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details, boolean isPrimaryStorage) { IscsiAdmStoragePool storagePool = new IscsiAdmStoragePool(uuid, host, port, storagePoolType, this); MapStorageUuidToStoragePool.put(uuid, storagePool); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 3c8026c7ffd..4e76030d06f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -389,7 +389,7 @@ public class KVMStoragePoolManager { //Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in parallel, so need to be synced. private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); - KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details); + KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 04662604382..ffffe877e6e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -26,9 +26,11 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -63,15 +65,19 @@ import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import org.apache.cloudstack.storage.command.SyncVolumePathCommand; +import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.BooleanUtils; @@ -80,6 +86,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; + import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -134,10 +141,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.VmDetailConstants; -import org.apache.cloudstack.utils.cryptsetup.KeyFile; -import org.apache.cloudstack.utils.qemu.QemuImageOptions; -import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat; -import java.util.ArrayList; + public class KVMStorageProcessor implements StorageProcessor { protected Logger logger = LogManager.getLogger(getClass()); @@ -1447,6 +1451,9 @@ public class KVMStorageProcessor implements StorageProcessor { diskdef.defFileBasedDisk(attachingDisk.getPath(), devId, busT, DiskDef.DiskFmtType.QCOW2); } else if (attachingDisk.getFormat() == PhysicalDiskFormat.RAW) { diskdef.defBlockBasedDisk(attachingDisk.getPath(), devId, busT); + if (attachingPool.getType() == StoragePoolType.Linstor && resource.isQemuDiscardBugFree(busT)) { + diskdef.setDiscard(DiscardType.UNMAP); + } } if (encryptDetails != null) { @@ -2449,6 +2456,22 @@ public class KVMStorageProcessor implements StorageProcessor { template = storagePoolMgr.createPhysicalDiskFromDirectDownloadTemplate(tempFilePath, destTemplatePath, destPool, cmd.getFormat(), cmd.getWaitInMillSeconds()); + String templatePath = template.getPath(); + if (templatePath != null) { + try { + Qcow2Inspector.validateQcow2File(templatePath); + } catch (RuntimeException e) { + try { + Files.deleteIfExists(Path.of(templatePath)); + } catch (IOException ioException) { + logger.warn("Unable to remove file [{}]; consider removing it manually.", templatePath, ioException); + } + + logger.error("The downloaded file [{}] is not a valid QCOW2.", templatePath, e); + return new DirectDownloadAnswer(false, "The downloaded file is not a valid QCOW2. Ask the administrator to check the logs for more details.", true); + } + } + if (!storagePoolMgr.disconnectPhysicalDisk(pool.getPoolType(), pool.getUuid(), destTemplatePath)) { logger.warn("Unable to disconnect physical disk at path: " + destTemplatePath + ", in storage pool id: " + pool.getUuid()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 45a65037340..c91c001858d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -73,6 +73,7 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -81,6 +82,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private StorageLayer _storageLayer; private String _mountPoint = "/mnt"; private String _manageSnapshotPath; + private static final ConcurrentHashMap storagePoolRefCounts = new ConcurrentHashMap<>(); private String rbdTemplateSnapName = "cloudstack-base-snap"; private static final int RBD_FEATURE_LAYERING = 1; @@ -350,7 +352,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { String volgroupName = path; volgroupName = volgroupName.replaceFirst("/", ""); - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.loggerICAL, volgroupName, uuid, host, volgroupPath, volgroupPath); + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, volgroupPath); StoragePool sp = null; try { logger.debug(spd.toString()); @@ -542,7 +544,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { type = StoragePoolType.Filesystem; } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.RBD) { type = StoragePoolType.RBD; - } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.loggerICAL) { + } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.LOGICAL) { type = StoragePoolType.CLVM; } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.GLUSTERFS) { type = StoragePoolType.Gluster; @@ -644,8 +646,44 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } } + /** + * adjust refcount + */ + private int adjustStoragePoolRefCount(String uuid, int adjustment) { + final String mutexKey = storagePoolRefCounts.keySet().stream() + .filter(k -> k.equals(uuid)) + .findFirst() + .orElse(uuid); + synchronized (mutexKey) { + // some access on the storagePoolRefCounts.key(mutexKey) element + int refCount = storagePoolRefCounts.computeIfAbsent(mutexKey, k -> 0); + refCount += adjustment; + if (refCount < 1) { + storagePoolRefCounts.remove(mutexKey); + } else { + storagePoolRefCounts.put(mutexKey, refCount); + } + return refCount; + } + } + /** + * Thread-safe increment storage pool usage refcount + * @param uuid UUID of the storage pool to increment the count + */ + private void incStoragePoolRefCount(String uuid) { + adjustStoragePoolRefCount(uuid, 1); + } + /** + * Thread-safe decrement storage pool usage refcount for the given uuid and return if storage pool still in use. + * @param uuid UUID of the storage pool to decrement the count + * @return true if the storage pool is still used, else false. + */ + private boolean decStoragePoolRefCount(String uuid) { + return adjustStoragePoolRefCount(uuid, -1) > 0; + } + @Override - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage) { logger.info("Attempting to create storage pool " + name + " (" + type.toString() + ") in libvirt"); StoragePool sp = null; @@ -751,6 +789,12 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } try { + if (!isPrimaryStorage) { + // only ref count storage pools for secondary storage, as primary storage is assumed + // to be always mounted, as long the primary storage isn't fully deleted. + incStoragePoolRefCount(name); + } + if (sp.isActive() == 0) { logger.debug("Attempting to activate pool " + name); sp.create(0); @@ -762,6 +806,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return getStoragePool(name); } catch (LibvirtException e) { + decStoragePoolRefCount(name); String error = e.toString(); if (error.contains("Storage source conflict")) { throw new CloudRuntimeException("A pool matching this location already exists in libvirt, " + @@ -812,6 +857,13 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { @Override public boolean deleteStoragePool(String uuid) { logger.info("Attempting to remove storage pool " + uuid + " from libvirt"); + + // decrement and check if storage pool still in use + if (decStoragePoolRefCount(uuid)) { + logger.info(String.format("deleteStoragePool: Storage pool %s still in use", uuid)); + return true; + } + Connect conn = null; try { conn = LibvirtConnection.getConnection(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java index 8ec56b885f2..2304e1cbac5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java @@ -56,7 +56,7 @@ public class ManagedNfsStorageAdaptor implements StorageAdaptor { } @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details, boolean isPrimaryStorage) { LibvirtStoragePool storagePool = new LibvirtStoragePool(uuid, path, StoragePoolType.ManagedNFS, this, null); storagePool.setSourceHost(host); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java index 03acfcc89ad..7a827d9afc3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -169,7 +169,7 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { } @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { LOGGER.info(String.format("createStoragePool(uuid,host,port,path,type) called with args (%s, %s, %s, %s, %s)", uuid, host, ""+port, path, type)); MultipathSCSIPool storagePool = new MultipathSCSIPool(uuid, host, port, path, type, details, this); MapStorageUuidToStoragePool.put(uuid, storagePool); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index b33f49404ad..7477d768e9a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -146,7 +146,7 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { } @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { ScaleIOStoragePool storagePool = new ScaleIOStoragePool(uuid, host, port, path, type, details, this); MapStorageUuidToStoragePool.put(uuid, storagePool); return storagePool; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java index 9a27d44ff92..15b03a55a3e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java @@ -40,7 +40,7 @@ public interface StorageAdaptor { // it with info from local disk, and return it public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool); - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details); + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage); public boolean deleteStoragePool(String uuid); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index 30d0b2ab163..15c2b46e077 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -58,9 +58,12 @@ import javax.xml.xpath.XPathFactory; import com.cloud.utils.net.NetUtils; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.linux.CPUStat; import org.apache.cloudstack.utils.linux.MemStat; @@ -6415,4 +6418,114 @@ public class LibvirtComputingResourceTest { Assert.assertEquals(newStats.getDiskReadKBs() - oldStats.getDiskReadKBs(), metrics.getDiskReadKBs(), 0); Assert.assertEquals(newStats.getDiskWriteKBs() - oldStats.getDiskWriteKBs(), metrics.getDiskWriteKBs(), 0); } + + @Test + public void createLinstorVdb() throws LibvirtException, InternalErrorException, URISyntaxException { + final Connect connect = Mockito.mock(Connect.class); + + final int id = random.nextInt(65534); + final String name = "test-instance-1"; + + final int cpus = 2; + final int speed = 1024; + final int minRam = 256 * 1024; + final int maxRam = 512 * 1024; + final String os = "Ubuntu"; + final String vncPassword = "mySuperSecretPassword"; + + final VirtualMachineTO to = new VirtualMachineTO(id, name, VirtualMachine.Type.User, cpus, speed, minRam, + maxRam, BootloaderType.HVM, os, false, false, vncPassword); + to.setVncAddr(""); + to.setArch("x86_64"); + to.setUuid("b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9"); + to.setVcpuMaxLimit(cpus + 1); + final HashMap vmToDetails = new HashMap<>(); + to.setDetails(vmToDetails); + + String diskLinPath = "9ebe53c1-3d35-46e5-b7aa-6fc223ba0fcf"; + final DiskTO diskTO = new DiskTO(); + diskTO.setDiskSeq(1L); + diskTO.setType(Volume.Type.ROOT); + diskTO.setDetails(new HashMap<>()); + diskTO.setPath(diskLinPath); + + final PrimaryDataStoreTO primaryDataStoreTO = Mockito.mock(PrimaryDataStoreTO.class); + String pDSTOUUID = "9ebe53c1-3d35-46e5-b7aa-6fc223ac4fcf"; + when(primaryDataStoreTO.getPoolType()).thenReturn(StoragePoolType.Linstor); + when(primaryDataStoreTO.getUuid()).thenReturn(pDSTOUUID); + + VolumeObjectTO dataTO = new VolumeObjectTO(); + + dataTO.setUuid("12be53c1-3d35-46e5-b7aa-6fc223ba0f34"); + dataTO.setPath(diskTO.getPath()); + dataTO.setDataStore(primaryDataStoreTO); + diskTO.setData(dataTO); + to.setDisks(new DiskTO[]{diskTO}); + + String path = "/dev/drbd1020"; + final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class); + final KVMStoragePool storagePool = Mockito.mock(KVMStoragePool.class); + final KVMPhysicalDisk vol = Mockito.mock(KVMPhysicalDisk.class); + + when(libvirtComputingResourceSpy.getStoragePoolMgr()).thenReturn(storagePoolMgr); + when(storagePool.getType()).thenReturn(StoragePoolType.Linstor); + when(storagePoolMgr.getPhysicalDisk(StoragePoolType.Linstor, pDSTOUUID, diskLinPath)).thenReturn(vol); + when(vol.getPath()).thenReturn(path); + when(vol.getPool()).thenReturn(storagePool); + when(vol.getFormat()).thenReturn(PhysicalDiskFormat.RAW); + + // 1. test Bus: IDE and broken qemu version -> NO discard + when(libvirtComputingResourceSpy.getHypervisorQemuVersion()).thenReturn(6000000L); + vmToDetails.put(VmDetailConstants.ROOT_DISK_CONTROLLER, DiskDef.DiskBus.IDE.name()); + { + LibvirtVMDef vm = new LibvirtVMDef(); + vm.addComp(new DevicesDef()); + libvirtComputingResourceSpy.createVbd(connect, to, name, vm); + + DiskDef rootDisk = vm.getDevices().getDisks().get(0); + assertEquals(DiskDef.DiskType.BLOCK, rootDisk.getDiskType()); + assertEquals(DiskDef.DiskBus.IDE, rootDisk.getBusType()); + assertEquals(DiskDef.DiscardType.IGNORE, rootDisk.getDiscard()); + } + + // 2. test Bus: VIRTIO and broken qemu version -> discard unmap + vmToDetails.put(VmDetailConstants.ROOT_DISK_CONTROLLER, DiskDef.DiskBus.VIRTIO.name()); + { + LibvirtVMDef vm = new LibvirtVMDef(); + vm.addComp(new DevicesDef()); + libvirtComputingResourceSpy.createVbd(connect, to, name, vm); + + DiskDef rootDisk = vm.getDevices().getDisks().get(0); + assertEquals(DiskDef.DiskType.BLOCK, rootDisk.getDiskType()); + assertEquals(DiskDef.DiskBus.VIRTIO, rootDisk.getBusType()); + assertEquals(DiskDef.DiscardType.UNMAP, rootDisk.getDiscard()); + } + + // 3. test Bus; IDE and "good" qemu version -> discard unmap + vmToDetails.put(VmDetailConstants.ROOT_DISK_CONTROLLER, DiskDef.DiskBus.IDE.name()); + when(libvirtComputingResourceSpy.getHypervisorQemuVersion()).thenReturn(7000000L); + { + LibvirtVMDef vm = new LibvirtVMDef(); + vm.addComp(new DevicesDef()); + libvirtComputingResourceSpy.createVbd(connect, to, name, vm); + + DiskDef rootDisk = vm.getDevices().getDisks().get(0); + assertEquals(DiskDef.DiskType.BLOCK, rootDisk.getDiskType()); + assertEquals(DiskDef.DiskBus.IDE, rootDisk.getBusType()); + assertEquals(DiskDef.DiscardType.UNMAP, rootDisk.getDiscard()); + } + + // 4. test Bus: VIRTIO and "good" qemu version -> discard unmap + vmToDetails.put(VmDetailConstants.ROOT_DISK_CONTROLLER, DiskDef.DiskBus.VIRTIO.name()); + { + LibvirtVMDef vm = new LibvirtVMDef(); + vm.addComp(new DevicesDef()); + libvirtComputingResourceSpy.createVbd(connect, to, name, vm); + + DiskDef rootDisk = vm.getDevices().getDisks().get(0); + assertEquals(DiskDef.DiskType.BLOCK, rootDisk.getDiskType()); + assertEquals(DiskDef.DiskBus.VIRTIO, rootDisk.getBusType()); + assertEquals(DiskDef.DiscardType.UNMAP, rootDisk.getDiscard()); + } + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java index 3a7491ec542..c2bbff7efb0 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptorTest.java @@ -86,6 +86,6 @@ public class LibvirtStorageAdaptorTest { Map details = new HashMap<>(); details.put("nfsmountopts", "vers=4.1, nconnect=4"); - KVMStoragePool pool = libvirtStorageAdaptor.createStoragePool(uuid, null, 0, dir, null, Storage.StoragePoolType.NetworkFilesystem, details); + KVMStoragePool pool = libvirtStorageAdaptor.createStoragePool(uuid, null, 0, dir, null, Storage.StoragePoolType.NetworkFilesystem, details, true); } } diff --git a/plugins/hypervisors/ovm/pom.xml b/plugins/hypervisors/ovm/pom.xml index aad6d80f27d..95a2401b202 100644 --- a/plugins/hypervisors/ovm/pom.xml +++ b/plugins/hypervisors/ovm/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/hypervisors/ovm3/pom.xml b/plugins/hypervisors/ovm3/pom.xml index 31f761bde78..b90398c2593 100644 --- a/plugins/hypervisors/ovm3/pom.xml +++ b/plugins/hypervisors/ovm3/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/hypervisors/simulator/pom.xml b/plugins/hypervisors/simulator/pom.xml index e545f7cd658..2c64334e996 100644 --- a/plugins/hypervisors/simulator/pom.xml +++ b/plugins/hypervisors/simulator/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml cloud-plugin-hypervisor-simulator diff --git a/plugins/hypervisors/ucs/pom.xml b/plugins/hypervisors/ucs/pom.xml index bb9b02f4fb2..b69b141ae2c 100644 --- a/plugins/hypervisors/ucs/pom.xml +++ b/plugins/hypervisors/ucs/pom.xml @@ -23,7 +23,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml cloud-plugin-hypervisor-ucs diff --git a/plugins/hypervisors/vmware/pom.xml b/plugins/hypervisors/vmware/pom.xml index dac359b30a2..e0b51636a9e 100644 --- a/plugins/hypervisors/vmware/pom.xml +++ b/plugins/hypervisors/vmware/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/hypervisors/xenserver/pom.xml b/plugins/hypervisors/xenserver/pom.xml index ab70f893488..81670d4699f 100644 --- a/plugins/hypervisors/xenserver/pom.xml +++ b/plugins/hypervisors/xenserver/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/integrations/cloudian/pom.xml b/plugins/integrations/cloudian/pom.xml index 5529abe0154..5106ae53080 100644 --- a/plugins/integrations/cloudian/pom.xml +++ b/plugins/integrations/cloudian/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/integrations/kubernetes-service/pom.xml b/plugins/integrations/kubernetes-service/pom.xml index 397a3b43b31..c87b369fa48 100644 --- a/plugins/integrations/kubernetes-service/pom.xml +++ b/plugins/integrations/kubernetes-service/pom.xml @@ -26,7 +26,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/integrations/prometheus/pom.xml b/plugins/integrations/prometheus/pom.xml index 52c4a7ef2da..e4f092b5aaa 100644 --- a/plugins/integrations/prometheus/pom.xml +++ b/plugins/integrations/prometheus/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/metrics/pom.xml b/plugins/metrics/pom.xml index 8621dc3c71d..e3e766ef6b4 100644 --- a/plugins/metrics/pom.xml +++ b/plugins/metrics/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/plugins/network-elements/bigswitch/pom.xml b/plugins/network-elements/bigswitch/pom.xml index 18d06997878..006a4f4c379 100644 --- a/plugins/network-elements/bigswitch/pom.xml +++ b/plugins/network-elements/bigswitch/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/brocade-vcs/pom.xml b/plugins/network-elements/brocade-vcs/pom.xml index 1363658d179..d1d4bb57f92 100644 --- a/plugins/network-elements/brocade-vcs/pom.xml +++ b/plugins/network-elements/brocade-vcs/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/cisco-vnmc/pom.xml b/plugins/network-elements/cisco-vnmc/pom.xml index a40164be290..1fe618202d1 100644 --- a/plugins/network-elements/cisco-vnmc/pom.xml +++ b/plugins/network-elements/cisco-vnmc/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/dns-notifier/pom.xml b/plugins/network-elements/dns-notifier/pom.xml index 8d2ecaa30f2..8370417e90b 100644 --- a/plugins/network-elements/dns-notifier/pom.xml +++ b/plugins/network-elements/dns-notifier/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml cloud-plugin-example-dns-notifier diff --git a/plugins/network-elements/elastic-loadbalancer/pom.xml b/plugins/network-elements/elastic-loadbalancer/pom.xml index 9e2f395cd0a..b817de089e9 100644 --- a/plugins/network-elements/elastic-loadbalancer/pom.xml +++ b/plugins/network-elements/elastic-loadbalancer/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/globodns/pom.xml b/plugins/network-elements/globodns/pom.xml index c0200921f20..ca01c3b6f24 100644 --- a/plugins/network-elements/globodns/pom.xml +++ b/plugins/network-elements/globodns/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/internal-loadbalancer/pom.xml b/plugins/network-elements/internal-loadbalancer/pom.xml index f3126a8e024..abd983d2c2f 100644 --- a/plugins/network-elements/internal-loadbalancer/pom.xml +++ b/plugins/network-elements/internal-loadbalancer/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/juniper-contrail/pom.xml b/plugins/network-elements/juniper-contrail/pom.xml index 3f71146d580..dc3ee96328f 100644 --- a/plugins/network-elements/juniper-contrail/pom.xml +++ b/plugins/network-elements/juniper-contrail/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java index 6bb9752d764..3a5541654bb 100644 --- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java +++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java @@ -454,6 +454,11 @@ public class MockAccountManager extends ManagerBase implements AccountManager { // TODO Auto-generated method stub } + @Override + public void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource) { + // TODO Auto-generated method stub + } + @Override public Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly) { // TODO Auto-generated method stub diff --git a/plugins/network-elements/netscaler/pom.xml b/plugins/network-elements/netscaler/pom.xml index 15c3569a046..cb1d8f741e9 100644 --- a/plugins/network-elements/netscaler/pom.xml +++ b/plugins/network-elements/netscaler/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/nicira-nvp/pom.xml b/plugins/network-elements/nicira-nvp/pom.xml index 902a479af90..6b56fa8f7c9 100644 --- a/plugins/network-elements/nicira-nvp/pom.xml +++ b/plugins/network-elements/nicira-nvp/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/nsx/pom.xml b/plugins/network-elements/nsx/pom.xml index 9e8d6ee7bce..9a6fb90390f 100644 --- a/plugins/network-elements/nsx/pom.xml +++ b/plugins/network-elements/nsx/pom.xml @@ -26,7 +26,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/opendaylight/pom.xml b/plugins/network-elements/opendaylight/pom.xml index d8f3bcfc8eb..d6850206d38 100644 --- a/plugins/network-elements/opendaylight/pom.xml +++ b/plugins/network-elements/opendaylight/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/ovs/pom.xml b/plugins/network-elements/ovs/pom.xml index f59442e6e4d..ddbe99919cb 100644 --- a/plugins/network-elements/ovs/pom.xml +++ b/plugins/network-elements/ovs/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/palo-alto/pom.xml b/plugins/network-elements/palo-alto/pom.xml index c0d816b4a89..8950ad1a2b0 100644 --- a/plugins/network-elements/palo-alto/pom.xml +++ b/plugins/network-elements/palo-alto/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/stratosphere-ssp/pom.xml b/plugins/network-elements/stratosphere-ssp/pom.xml index 7e16b525dbb..53bfee839da 100644 --- a/plugins/network-elements/stratosphere-ssp/pom.xml +++ b/plugins/network-elements/stratosphere-ssp/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/tungsten/pom.xml b/plugins/network-elements/tungsten/pom.xml index 1cd1ae2d823..009673f264f 100644 --- a/plugins/network-elements/tungsten/pom.xml +++ b/plugins/network-elements/tungsten/pom.xml @@ -26,7 +26,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/network-elements/vxlan/pom.xml b/plugins/network-elements/vxlan/pom.xml index 34d7890bae9..39c77b14968 100644 --- a/plugins/network-elements/vxlan/pom.xml +++ b/plugins/network-elements/vxlan/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/outofbandmanagement-drivers/ipmitool/pom.xml b/plugins/outofbandmanagement-drivers/ipmitool/pom.xml index af7952a8a56..436dd8e3acb 100644 --- a/plugins/outofbandmanagement-drivers/ipmitool/pom.xml +++ b/plugins/outofbandmanagement-drivers/ipmitool/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/outofbandmanagement-drivers/nested-cloudstack/pom.xml b/plugins/outofbandmanagement-drivers/nested-cloudstack/pom.xml index 25d8e4054db..99f55414020 100644 --- a/plugins/outofbandmanagement-drivers/nested-cloudstack/pom.xml +++ b/plugins/outofbandmanagement-drivers/nested-cloudstack/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/outofbandmanagement-drivers/redfish/pom.xml b/plugins/outofbandmanagement-drivers/redfish/pom.xml index df19ef997b0..81f9a9f6b99 100644 --- a/plugins/outofbandmanagement-drivers/redfish/pom.xml +++ b/plugins/outofbandmanagement-drivers/redfish/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/pom.xml b/plugins/pom.xml index 3580976fa16..d2c8f5becee 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/plugins/shutdown/pom.xml b/plugins/shutdown/pom.xml index f995e5c82d2..44ac0459b9f 100644 --- a/plugins/shutdown/pom.xml +++ b/plugins/shutdown/pom.xml @@ -26,7 +26,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/plugins/storage-allocators/random/pom.xml b/plugins/storage-allocators/random/pom.xml index 4442f000e73..d0828854feb 100644 --- a/plugins/storage-allocators/random/pom.xml +++ b/plugins/storage-allocators/random/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/storage/image/default/pom.xml b/plugins/storage/image/default/pom.xml index fe91a9dbe96..3c4990975e1 100644 --- a/plugins/storage/image/default/pom.xml +++ b/plugins/storage/image/default/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/image/s3/pom.xml b/plugins/storage/image/s3/pom.xml index 185173d0fe6..6c3885338dd 100644 --- a/plugins/storage/image/s3/pom.xml +++ b/plugins/storage/image/s3/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/image/sample/pom.xml b/plugins/storage/image/sample/pom.xml index fbc9cec1440..49aef085f54 100644 --- a/plugins/storage/image/sample/pom.xml +++ b/plugins/storage/image/sample/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/image/swift/pom.xml b/plugins/storage/image/swift/pom.xml index 832972d3816..80329f00c9c 100644 --- a/plugins/storage/image/swift/pom.xml +++ b/plugins/storage/image/swift/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/object/ceph/pom.xml b/plugins/storage/object/ceph/pom.xml index 43b3a15731d..0bd5b7c431a 100644 --- a/plugins/storage/object/ceph/pom.xml +++ b/plugins/storage/object/ceph/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 6fece40e6ac..b2e1d23917b 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -46,8 +46,6 @@ import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; import org.apache.cloudstack.storage.object.BucketObject; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.twonote.rgwadmin4j.RgwAdmin; import org.twonote.rgwadmin4j.RgwAdminBuilder; import org.twonote.rgwadmin4j.model.BucketInfo; @@ -62,7 +60,6 @@ import java.util.Map; import java.util.HashMap; public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { - private static final Logger s_logger = LogManager.getLogger(CephObjectStoreDriverImpl.class); @Inject AccountDao _accountDao; @@ -168,7 +165,7 @@ public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { String policyConfig; if (policy.equalsIgnoreCase("public")) { - s_logger.debug("Setting public policy on bucket " + bucket.getName()); + logger.debug("Setting public policy on bucket " + bucket.getName()); StringBuilder builder = new StringBuilder(); builder.append("{\n"); builder.append(" \"Statement\": [\n"); @@ -192,7 +189,7 @@ public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { builder.append("}\n"); policyConfig = builder.toString(); } else { - s_logger.debug("Setting private policy on bucket " + bucket.getName()); + logger.debug("Setting private policy on bucket " + bucket.getName()); policyConfig = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; } @@ -218,15 +215,15 @@ public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { RgwAdmin rgwAdmin = getRgwAdminClient(storeId); String username = account.getUuid(); - s_logger.debug("Attempting to create Ceph RGW user for account " + account.getAccountName() + " with UUID " + username); + logger.debug("Attempting to create Ceph RGW user for account " + account.getAccountName() + " with UUID " + username); try { Optional user = rgwAdmin.getUserInfo(username); if (user.isPresent()) { - s_logger.info("User already exists in Ceph RGW: " + username); + logger.info("User already exists in Ceph RGW: " + username); return true; } } catch (Exception e) { - s_logger.debug("User does not exist. Creating user in Ceph RGW: " + username); + logger.debug("User does not exist. Creating user in Ceph RGW: " + username); } try { diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java index a9b13bf338e..8740d188ce0 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java @@ -39,7 +39,7 @@ import java.util.Map; public class CephObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { - private static final Logger s_logger = LogManager.getLogger(CephObjectStoreLifeCycleImpl.class); + private Logger logger = LogManager.getLogger(CephObjectStoreLifeCycleImpl.class); @Inject ObjectStoreHelper objectStoreHelper; @@ -72,7 +72,7 @@ public class CephObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { objectStoreParameters.put("accesskey", accessKey); objectStoreParameters.put("secretkey", secretKey); - s_logger.info("Attempting to connect to Ceph RGW at " + url + " with access key " + accessKey); + logger.info("Attempting to connect to Ceph RGW at " + url + " with access key " + accessKey); RgwAdmin rgwAdmin = new RgwAdminBuilder() .accessKey(accessKey) @@ -81,10 +81,10 @@ public class CephObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { .build(); try { List buckets = rgwAdmin.listBucket(); - s_logger.debug("Found " + buckets + " buckets at Ceph RGW: " + url); - s_logger.info("Successfully connected to Ceph RGW: " + url); + logger.debug("Found " + buckets + " buckets at Ceph RGW: " + url); + logger.info("Successfully connected to Ceph RGW: " + url); } catch (Exception e) { - s_logger.debug("Error while initializing Ceph RGW Object Store: " + e.getMessage()); + logger.debug("Error while initializing Ceph RGW Object Store: " + e.getMessage()); throw new RuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL"); } diff --git a/plugins/storage/object/minio/pom.xml b/plugins/storage/object/minio/pom.xml index 6358ee590e5..73bdab1abb1 100644 --- a/plugins/storage/object/minio/pom.xml +++ b/plugins/storage/object/minio/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/object/simulator/pom.xml b/plugins/storage/object/simulator/pom.xml index 803a03dba63..afe375b8116 100644 --- a/plugins/storage/object/simulator/pom.xml +++ b/plugins/storage/object/simulator/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/sharedfs/storagevm/pom.xml b/plugins/storage/sharedfs/storagevm/pom.xml index 200663b0d50..9446f5d738e 100644 --- a/plugins/storage/sharedfs/storagevm/pom.xml +++ b/plugins/storage/sharedfs/storagevm/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/adaptive/pom.xml b/plugins/storage/volume/adaptive/pom.xml index 724ecddcf03..d5a2f04e159 100644 --- a/plugins/storage/volume/adaptive/pom.xml +++ b/plugins/storage/volume/adaptive/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/cloudbyte/pom.xml b/plugins/storage/volume/cloudbyte/pom.xml index a7bd35074dc..6ed6cd0ff31 100644 --- a/plugins/storage/volume/cloudbyte/pom.xml +++ b/plugins/storage/volume/cloudbyte/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/datera/pom.xml b/plugins/storage/volume/datera/pom.xml index 3007095a1ad..45a6bfa646a 100644 --- a/plugins/storage/volume/datera/pom.xml +++ b/plugins/storage/volume/datera/pom.xml @@ -16,7 +16,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/default/pom.xml b/plugins/storage/volume/default/pom.xml index 1778a2b5c0f..d892b41c115 100644 --- a/plugins/storage/volume/default/pom.xml +++ b/plugins/storage/volume/default/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/flasharray/pom.xml b/plugins/storage/volume/flasharray/pom.xml index 91c3f6add5c..6c767f04cf6 100644 --- a/plugins/storage/volume/flasharray/pom.xml +++ b/plugins/storage/volume/flasharray/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 30f0225b45e..a0a53d20119 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2024-10-28] + +### Fixed + +- Disable discard="unmap" for ide devices and qemu < 7.0 + https://bugzilla.redhat.com/show_bug.cgi?id=2029980 + +## [2024-10-04] + +### Added + +- Enable qemu discard="unmap" for Linstor block disks + ## [2024-08-27] ### Changed diff --git a/plugins/storage/volume/linstor/pom.xml b/plugins/storage/volume/linstor/pom.xml index ad7193255e0..e437972a659 100644 --- a/plugins/storage/volume/linstor/pom.xml +++ b/plugins/storage/volume/linstor/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index fabe2ab3536..4148dcfbf4c 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -17,6 +17,7 @@ package com.cloud.hypervisor.kvm.storage; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -48,6 +49,7 @@ import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; import com.linbit.linstor.api.model.ResourceConnectionModify; import com.linbit.linstor.api.model.ResourceDefinition; +import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.ResourceWithVolumes; @@ -152,7 +154,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { @Override public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, - Storage.StoragePoolType type, Map details) + Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { logger.debug("Linstor createStoragePool: name: '{}', host: '{}', path: {}, userinfo: {}", name, host, path, userInfo); LinstorStoragePool storagePool = new LinstorStoragePool(name, host, port, userInfo, type, this); @@ -235,6 +237,34 @@ public class LinstorStorageAdaptor implements StorageAdaptor { } } + private void setAllowTwoPrimariesOnRD(DevelopersApi api, String rscName) throws ApiException { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + Properties props = new Properties(); + props.put("DrbdOptions/Net/allow-two-primaries", "yes"); + props.put("DrbdOptions/Net/protocol", "C"); + rdm.setOverrideProps(props); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + if (answers.hasError()) { + logger.error(String.format("Unable to set protocol C and 'allow-two-primaries' on %s", rscName)); + // do not fail here as adding allow-two-primaries property is only a problem while live migrating + } + } + + private void setAllowTwoPrimariesOnRc(DevelopersApi api, String rscName, String inUseNode) throws ApiException { + ResourceConnectionModify rcm = new ResourceConnectionModify(); + Properties props = new Properties(); + props.put("DrbdOptions/Net/allow-two-primaries", "yes"); + props.put("DrbdOptions/Net/protocol", "C"); + rcm.setOverrideProps(props); + ApiCallRcList answers = api.resourceConnectionModify(rscName, inUseNode, localNodeName, rcm); + if (answers.hasError()) { + logger.error(String.format( + "Unable to set protocol C and 'allow-two-primaries' on %s/%s/%s", + inUseNode, localNodeName, rscName)); + // do not fail here as adding allow-two-primaries property is only a problem while live migrating + } + } + /** * Checks if the given resource is in use by drbd on any host and * if so set the drbd option allow-two-primaries @@ -246,16 +276,13 @@ public class LinstorStorageAdaptor implements StorageAdaptor { String inUseNode = LinstorUtil.isResourceInUse(api, rscName); if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { // allow 2 primaries for live migration, should be removed by disconnect on the other end - ResourceConnectionModify rcm = new ResourceConnectionModify(); - Properties props = new Properties(); - props.put("DrbdOptions/Net/allow-two-primaries", "yes"); - props.put("DrbdOptions/Net/protocol", "C"); - rcm.setOverrideProps(props); - ApiCallRcList answers = api.resourceConnectionModify(rscName, inUseNode, localNodeName, rcm); - if (answers.hasError()) { - logger.error("Unable to set protocol C and 'allow-two-primaries' on {}/{}/{}", - inUseNode, localNodeName, rscName); - // do not fail here as adding allow-two-primaries property is only a problem while live migrating + + // if non hyperconverged setup, we have to set allow-two-primaries on the resource-definition + // as there is no resource connection between diskless nodes. + if (LinstorUtil.areResourcesDiskless(api, rscName, Arrays.asList(inUseNode, localNodeName))) { + setAllowTwoPrimariesOnRD(api, rscName); + } else { + setAllowTwoPrimariesOnRc(api, rscName, inUseNode); } } } @@ -294,11 +321,22 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return true; } - private void removeTwoPrimariesRcProps(DevelopersApi api, String inUseNode, String rscName) throws ApiException { + private void removeTwoPrimariesRDProps(DevelopersApi api, String rscName, List deleteProps) + throws ApiException { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.deleteProps(deleteProps); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + if (answers.hasError()) { + logger.error( + String.format("Failed to remove 'protocol' and 'allow-two-primaries' on %s: %s", + rscName, LinstorUtil.getBestErrorMessage(answers))); + // do not fail here as removing allow-two-primaries property isn't fatal + } + } + + private void removeTwoPrimariesRcProps(DevelopersApi api, String rscName, String inUseNode, List deleteProps) + throws ApiException { ResourceConnectionModify rcm = new ResourceConnectionModify(); - List deleteProps = new ArrayList<>(); - deleteProps.add("DrbdOptions/Net/allow-two-primaries"); - deleteProps.add("DrbdOptions/Net/protocol"); rcm.deleteProps(deleteProps); ApiCallRcList answers = api.resourceConnectionModify(rscName, localNodeName, inUseNode, rcm); if (answers.hasError()) { @@ -310,6 +348,15 @@ public class LinstorStorageAdaptor implements StorageAdaptor { } } + private void removeTwoPrimariesProps(DevelopersApi api, String inUseNode, String rscName) throws ApiException { + List deleteProps = new ArrayList<>(); + deleteProps.add("DrbdOptions/Net/allow-two-primaries"); + deleteProps.add("DrbdOptions/Net/protocol"); + + removeTwoPrimariesRDProps(api, rscName, deleteProps); + removeTwoPrimariesRcProps(api, rscName, inUseNode, deleteProps); + } + private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) { if (volumePath == null) { @@ -343,7 +390,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { try { String inUseNode = LinstorUtil.isResourceInUse(api, rsc.getName()); if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { - removeTwoPrimariesRcProps(api, inUseNode, rsc.getName()); + removeTwoPrimariesProps(api, inUseNode, rsc.getName()); } } catch (ApiException apiEx) { logger.error(apiEx.getBestMessage()); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index d3a8b97cf2a..cbdc9940b80 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.datastore.util; import com.linbit.linstor.api.ApiClient; +import com.linbit.linstor.api.ApiConsts; import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRc; @@ -33,6 +34,7 @@ import com.linbit.linstor.api.model.Volume; import javax.annotation.Nonnull; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -210,6 +212,28 @@ public class LinstorUtil { return null; } + /** + * Check if the given resources are diskless. + * + * @param api developer api object to use + * @param rscName resource name to check in use state. + * @return NodeName where the resource is inUse, if not in use `null` + * @throws ApiException forwards api errors + */ + public static boolean areResourcesDiskless(DevelopersApi api, String rscName, Collection nodeNames) + throws ApiException { + List rscs = api.resourceList(rscName, null, null); + if (rscs != null) { + Collection disklessNodes = rscs.stream() + .filter(rsc -> rsc.getFlags() != null && (rsc.getFlags().contains(ApiConsts.FLAG_DISKLESS) || + rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS))) + .map(rsc -> rsc.getNodeName().toLowerCase()) + .collect(Collectors.toList()); + return disklessNodes.containsAll(nodeNames.stream().map(String::toLowerCase).collect(Collectors.toList())); + } + return false; + } + /** * Try to get the device path for the given resource name. * This could be made a bit more direct after java-linstor api is fixed for layer data subtypes. diff --git a/plugins/storage/volume/nexenta/pom.xml b/plugins/storage/volume/nexenta/pom.xml index 0ea2c4036f2..c9d26d2a325 100644 --- a/plugins/storage/volume/nexenta/pom.xml +++ b/plugins/storage/volume/nexenta/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/primera/pom.xml b/plugins/storage/volume/primera/pom.xml index 489b81dd405..ec3a062f44a 100644 --- a/plugins/storage/volume/primera/pom.xml +++ b/plugins/storage/volume/primera/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/sample/pom.xml b/plugins/storage/volume/sample/pom.xml index 2050a6d3eda..fba65349ee3 100644 --- a/plugins/storage/volume/sample/pom.xml +++ b/plugins/storage/volume/sample/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/scaleio/pom.xml b/plugins/storage/volume/scaleio/pom.xml index d894cea24bd..a58e30a269c 100644 --- a/plugins/storage/volume/scaleio/pom.xml +++ b/plugins/storage/volume/scaleio/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/solidfire/pom.xml b/plugins/storage/volume/solidfire/pom.xml index 46c0579a69f..62de0caeace 100644 --- a/plugins/storage/volume/solidfire/pom.xml +++ b/plugins/storage/volume/solidfire/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/storpool/pom.xml b/plugins/storage/volume/storpool/pom.xml index e1555dc8958..e639d04fbe2 100644 --- a/plugins/storage/volume/storpool/pom.xml +++ b/plugins/storage/volume/storpool/pom.xml @@ -17,7 +17,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../../pom.xml diff --git a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java index c05d8b3ae08..de3886f6294 100644 --- a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java +++ b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java @@ -57,7 +57,7 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { private static final Map storageUuidToStoragePool = new HashMap(); @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details, boolean isPrimaryStorage) { SP_LOG("StorPoolStorageAdaptor.createStoragePool: uuid=%s, host=%s:%d, path=%s, userInfo=%s, type=%s", uuid, host, port, path, userInfo, storagePoolType); StorPoolStoragePool storagePool = new StorPoolStoragePool(uuid, host, port, storagePoolType, this); diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java index 97f4e2fe155..dcb190a573b 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java @@ -138,6 +138,8 @@ public class StorPoolUtil { public static final String SP_TIER = "SP_QOSCLASS"; + public static final String OBJECT_DOES_NOT_EXIST = "objectDoesNotExist"; + public static enum StorpoolRights { RO("ro"), RW("rw"), DETACH("detach"); @@ -458,7 +460,7 @@ public class StorPoolUtil { } private static boolean objectExists(SpApiError err) { - if (!err.getName().equals("objectDoesNotExist")) { + if (!err.getName().equals(OBJECT_DOES_NOT_EXIST)) { throw new CloudRuntimeException(err.getDescr()); } return false; diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java index 5cdb7b8cda1..c7bcc8a46b7 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java @@ -16,10 +16,19 @@ // under the License. package org.apache.cloudstack.storage.snapshot; -import java.util.ArrayList; -import java.util.List; - -import javax.inject.Inject; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.kvm.storage.StorPoolStorageAdaptor; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotDetailsDao; +import com.cloud.storage.dao.SnapshotDetailsVO; +import com.cloud.storage.dao.SnapshotZoneDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -40,23 +49,13 @@ import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpApiResponse; import org.apache.cloudstack.storage.datastore.util.StorPoolUtil.SpConnectionDesc; import org.apache.commons.collections.CollectionUtils; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.hypervisor.kvm.storage.StorPoolStorageAdaptor; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.SnapshotDao; -import com.cloud.storage.dao.SnapshotDetailsDao; -import com.cloud.storage.dao.SnapshotDetailsVO; -import com.cloud.storage.dao.SnapshotZoneDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.fsm.NoTransitionException; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; @Component @@ -117,10 +116,11 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { if (resp.getError() != null) { final String err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError()); StorPoolUtil.spLog(err); - markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId, resp); + markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId, resp.getError().getName().equals(StorPoolUtil.OBJECT_DOES_NOT_EXIST)); throw new CloudRuntimeException(err); } else { res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId); + markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId,true); StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed successfully=%s, snapshot uuid=%s, name=%s", res, snapshotVO.getUuid(), name); } } catch (Exception e) { @@ -129,15 +129,23 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { } } + List snapshots = _snapshotStoreDao.listBySnapshotIdAndState(snapshotId, State.Ready); + if (res || CollectionUtils.isEmpty(snapshots)) { + updateSnapshotToDestroyed(snapshotVO); + return true; + } return res; } - private void markSnapshotAsDestroyedIfAlreadyRemoved(Long snapshotId, SpApiResponse resp) { - if (resp.getError().getName().equals("objectDoesNotExist")) { - SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findBySourceSnapshot(snapshotId, DataStoreRole.Primary); - if (snapshotOnPrimary != null) { - snapshotOnPrimary.setState(State.Destroyed); - _snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary); + private void markSnapshotAsDestroyedIfAlreadyRemoved(Long snapshotId, boolean isSnapshotDeleted) { + if (!isSnapshotDeleted) { + return; + } + List snapshotsOnStore = _snapshotStoreDao.listBySnapshotIdAndState(snapshotId, State.Ready); + for (SnapshotDataStoreVO snapshot : snapshotsOnStore) { + if (snapshot.getInstallPath() != null && snapshot.getInstallPath().contains(StorPoolUtil.SP_DEV_PATH)) { + snapshot.setState(State.Destroyed); + _snapshotStoreDao.update(snapshot.getId(), snapshot); } } } diff --git a/plugins/user-authenticators/ldap/pom.xml b/plugins/user-authenticators/ldap/pom.xml index 010284e3766..bdbb3068823 100644 --- a/plugins/user-authenticators/ldap/pom.xml +++ b/plugins/user-authenticators/ldap/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-authenticators/md5/pom.xml b/plugins/user-authenticators/md5/pom.xml index e63f9774341..cd26f274763 100644 --- a/plugins/user-authenticators/md5/pom.xml +++ b/plugins/user-authenticators/md5/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-authenticators/oauth2/pom.xml b/plugins/user-authenticators/oauth2/pom.xml index 5a1e49874a8..c055a40a48e 100644 --- a/plugins/user-authenticators/oauth2/pom.xml +++ b/plugins/user-authenticators/oauth2/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-authenticators/pbkdf2/pom.xml b/plugins/user-authenticators/pbkdf2/pom.xml index f030e38a6a4..832452b1329 100644 --- a/plugins/user-authenticators/pbkdf2/pom.xml +++ b/plugins/user-authenticators/pbkdf2/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-authenticators/plain-text/pom.xml b/plugins/user-authenticators/plain-text/pom.xml index e378ec8399b..5590d0764e1 100644 --- a/plugins/user-authenticators/plain-text/pom.xml +++ b/plugins/user-authenticators/plain-text/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-authenticators/saml2/pom.xml b/plugins/user-authenticators/saml2/pom.xml index 7a19768fab7..6ba93021b6d 100644 --- a/plugins/user-authenticators/saml2/pom.xml +++ b/plugins/user-authenticators/saml2/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java index 3e6b093abe1..4851973c2ad 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmd.java @@ -47,6 +47,7 @@ import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.saml.SAML2AuthManager; import org.apache.cloudstack.saml.SAMLUtils; +import com.cloud.api.ApiServer; import com.cloud.api.response.ApiResponseSerializer; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; @@ -59,6 +60,8 @@ import com.cloud.user.dao.UserAccountDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.HttpUtils; +import org.apache.commons.lang3.EnumUtils; + @APICommand(name = "listAndSwitchSamlAccount", description = "Lists and switches to other SAML accounts owned by the SAML user", responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListAndSwitchSAMLAccountCmd extends BaseCmd implements APIAuthenticator { @@ -102,7 +105,9 @@ public class ListAndSwitchSAMLAccountCmd extends BaseCmd implements APIAuthentic params, responseType)); } - if (!HttpUtils.validateSessionKey(session, params, req.getCookies(), ApiConstants.SESSIONKEY)) { + HttpUtils.ApiSessionKeyCheckOption sessionKeyCheckOption = EnumUtils.getEnumIgnoreCase(HttpUtils.ApiSessionKeyCheckOption.class, + ApiServer.ApiSessionKeyCheckLocations.value(), HttpUtils.ApiSessionKeyCheckOption.CookieAndParameter); + if (!HttpUtils.validateSessionKey(session, params, req.getCookies(), ApiConstants.SESSIONKEY, sessionKeyCheckOption)) { throw new ServerApiException(ApiErrorCode.UNAUTHORIZED, _apiServer.getSerializedApiError(ApiErrorCode.UNAUTHORIZED.getHttpCode(), "Unauthorized session, please re-login", params, responseType)); diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java index 3a4030f9c0d..4e8ba16c739 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java @@ -73,6 +73,12 @@ public interface SAML2AuthManager extends PluggableAPIAuthenticator, PluggableSe ConfigKey SAMLCheckSignature = new ConfigKey("Advanced", Boolean.class, "saml2.check.signature", "true", "When enabled (default and recommended), SAML2 signature checks are enforced and lack of signature in the SAML SSO response will cause login exception. Disabling this is not advisable but provided for backward compatibility for users who are able to accept the risks.", false); + ConfigKey SAMLForceAuthn = new ConfigKey("Advanced", Boolean.class, "saml2.force.authn", "false", + "When enabled (default false), SAML2 will force a new authentication. This can be useful if multiple application use different saml logins from the same application (I.E. browser)", true); + + ConfigKey SAMLUserSessionKeyPathAttribute = new ConfigKey("Advanced", String.class, "saml2.user.sessionkey.path", "", + "The Path attribute of sessionkey cookie when SAML users have logged in. If not set, it will be set to the path of SAML redirection URL (saml2.redirect.url).", true); + SAMLProviderMetadata getSPMetadata(); SAMLProviderMetadata getIdPMetadata(String entityId); Collection getAllIdPMetadata(); diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java index 230c53ac4a9..545b01a4a31 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java @@ -540,6 +540,7 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage SAMLServiceProviderSingleSignOnURL, SAMLServiceProviderSingleLogOutURL, SAMLCloudStackRedirectionUrl, SAMLUserAttributeName, SAMLIdentityProviderMetadataURL, SAMLDefaultIdentityProviderId, - SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature}; + SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature, + SAMLForceAuthn, SAMLUserSessionKeyPathAttribute}; } } diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java index 7ffe07a8609..54f6e84fe36 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAMLUtils.java @@ -25,6 +25,8 @@ import java.io.IOException; import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.math.BigInteger; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.Charset; import java.security.InvalidKeyException; @@ -102,7 +104,9 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.SAXException; +import com.cloud.api.ApiServlet; import com.cloud.utils.HttpUtils; +import com.cloud.utils.exception.CloudRuntimeException; public class SAMLUtils { protected static Logger LOGGER = LogManager.getLogger(SAMLUtils.class); @@ -190,7 +194,7 @@ public class SAMLUtils { authnRequest.setID(authnId); authnRequest.setDestination(idpUrl); authnRequest.setVersion(SAMLVersion.VERSION_20); - authnRequest.setForceAuthn(false); + authnRequest.setForceAuthn(SAML2AuthManager.SAMLForceAuthn.value()); authnRequest.setIsPassive(false); authnRequest.setIssueInstant(new DateTime()); authnRequest.setProtocolBinding(SAMLConstants.SAML2_POST_BINDING_URI); @@ -297,7 +301,26 @@ public class SAMLUtils { resp.addCookie(new Cookie("timezone", URLEncoder.encode(timezone, HttpUtils.UTF_8))); } resp.addCookie(new Cookie("userfullname", URLEncoder.encode(loginResponse.getFirstName() + " " + loginResponse.getLastName(), HttpUtils.UTF_8).replace("+", "%20"))); - resp.addHeader("SET-COOKIE", String.format("%s=%s;HttpOnly;Path=/client/api", ApiConstants.SESSIONKEY, loginResponse.getSessionKey())); + + String redirectUrl = SAML2AuthManager.SAMLCloudStackRedirectionUrl.value(); + String path = SAML2AuthManager.SAMLUserSessionKeyPathAttribute.value(); + String domain = null; + try { + URI redirectUri = new URI(redirectUrl); + domain = redirectUri.getHost(); + if (StringUtils.isBlank(path)) { + path = redirectUri.getPath(); + } + if (StringUtils.isBlank(path)) { + path = "/"; + } + } catch (URISyntaxException ex) { + throw new CloudRuntimeException("Invalid URI: " + redirectUrl); + } + String sameSite = ApiServlet.getApiSessionKeySameSite(); + String sessionKeyCookie = String.format("%s=%s;Domain=%s;Path=%s;%s", ApiConstants.SESSIONKEY, loginResponse.getSessionKey(), domain, path, sameSite); + LOGGER.debug("Adding sessionkey cookie to response: " + sessionKeyCookie); + resp.addHeader("SET-COOKIE", sessionKeyCookie); } /** diff --git a/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmdTest.java b/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmdTest.java index 9342a0c7f09..9225574ff60 100644 --- a/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmdTest.java +++ b/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/ListAndSwitchSAMLAccountCmdTest.java @@ -27,6 +27,7 @@ import java.net.InetAddress; import java.util.HashMap; import java.util.Map; +import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; @@ -88,6 +89,9 @@ public class ListAndSwitchSAMLAccountCmdTest extends TestCase { @Mock HttpServletRequest req; + final String sessionId = "node0xxxxxxxxxxxxx"; + Cookie[] cookies; + @Test public void testListAndSwitchSAMLAccountCmd() throws Exception { // Setup @@ -95,6 +99,7 @@ public class ListAndSwitchSAMLAccountCmdTest extends TestCase { final String sessionKeyValue = "someSessionIDValue"; Mockito.when(session.getAttribute(ApiConstants.SESSIONKEY)).thenReturn(sessionKeyValue); Mockito.when(session.getAttribute("userid")).thenReturn(2L); + Mockito.when(session.getId()).thenReturn(sessionId); params.put(ApiConstants.USER_ID, new String[]{"2"}); params.put(ApiConstants.DOMAIN_ID, new String[]{"1"}); Mockito.when(userDao.findByUuid(anyString())).thenReturn(new UserVO(2L)); @@ -146,7 +151,25 @@ public class ListAndSwitchSAMLAccountCmdTest extends TestCase { Mockito.verify(accountService, Mockito.times(0)).getUserAccountById(Mockito.anyLong()); } - // valid sessionkey value test + // valid sessionkey value and invalid JSESSIONID test + cookies = new Cookie[2]; + cookies[0] = new Cookie(ApiConstants.SESSIONKEY, sessionKeyValue); + cookies[1] = new Cookie("JSESSIONID", "invalid-JSESSIONID"); + Mockito.when(req.getCookies()).thenReturn(cookies); + params.put(ApiConstants.SESSIONKEY, new String[]{sessionKeyValue}); + try { + cmd.authenticate("command", params, session, null, HttpUtils.RESPONSE_TYPE_JSON, new StringBuilder(), req, resp); + } catch (ServerApiException exception) { + assertEquals(exception.getErrorCode(), ApiErrorCode.UNAUTHORIZED); + } finally { + Mockito.verify(accountService, Mockito.times(0)).getUserAccountById(Mockito.anyLong()); + } + + // valid sessionkey value and valid JSESSIONID test + cookies = new Cookie[2]; + cookies[0] = new Cookie(ApiConstants.SESSIONKEY, sessionKeyValue); + cookies[1] = new Cookie("JSESSIONID", sessionId + ".node0"); + Mockito.when(req.getCookies()).thenReturn(cookies); params.put(ApiConstants.SESSIONKEY, new String[]{sessionKeyValue}); try { cmd.authenticate("command", params, session, null, HttpUtils.RESPONSE_TYPE_JSON, new StringBuilder(), req, resp); diff --git a/plugins/user-authenticators/sha256salted/pom.xml b/plugins/user-authenticators/sha256salted/pom.xml index 823ab510568..853e97e47e6 100644 --- a/plugins/user-authenticators/sha256salted/pom.xml +++ b/plugins/user-authenticators/sha256salted/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-two-factor-authenticators/static-pin/pom.xml b/plugins/user-two-factor-authenticators/static-pin/pom.xml index bde07b6c185..335faa428c0 100644 --- a/plugins/user-two-factor-authenticators/static-pin/pom.xml +++ b/plugins/user-two-factor-authenticators/static-pin/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/plugins/user-two-factor-authenticators/totp/pom.xml b/plugins/user-two-factor-authenticators/totp/pom.xml index cda38336291..ed0cfb3fe8e 100644 --- a/plugins/user-two-factor-authenticators/totp/pom.xml +++ b/plugins/user-two-factor-authenticators/totp/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../../pom.xml diff --git a/pom.xml b/pom.xml index 29fc939f553..a3003797950 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 pom Apache CloudStack Apache CloudStack is an IaaS ("Infrastructure as a Service") cloud orchestration platform. diff --git a/quickcloud/pom.xml b/quickcloud/pom.xml index c02b4932ee7..71ba74ae71d 100644 --- a/quickcloud/pom.xml +++ b/quickcloud/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/server/pom.xml b/server/pom.xml index 8f7f5e85f86..aa8f362ad79 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index 739ad765afa..72e97c3a6ee 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -33,6 +33,7 @@ import java.text.ParseException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; +import java.util.EnumSet; import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; @@ -47,6 +48,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -169,6 +171,8 @@ import com.cloud.storage.VolumeApiService; import com.cloud.utils.ConstantTimeComparator; import com.cloud.utils.DateUtil; import com.cloud.utils.HttpUtils; +import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; +import com.cloud.utils.HttpUtils.ApiSessionKeyCheckOption; import com.cloud.utils.Pair; import com.cloud.utils.ReflectUtil; import com.cloud.utils.StringUtils; @@ -310,6 +314,24 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer , true , ConfigKey.Scope.Global); + static final ConfigKey ApiSessionKeyCookieSameSiteSetting = new ConfigKey<>(String.class + , "api.sessionkey.cookie.samesite" + , ConfigKey.CATEGORY_ADVANCED + , ApiSessionKeySameSite.Lax.name() + , "The SameSite attribute of cookie 'sessionkey'. Valid options are: Lax (default), Strict, NoneAndSecure and Null." + , true + , ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, + EnumSet.allOf(ApiSessionKeySameSite.class).stream().map(Enum::toString).collect(Collectors.joining(", "))); + + public static final ConfigKey ApiSessionKeyCheckLocations = new ConfigKey<>(String.class + , "api.sessionkey.check.locations" + , ConfigKey.CATEGORY_ADVANCED + , ApiSessionKeyCheckOption.CookieAndParameter.name() + , "The locations of 'sessionkey' during the validation of the API requests. Valid options are: CookieOrParameter, ParameterOnly, CookieAndParameter (default)." + , true + , ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, + EnumSet.allOf(ApiSessionKeyCheckOption.class).stream().map(Enum::toString).collect(Collectors.joining(", "))); + @Override public boolean configure(final String name, final Map params) throws ConfigurationException { messageBus.subscribe(AsyncJob.Topics.JOB_EVENT_PUBLISH, MessageDispatcher.getDispatcher(this)); @@ -1582,7 +1604,9 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer JSONDefaultContentType, proxyForwardList, useForwardHeader, - listOfForwardHeaders + listOfForwardHeaders, + ApiSessionKeyCookieSameSiteSetting, + ApiSessionKeyCheckLocations }; } } diff --git a/server/src/main/java/com/cloud/api/ApiServlet.java b/server/src/main/java/com/cloud/api/ApiServlet.java index f2b5d3c4797..e2ff411f8f4 100644 --- a/server/src/main/java/com/cloud/api/ApiServlet.java +++ b/server/src/main/java/com/cloud/api/ApiServlet.java @@ -47,8 +47,10 @@ import org.apache.cloudstack.api.command.user.consoleproxy.CreateConsoleEndpoint import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.cloudstack.utils.consoleproxy.ConsoleAccessUtils; + import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.commons.lang3.EnumUtils; import org.jetbrains.annotations.Nullable; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; @@ -65,6 +67,8 @@ import com.cloud.user.User; import com.cloud.user.UserAccount; import com.cloud.utils.HttpUtils; +import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; +import com.cloud.utils.HttpUtils.ApiSessionKeyCheckOption; import com.cloud.utils.StringUtils; import com.cloud.utils.db.EntityManager; import com.cloud.utils.net.NetUtils; @@ -255,7 +259,8 @@ public class ApiServlet extends HttpServlet { } responseString = apiAuthenticator.authenticate(command, params, session, remoteAddress, responseType, auditTrailSb, req, resp); if (session != null && session.getAttribute(ApiConstants.SESSIONKEY) != null) { - resp.addHeader("SET-COOKIE", String.format("%s=%s;HttpOnly", ApiConstants.SESSIONKEY, session.getAttribute(ApiConstants.SESSIONKEY))); + String sameSite = getApiSessionKeySameSite(); + resp.addHeader("SET-COOKIE", String.format("%s=%s;HttpOnly;%s", ApiConstants.SESSIONKEY, session.getAttribute(ApiConstants.SESSIONKEY), sameSite)); } } catch (ServerApiException e) { httpResponseCode = e.getErrorCode().getHttpCode(); @@ -264,19 +269,22 @@ public class ApiServlet extends HttpServlet { } if (apiAuthenticator.getAPIType() == APIAuthenticationType.LOGOUT_API) { - if (session != null) { - final Long userId = (Long) session.getAttribute("userid"); - final Account account = (Account) session.getAttribute("accountobj"); - Long accountId = null; - if (account != null) { - accountId = account.getId(); - } - auditTrailSb.insert(0, "(userId=" + userId + " accountId=" + accountId + " sessionId=" + session.getId() + ")"); - if (userId != null) { - apiServer.logoutUser(userId); - } - invalidateHttpSession(session, "invalidating session after logout call"); + if (session == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Session not found for the logout process."); } + + final Long userId = (Long) session.getAttribute("userid"); + final Account account = (Account) session.getAttribute("accountobj"); + Long accountId = null; + if (account != null) { + accountId = account.getId(); + } + auditTrailSb.insert(0, "(userId=" + userId + " accountId=" + accountId + " sessionId=" + session.getId() + ")"); + if (userId != null) { + apiServer.logoutUser(userId); + } + invalidateHttpSession(session, "invalidating session after logout call"); + final Cookie[] cookies = req.getCookies(); if (cookies != null) { for (final Cookie cookie : cookies) { @@ -375,6 +383,22 @@ public class ApiServlet extends HttpServlet { } } + public static String getApiSessionKeySameSite() { + ApiSessionKeySameSite sameSite = EnumUtils.getEnumIgnoreCase(ApiSessionKeySameSite.class, + ApiServer.ApiSessionKeyCookieSameSiteSetting.value(), ApiSessionKeySameSite.Lax); + switch (sameSite) { + case Strict: + return "SameSite=Strict"; + case NoneAndSecure: + return "SameSite=None;Secure"; + case Null: + return ""; + case Lax: + default: + return "SameSite=Lax"; + } + } + private boolean checkIfAuthenticatorIsOf2FA(String command) { boolean verify2FA = false; APIAuthenticator apiAuthenticator = authManager.getAPIAuthenticator(command); @@ -510,7 +534,9 @@ public class ApiServlet extends HttpServlet { } private boolean invalidateHttpSessionIfNeeded(HttpServletRequest req, HttpServletResponse resp, StringBuilder auditTrailSb, String responseType, Map params, HttpSession session, String account) { - if (!HttpUtils.validateSessionKey(session, params, req.getCookies(), ApiConstants.SESSIONKEY)) { + ApiSessionKeyCheckOption sessionKeyCheckOption = EnumUtils.getEnumIgnoreCase(ApiSessionKeyCheckOption.class, + ApiServer.ApiSessionKeyCheckLocations.value(), ApiSessionKeyCheckOption.CookieAndParameter); + if (!HttpUtils.validateSessionKey(session, params, req.getCookies(), ApiConstants.SESSIONKEY, sessionKeyCheckOption)) { String msg = String.format("invalidating session %s for account %s", session.getId(), account); invalidateHttpSession(session, msg); auditTrailSb.append(" " + HttpServletResponse.SC_UNAUTHORIZED + " " + "unable to verify user credentials"); diff --git a/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java b/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java index 314b83acdb5..c4145755b56 100644 --- a/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java +++ b/server/src/main/java/com/cloud/api/dispatch/ParamProcessWorker.java @@ -32,8 +32,6 @@ import java.util.regex.Matcher; import javax.inject.Inject; -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.acl.InfrastructureEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -312,7 +310,7 @@ public class ParamProcessWorker implements DispatchWorker { doAccessChecks(cmd, entitiesToAccess); } - private void doAccessChecks(BaseCmd cmd, Map entitiesToAccess) { + protected void doAccessChecks(BaseCmd cmd, Map entitiesToAccess) { Account caller = CallContext.current().getCallingAccount(); List entityOwners = cmd.getEntityOwnerIds(); Account[] owners = null; @@ -334,19 +332,35 @@ public class ParamProcessWorker implements DispatchWorker { _accountMgr.checkAccess(caller, null, false, owners); } - if (!entitiesToAccess.isEmpty()) { - // check that caller can access the owner account. - _accountMgr.checkAccess(caller, null, false, owners); - for (Map.Entryentry : entitiesToAccess.entrySet()) { - Object entity = entry.getKey(); - if (entity instanceof ControlledEntity) { - _accountMgr.checkAccess(caller, entry.getValue(), true, (ControlledEntity) entity); - } else if (entity instanceof InfrastructureEntity) { - // FIXME: Move this code in adapter, remove code from - // Account manager - } + checkCallerAccessToEntities(caller, owners, entitiesToAccess); + } + + protected Account[] getEntityOwners(BaseCmd cmd) { + List entityOwners = cmd.getEntityOwnerIds(); + if (entityOwners != null) { + return entityOwners.stream().map(id -> _accountMgr.getAccount(id)).toArray(Account[]::new); + } + + if (cmd.getEntityOwnerId() == Account.ACCOUNT_ID_SYSTEM && cmd instanceof BaseAsyncCmd && cmd.getApiResourceType() == ApiCommandResourceType.Network) { + logger.debug("Skipping access check on the network owner if the owner is ROOT/system."); + } else { + Account owner = _accountMgr.getAccount(cmd.getEntityOwnerId()); + if (owner != null) { + return new Account[]{owner}; } } + return new Account[]{}; + } + + protected void checkCallerAccessToEntities(Account caller, Account[] owners, Map entitiesToAccess) { + if (entitiesToAccess.isEmpty()) { + return; + } + _accountMgr.checkAccess(caller, null, false, owners); + for (Map.Entry entry : entitiesToAccess.entrySet()) { + Object entity = entry.getKey(); + _accountMgr.validateAccountHasAccessToResource(caller, entry.getValue(), entity); + } } @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 4063cfe7a18..e7bd2174a61 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -245,6 +245,8 @@ import com.cloud.network.VpcVirtualNetworkApplianceService; import com.cloud.network.as.AutoScaleVmGroupVmMapVO; import com.cloud.network.as.dao.AutoScaleVmGroupDao; import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PublicIpQuarantineDao; @@ -551,6 +553,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private NetworkDao networkDao; + @Inject + private IPAddressDao ipAddressDao; + @Inject private NicDao nicDao; @@ -1461,6 +1466,22 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (isRootAdmin) { userVmSearchBuilder.or("keywordInstanceName", userVmSearchBuilder.entity().getInstanceName(), Op.LIKE ); } + + SearchBuilder ipAddressSearch = ipAddressDao.createSearchBuilder(); + userVmSearchBuilder.join("ipAddressSearch", ipAddressSearch, + ipAddressSearch.entity().getAssociatedWithVmId(), userVmSearchBuilder.entity().getId(), JoinBuilder.JoinType.LEFT); + + SearchBuilder nicSearch = nicDao.createSearchBuilder(); + userVmSearchBuilder.join("nicSearch", nicSearch, JoinBuilder.JoinType.LEFT, + JoinBuilder.JoinCondition.AND, + nicSearch.entity().getInstanceId(), userVmSearchBuilder.entity().getId(), + nicSearch.entity().getRemoved(), userVmSearchBuilder.entity().setLong(null)); + + userVmSearchBuilder.or("ipAddressSearch", "keywordPublicIpAddress", ipAddressSearch.entity().getAddress(), Op.LIKE); + + userVmSearchBuilder.or("nicSearch", "keywordIpAddress", nicSearch.entity().getIPv4Address(), Op.LIKE); + userVmSearchBuilder.or("nicSearch", "keywordIp6Address", nicSearch.entity().getIPv6Address(), Op.LIKE); + userVmSearchBuilder.cp(); } @@ -1554,6 +1575,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q userVmSearchCriteria.setParameters("keywordDisplayName", keywordMatch); userVmSearchCriteria.setParameters("keywordName", keywordMatch); userVmSearchCriteria.setParameters("keywordState", keyword); + userVmSearchCriteria.setParameters("keywordIpAddress", keywordMatch); + userVmSearchCriteria.setParameters("keywordPublicIpAddress", keywordMatch); + userVmSearchCriteria.setParameters("keywordIp6Address", keywordMatch); if (isRootAdmin) { userVmSearchCriteria.setParameters("keywordInstanceName", keywordMatch); } @@ -3837,7 +3861,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q serviceOfferingSearch.and().op("vmMemory", serviceOfferingSearch.entity().getRamSize(), Op.GTEQ); serviceOfferingSearch.or().op("vmMemoryNull", serviceOfferingSearch.entity().getRamSize(), Op.NULL); serviceOfferingSearch.and().op("maxMemoryDetailsSearch", "vmMaxMemoryNull", maxMemoryDetailsSearch.entity().getValue(), Op.NULL); - serviceOfferingSearch.and("maxMemoryDetailsSearch", "vmMaxMemoryGTEQ", maxMemoryDetailsSearch.entity().getValue(), Op.GTEQ).cp(); + serviceOfferingSearch.or("maxMemoryDetailsSearch", "vmMaxMemoryGTEQ", maxMemoryDetailsSearch.entity().getValue(), Op.GTEQ).cp(); serviceOfferingSearch.cp().cp(); } diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index 78234497cd0..39e8518f760 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -43,6 +43,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.acl.APIChecker; import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.InfrastructureEntity; import org.apache.cloudstack.acl.QuerySelector; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; @@ -740,6 +741,19 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } + @Override + public void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource) { + Class resourceClass = resource.getClass(); + if (ControlledEntity.class.isAssignableFrom(resourceClass)) { + checkAccess(account, accessType, true, (ControlledEntity) resource); + } else if (Domain.class.isAssignableFrom(resourceClass)) { + checkAccess(account, (Domain) resource); + } else if (InfrastructureEntity.class.isAssignableFrom(resourceClass)) { + logger.trace("Validation of access to infrastructure entity has been disabled in CloudStack version 4.4."); + } + logger.debug(String.format("Account [%s] has access to resource.", account.getUuid())); + } + @Override public Long checkAccessAndSpecifyAuthority(Account caller, Long zoneId) { // We just care for resource domain admins for now, and they should be permitted to see only their zone. diff --git a/server/src/test/java/com/cloud/api/dispatch/ParamProcessWorkerTest.java b/server/src/test/java/com/cloud/api/dispatch/ParamProcessWorkerTest.java index da70bc1c1bf..a998d2e9ab4 100644 --- a/server/src/test/java/com/cloud/api/dispatch/ParamProcessWorkerTest.java +++ b/server/src/test/java/com/cloud/api/dispatch/ParamProcessWorkerTest.java @@ -18,6 +18,30 @@ */ package com.cloud.api.dispatch; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.InjectMocks; +import org.mockito.Spy; + +import org.mockito.junit.MockitoJUnitRunner; + +import org.apache.cloudstack.api.ApiArgValidator; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.context.CallContext; + import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.NetworkRuleConflictException; @@ -26,29 +50,33 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; -import org.apache.cloudstack.api.ApiArgValidator; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.context.CallContext; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import java.util.HashMap; +import com.cloud.vm.VMInstanceVO; @RunWith(MockitoJUnitRunner.class) public class ParamProcessWorkerTest { - @Mock - protected AccountManager accountManager; + @Spy + @InjectMocks + private ParamProcessWorker paramProcessWorkerSpy; - protected ParamProcessWorker paramProcessWorker; + @Mock + private AccountManager accountManagerMock; + + @Mock + private Account callingAccountMock; + + @Mock + private User callingUserMock; + + @Mock + private Account ownerAccountMock; + + @Mock + BaseCmd baseCmdMock; + + private Account[] owners = new Account[]{ownerAccountMock}; + + private Map entities = new HashMap<>(); public static class TestCmd extends BaseCmd { @@ -69,7 +97,7 @@ public class ParamProcessWorkerTest { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, - ResourceAllocationException, NetworkRuleConflictException { + ResourceAllocationException, NetworkRuleConflictException { // well documented nothing } @@ -87,9 +115,7 @@ public class ParamProcessWorkerTest { @Before public void setup() { - CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); - paramProcessWorker = new ParamProcessWorker(); - paramProcessWorker._accountMgr = accountManager; + CallContext.register(callingUserMock, callingAccountMock); } @After @@ -106,7 +132,7 @@ public class ParamProcessWorkerTest { params.put("doubleparam1", "11.89"); params.put("vmHostNameParam", "test-host-name-123"); final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); + paramProcessWorkerSpy.processParameters(cmd, params); Assert.assertEquals("foo", cmd.strparam1); Assert.assertEquals(100, cmd.intparam1); Assert.assertTrue(Double.compare(cmd.doubleparam1, 11.89) == 0); @@ -118,7 +144,7 @@ public class ParamProcessWorkerTest { final HashMap params = new HashMap(); params.put("vmHostNameParam", "123test"); final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); + paramProcessWorkerSpy.processParameters(cmd, params); } @Test(expected = ServerApiException.class) @@ -126,7 +152,7 @@ public class ParamProcessWorkerTest { final HashMap params = new HashMap(); params.put("vmHostNameParam", "-test"); final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); + paramProcessWorkerSpy.processParameters(cmd, params); } @Test(expected = ServerApiException.class) @@ -134,7 +160,7 @@ public class ParamProcessWorkerTest { final HashMap params = new HashMap(); params.put("vmHostNameParam", "test-"); final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); + paramProcessWorkerSpy.processParameters(cmd, params); } @Test(expected = ServerApiException.class) @@ -142,6 +168,68 @@ public class ParamProcessWorkerTest { final HashMap params = new HashMap(); params.put("vmHostNameParam", "test-f2405112-d5a1-47c1-9f00-976909e3a6d3-1e6f3264-955ee76011a99"); final TestCmd cmd = new TestCmd(); - paramProcessWorker.processParameters(cmd, params); + paramProcessWorkerSpy.processParameters(cmd, params); + Mockito.verify(paramProcessWorkerSpy).doAccessChecks(Mockito.any(), Mockito.any()); + } + + @Test + public void doAccessChecksTestChecksCallerAccessToOwnerWhenCmdExtendsBaseAsyncCreateCmd() { + Mockito.lenient().doReturn(owners).when(paramProcessWorkerSpy).getEntityOwners(Mockito.any()); + Mockito.doNothing().when(paramProcessWorkerSpy).checkCallerAccessToEntities(Mockito.any(), Mockito.any(), Mockito.any()); + + paramProcessWorkerSpy.doAccessChecks(new AssociateIPAddrCmd(), entities); + + Mockito.verify(accountManagerMock).checkAccess(callingAccountMock, null, false, owners); + } + + @Test + public void doAccessChecksTestChecksCallerAccessToEntities() { + Mockito.lenient().doReturn(owners).when(paramProcessWorkerSpy).getEntityOwners(Mockito.any()); + Mockito.doNothing().when(paramProcessWorkerSpy).checkCallerAccessToEntities(Mockito.any(), Mockito.any(), Mockito.any()); + + paramProcessWorkerSpy.doAccessChecks(new AssociateIPAddrCmd(), entities); + + Mockito.verify(paramProcessWorkerSpy).checkCallerAccessToEntities(callingAccountMock, owners, entities); + } + + @Test + public void getEntityOwnersTestReturnsAccountsWhenCmdHasMultipleEntityOwners() { + Mockito.when(baseCmdMock.getEntityOwnerIds()).thenReturn(List.of(1L, 2L)); + Mockito.doReturn(callingAccountMock).when(accountManagerMock).getAccount(1L); + Mockito.doReturn(ownerAccountMock).when(accountManagerMock).getAccount(2L); + + List result = List.of(paramProcessWorkerSpy.getEntityOwners(baseCmdMock)); + + Assert.assertEquals(List.of(callingAccountMock, ownerAccountMock), result); + } + + @Test + public void getEntityOwnersTestReturnsAccountWhenCmdHasOneEntityOwner() { + Mockito.when(baseCmdMock.getEntityOwnerId()).thenReturn(1L); + Mockito.when(baseCmdMock.getEntityOwnerIds()).thenReturn(null); + Mockito.doReturn(ownerAccountMock).when(accountManagerMock).getAccount(1L); + + List result = List.of(paramProcessWorkerSpy.getEntityOwners(baseCmdMock)); + + Assert.assertEquals(List.of(ownerAccountMock), result); + } + + @Test + public void checkCallerAccessToEntitiesTestChecksCallerAccessToOwners() { + entities.put(ownerAccountMock, SecurityChecker.AccessType.UseEntry); + + paramProcessWorkerSpy.checkCallerAccessToEntities(callingAccountMock, owners, entities); + + Mockito.verify(accountManagerMock).checkAccess(callingAccountMock, null, false, owners); + } + + @Test + public void checkCallerAccessToEntitiesTestChecksCallerAccessToResource() { + VMInstanceVO vmInstanceVo = new VMInstanceVO(); + entities.put(vmInstanceVo, SecurityChecker.AccessType.UseEntry); + + paramProcessWorkerSpy.checkCallerAccessToEntities(callingAccountMock, owners, entities); + + Mockito.verify(accountManagerMock).validateAccountHasAccessToResource(callingAccountMock, SecurityChecker.AccessType.UseEntry, vmInstanceVo); } } diff --git a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java index 4cf7413f3f3..bd6632af1ca 100644 --- a/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java +++ b/server/src/test/java/com/cloud/user/MockAccountManagerImpl.java @@ -438,6 +438,11 @@ public class MockAccountManagerImpl extends ManagerBase implements Manager, Acco // TODO Auto-generated method stub } + @Override + public void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource) { + // TODO Auto-generated method stub + } + @Override public Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly) { // TODO Auto-generated method stub diff --git a/server/src/test/java/com/cloud/user/MockUsageEventDao.java b/server/src/test/java/com/cloud/user/MockUsageEventDao.java index fb8c193780f..7e15a4f0aaf 100644 --- a/server/src/test/java/com/cloud/user/MockUsageEventDao.java +++ b/server/src/test/java/com/cloud/user/MockUsageEventDao.java @@ -210,11 +210,6 @@ public class MockUsageEventDao implements UsageEventDao{ return 0; } - @Override - public int expunge(SearchCriteria sc, long limit) { - return 0; - } - @Override public void expunge() { diff --git a/services/console-proxy/pom.xml b/services/console-proxy/pom.xml index 0a724deab60..455bbc58cd5 100644 --- a/services/console-proxy/pom.xml +++ b/services/console-proxy/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack-services - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/services/console-proxy/rdpconsole/pom.xml b/services/console-proxy/rdpconsole/pom.xml index b88153fa132..f5f9cdb9837 100644 --- a/services/console-proxy/rdpconsole/pom.xml +++ b/services/console-proxy/rdpconsole/pom.xml @@ -26,7 +26,7 @@ org.apache.cloudstack cloudstack-service-console-proxy - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/services/console-proxy/server/pom.xml b/services/console-proxy/server/pom.xml index ce6fe5943ad..05f8c21aca4 100644 --- a/services/console-proxy/server/pom.xml +++ b/services/console-proxy/server/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-service-console-proxy - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/services/pom.xml b/services/pom.xml index 1e8fa4a229c..665c2f71d27 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/services/secondary-storage/controller/pom.xml b/services/secondary-storage/controller/pom.xml index 8766791defc..69866f5b1cd 100644 --- a/services/secondary-storage/controller/pom.xml +++ b/services/secondary-storage/controller/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-service-secondary-storage - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/services/secondary-storage/pom.xml b/services/secondary-storage/pom.xml index c4f4650d560..d1caa0323bc 100644 --- a/services/secondary-storage/pom.xml +++ b/services/secondary-storage/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack-services - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/services/secondary-storage/server/pom.xml b/services/secondary-storage/server/pom.xml index 0e72754561c..e38cdd052b3 100644 --- a/services/secondary-storage/server/pom.xml +++ b/services/secondary-storage/server/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-service-secondary-storage - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2HeaderField.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2HeaderField.java new file mode 100644 index 00000000000..4a8e8b51a47 --- /dev/null +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2HeaderField.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.formatinspector; + +public enum Qcow2HeaderField { + MAGIC(0, 4), + VERSION(4, 4), + BACKING_FILE_OFFSET(8, 8), + BACKING_FILE_NAME_LENGTH(16, 4), + CLUSTER_BITS(20, 4), + SIZE(24, 8), + CRYPT_METHOD(32, 4), + L1_SIZE(36, 4), + LI_TABLE_OFFSET(40, 8), + REFCOUNT_TABLE_OFFSET(48, 8), + REFCOUNT_TABLE_CLUSTERS(56, 4), + NB_SNAPSHOTS(60, 4), + SNAPSHOTS_OFFSET(64, 8), + INCOMPATIBLE_FEATURES(72, 8); + + private final int offset; + private final int length; + + Qcow2HeaderField(int offset, int length) { + this.offset = offset; + this.length = length; + } + + public int getLength() { + return length; + } + + public int getOffset() { + return offset; + } +} diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2Inspector.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2Inspector.java new file mode 100644 index 00000000000..9d80064d292 --- /dev/null +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2Inspector.java @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.formatinspector; + +import com.cloud.utils.NumbersUtil; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Class to inspect QCOW2 files/objects. In our context, a QCOW2 might be a threat to the environment if it meets one of the following criteria when coming from external sources + * (like registering or uploading volumes and templates): + *
    + *
  • has a backing file reference;
    • + *
  • has an external data file reference;
    • + *
  • has unknown incompatible features.
    • + *
+ * + * The implementation was done based on the QEMU's official interoperability documentation + * and on the OpenStack's Cinder implementation for Python. + */ +public class Qcow2Inspector { + protected static Logger LOGGER = LogManager.getLogger(Qcow2Inspector.class); + + private static final byte[] QCOW_MAGIC_STRING = ArrayUtils.add("QFI".getBytes(), (byte) 0xfb); + private static final int INCOMPATIBLE_FEATURES_MAX_KNOWN_BIT = 4; + private static final int INCOMPATIBLE_FEATURES_MAX_KNOWN_BYTE = 0; + private static final int EXTERNAL_DATA_FILE_BYTE_POSITION = 7; + private static final int EXTERNAL_DATA_FILE_BIT = 2; + private static final byte EXTERNAL_DATA_FILE_BITMASK = (byte) (1 << EXTERNAL_DATA_FILE_BIT); + + private static final Set SET_OF_HEADER_FIELDS_TO_READ = Set.of(Qcow2HeaderField.MAGIC, + Qcow2HeaderField.VERSION, + Qcow2HeaderField.SIZE, + Qcow2HeaderField.BACKING_FILE_OFFSET, + Qcow2HeaderField.INCOMPATIBLE_FEATURES); + + /** + * Validates if the file is a valid and allowed QCOW2 (i.e.: does not contain external references). + * @param filePath Path of the file to be validated. + * @throws RuntimeException If the QCOW2 file meets one of the following criteria: + *
    + *
  • has a backing file reference;
    • + *
  • has an external data file reference;
    • + *
  • has unknown incompatible features.
    • + *
+ */ + public static void validateQcow2File(String filePath) throws RuntimeException { + LOGGER.info(String.format("Verifying if [%s] is a valid and allowed QCOW2 file .", filePath)); + + Map headerFieldsAndValues; + try (InputStream inputStream = new FileInputStream(filePath)) { + headerFieldsAndValues = unravelQcow2Header(inputStream, filePath); + } catch (IOException ex) { + throw new RuntimeException(String.format("Unable to validate file [%s] due to: ", filePath), ex); + } + + validateQcow2HeaderFields(headerFieldsAndValues, filePath); + + LOGGER.info(String.format("[%s] is a valid and allowed QCOW2 file.", filePath)); + } + + /** + * Unravels the QCOW2 header in a serial fashion, iterating through the {@link Qcow2HeaderField}, reading the fields specified in + * {@link Qcow2Inspector#SET_OF_HEADER_FIELDS_TO_READ} and skipping the others. + * @param qcow2InputStream InputStream of the QCOW2 being unraveled. + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @return A map of the header fields and their values according to the {@link Qcow2Inspector#SET_OF_HEADER_FIELDS_TO_READ}. + * @throws IOException If the field cannot be read or skipped. + */ + public static Map unravelQcow2Header(InputStream qcow2InputStream, String qcow2LogReference) throws IOException { + Map result = new HashMap<>(); + + LOGGER.debug(String.format("Unraveling QCOW2 [%s] headers.", qcow2LogReference)); + for (Qcow2HeaderField qcow2Header : Qcow2HeaderField.values()) { + if (!SET_OF_HEADER_FIELDS_TO_READ.contains(qcow2Header)) { + skipHeader(qcow2InputStream, qcow2Header, qcow2LogReference); + continue; + } + + byte[] headerValue = readHeader(qcow2InputStream, qcow2Header, qcow2LogReference); + result.put(qcow2Header.name(), headerValue); + } + + return result; + } + + /** + * Skips the field's length in the InputStream. + * @param qcow2InputStream InputStream of the QCOW2 being unraveled. + * @param field Field being skipped (name and length). + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws IOException If the bytes skipped do not match the field length. + */ + protected static void skipHeader(InputStream qcow2InputStream, Qcow2HeaderField field, String qcow2LogReference) throws IOException { + LOGGER.trace(String.format("Skipping field [%s] of QCOW2 [%s].", field, qcow2LogReference)); + + if (qcow2InputStream.skip(field.getLength()) != field.getLength()) { + throw new IOException(String.format("Unable to skip field [%s] of QCOW2 [%s].", field, qcow2LogReference)); + } + } + + /** + * Reads the field's length in the InputStream. + * @param qcow2InputStream InputStream of the QCOW2 being unraveled. + * @param field Field being read (name and length). + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws IOException If the bytes read do not match the field length. + */ + protected static byte[] readHeader(InputStream qcow2InputStream, Qcow2HeaderField field, String qcow2LogReference) throws IOException { + byte[] readBytes = new byte[field.getLength()]; + + LOGGER.trace(String.format("Reading field [%s] of QCOW2 [%s].", field, qcow2LogReference)); + if (qcow2InputStream.read(readBytes) != field.getLength()) { + throw new IOException(String.format("Unable to read field [%s] of QCOW2 [%s].", field, qcow2LogReference)); + } + + LOGGER.trace(String.format("Read %s as field [%s] of QCOW2 [%s].", ArrayUtils.toString(readBytes), field, qcow2LogReference)); + return readBytes; + } + + /** + * Validates the values of the header fields {@link Qcow2HeaderField#MAGIC}, {@link Qcow2HeaderField#BACKING_FILE_OFFSET}, and {@link Qcow2HeaderField#INCOMPATIBLE_FEATURES}. + * @param headerFieldsAndValues A map of the header fields and their values. + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws SecurityException If the QCOW2 does not contain the QCOW magic string or contains a backing file reference or incompatible features. + */ + public static void validateQcow2HeaderFields(Map headerFieldsAndValues, String qcow2LogReference) throws SecurityException{ + byte[] fieldValue = headerFieldsAndValues.get(Qcow2HeaderField.MAGIC.name()); + validateQcowMagicString(fieldValue, qcow2LogReference); + + fieldValue = headerFieldsAndValues.get(Qcow2HeaderField.BACKING_FILE_OFFSET.name()); + validateAbsenceOfBackingFileReference(NumbersUtil.bytesToLong(fieldValue), qcow2LogReference); + + fieldValue = headerFieldsAndValues.get(Qcow2HeaderField.INCOMPATIBLE_FEATURES.name()); + validateAbsenceOfIncompatibleFeatures(fieldValue, qcow2LogReference); + } + + /** + * Verifies if the first 4 bytes of the header are the QCOW magic string. Throws an exception if not. + * @param headerMagicString The first 4 bytes of the header. + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws SecurityException If the header's magic string is not the QCOW magic string. + */ + private static void validateQcowMagicString(byte[] headerMagicString, String qcow2LogReference) throws SecurityException { + LOGGER.debug(String.format("Verifying if [%s] has a valid QCOW magic string.", qcow2LogReference)); + + if (!Arrays.equals(QCOW_MAGIC_STRING, headerMagicString)) { + throw new SecurityException(String.format("[%s] is not a valid QCOW2 because its first 4 bytes are not the QCOW magic string.", qcow2LogReference)); + } + + LOGGER.debug(String.format("[%s] has a valid QCOW magic string.", qcow2LogReference)); + } + + /** + * Verifies if the QCOW2 has a backing file and throws an exception if so. + * @param backingFileOffset The backing file offset value of the QCOW2 header. + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws SecurityException If the QCOW2 has a backing file reference. + */ + private static void validateAbsenceOfBackingFileReference(long backingFileOffset, String qcow2LogReference) throws SecurityException { + LOGGER.debug(String.format("Verifying if [%s] has a backing file reference.", qcow2LogReference)); + + if (backingFileOffset != 0) { + throw new SecurityException(String.format("[%s] has a backing file reference. This can be an attack to the infrastructure; therefore, we will not accept" + + " this QCOW2.", qcow2LogReference)); + } + + LOGGER.debug(String.format("[%s] does not have a backing file reference.", qcow2LogReference)); + } + + /** + * Verifies if the QCOW2 has incompatible features and throw an exception if it has an external data file reference or unknown incompatible features. + * @param incompatibleFeatures The incompatible features bytes of the QCOW2 header. + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws SecurityException If the QCOW2 has an external data file reference or unknown incompatible features. + */ + private static void validateAbsenceOfIncompatibleFeatures(byte[] incompatibleFeatures, String qcow2LogReference) throws SecurityException { + LOGGER.debug(String.format("Verifying if [%s] has incompatible features.", qcow2LogReference)); + + if (NumbersUtil.bytesToLong(incompatibleFeatures) == 0) { + LOGGER.debug(String.format("[%s] does not have incompatible features.", qcow2LogReference)); + return; + } + + LOGGER.debug(String.format("[%s] has incompatible features.", qcow2LogReference)); + + validateAbsenceOfExternalDataFileReference(incompatibleFeatures, qcow2LogReference); + validateAbsenceOfUnknownIncompatibleFeatures(incompatibleFeatures, qcow2LogReference); + } + + /** + * Verifies if the QCOW2 has an external data file reference and throw an exception if so. + * @param incompatibleFeatures The incompatible features bytes of the QCOW2 header. + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws SecurityException If the QCOW2 has an external data file reference. + */ + private static void validateAbsenceOfExternalDataFileReference(byte[] incompatibleFeatures, String qcow2LogReference) throws SecurityException { + LOGGER.debug(String.format("Verifying if [%s] has an external data file reference.", qcow2LogReference)); + + if ((incompatibleFeatures[EXTERNAL_DATA_FILE_BYTE_POSITION] & EXTERNAL_DATA_FILE_BITMASK) != 0) { + throw new SecurityException(String.format("[%s] has an external data file reference. This can be an attack to the infrastructure; therefore, we will discard" + + " this file.", qcow2LogReference)); + } + + LOGGER.info(String.format("[%s] does not have an external data file reference.", qcow2LogReference)); + } + + /** + * Verifies if the QCOW2 has unknown incompatible features and throw an exception if so. + *

+ * Unknown incompatible features are those with bit greater than + * {@link Qcow2Inspector#INCOMPATIBLE_FEATURES_MAX_KNOWN_BIT}, which will be the represented by bytes in positions greater than + * {@link Qcow2Inspector#INCOMPATIBLE_FEATURES_MAX_KNOWN_BYTE} (in Big Endian order). Therefore, we expect that those bytes are always zero. If not, an exception is thrown. + * @param incompatibleFeatures The incompatible features bytes of the QCOW2 header. + * @param qcow2LogReference A reference (like the filename) of the QCOW2 being unraveled to print in the logs and exceptions. + * @throws SecurityException If the QCOW2 has unknown incompatible features. + */ + private static void validateAbsenceOfUnknownIncompatibleFeatures(byte[] incompatibleFeatures, String qcow2LogReference) throws SecurityException { + LOGGER.debug(String.format("Verifying if [%s] has unknown incompatible features [%s].", qcow2LogReference, ArrayUtils.toString(incompatibleFeatures))); + + for (int byteNum = incompatibleFeatures.length - 1; byteNum >= 0; byteNum--) { + int bytePosition = incompatibleFeatures.length - 1 - byteNum; + LOGGER.trace(String.format("Looking for unknown incompatible feature bit in position [%s].", bytePosition)); + + byte bitmask = 0; + if (byteNum == INCOMPATIBLE_FEATURES_MAX_KNOWN_BYTE) { + bitmask = ((1 << INCOMPATIBLE_FEATURES_MAX_KNOWN_BIT) - 1); + } + + LOGGER.trace(String.format("Bitmask for byte in position [%s] is [%s].", bytePosition, Integer.toBinaryString(bitmask))); + + int featureBit = incompatibleFeatures[bytePosition] & ~bitmask; + if (featureBit != 0) { + throw new SecurityException(String.format("Found unknown incompatible feature bit [%s] in byte [%s] of [%s]. This can be an attack to the infrastructure; " + + "therefore, we will discard this QCOW2.", featureBit, bytePosition + Qcow2HeaderField.INCOMPATIBLE_FEATURES.getOffset(), qcow2LogReference)); + } + + LOGGER.trace(String.format("Did not find unknown incompatible feature in position [%s].", bytePosition)); + } + + LOGGER.info(String.format("[%s] does not have unknown incompatible features.", qcow2LogReference)); + } + +} diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 26b9e3ea65e..4cd4e8caa30 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -71,6 +71,7 @@ import org.apache.cloudstack.storage.command.UploadStatusCommand; import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.configdrive.ConfigDriveBuilder; +import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; import org.apache.cloudstack.storage.template.DownloadManager; import org.apache.cloudstack.storage.template.DownloadManagerImpl; import org.apache.cloudstack.storage.template.UploadEntity; @@ -3482,8 +3483,19 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return result; } + String finalFilename = resourcePath + "/" + templateFilename; + + if (ImageStoreUtil.isCorrectExtension(finalFilename, "qcow2")) { + try { + Qcow2Inspector.validateQcow2File(finalFilename); + } catch (RuntimeException e) { + logger.error(String.format("Uploaded file [%s] is not a valid QCOW2.", finalFilename), e); + return "The uploaded file is not a valid QCOW2. Ask the administrator to check the logs for more details."; + } + } + // Set permissions for the downloaded template - File downloadedTemplate = new File(resourcePath + "/" + templateFilename); + File downloadedTemplate = new File(finalFilename); _storage.setWorldReadableAndWriteable(downloadedTemplate); // Set permissions for template/volume.properties diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index ad73a9bc708..e2eecbfcf22 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -16,8 +16,6 @@ // under the License. package org.apache.cloudstack.storage.template; -import static com.cloud.utils.NumbersUtil.toHumanReadableSize; - import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -51,9 +49,10 @@ import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType import org.apache.cloudstack.storage.resource.IpTablesHelper; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; +import org.apache.cloudstack.storage.formatinspector.Qcow2HeaderField; +import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; import org.apache.cloudstack.utils.security.ChecksumValue; import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.commons.lang3.StringUtils; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.to.DataStoreTO; @@ -89,11 +88,14 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Proxy; +import com.cloud.utils.StringUtils; import com.cloud.utils.script.Script; -import com.cloud.utils.storage.QCOW2Utils; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import static com.cloud.utils.NumbersUtil.toHumanReadableSize; + public class DownloadManagerImpl extends ManagerBase implements DownloadManager { protected static Logger LOGGER = LogManager.getLogger(DownloadManagerImpl.class); private String _name; @@ -366,11 +368,17 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager // The QCOW2 is the only format with a header, // and as such can be easily read. - try (InputStream inputStream = td.getS3ObjectInputStream();) { - dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream, false)); - } - catch (IOException e) { - result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage(); + try (InputStream inputStream = td.getS3ObjectInputStream()) { + Map qcow2HeaderFieldsAndValues = Qcow2Inspector.unravelQcow2Header(inputStream, td.getDownloadUrl()); + Qcow2Inspector.validateQcow2HeaderFields(qcow2HeaderFieldsAndValues, td.getDownloadUrl()); + + dnld.setTemplatesize(NumbersUtil.bytesToLong(qcow2HeaderFieldsAndValues.get(Qcow2HeaderField.SIZE.name()))); + } catch (IOException ex) { + result = String.format("Unable to read QCOW2 metadata. Error: %s", ex.getMessage()); + LOGGER.error(result, ex); + } catch (SecurityException ex) { + result = String.format("[%s] is not a valid QCOW2:", td.getDownloadUrl()); + LOGGER.error(result, ex); } } @@ -516,8 +524,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return result; } + String finalFilename = resourcePath + "/" + templateFilename; + + if (ImageFormat.QCOW2.equals(dnld.getFormat())) { + try { + Qcow2Inspector.validateQcow2File(finalFilename); + } catch (RuntimeException e) { + LOGGER.error(String.format("The downloaded file [%s] is not a valid QCOW2.", finalFilename), e); + return "The downloaded file is not a valid QCOW2. Ask the administrator to check the logs for more details."; + } + } + // Set permissions for the downloaded template - File downloadedTemplate = new File(resourcePath + "/" + templateFilename); + File downloadedTemplate = new File(finalFilename); _storage.setWorldReadableAndWriteable(downloadedTemplate); setPermissionsForTheDownloadedTemplate(resourcePath, resourceType); diff --git a/systemvm/debian/opt/cloud/bin/cs/CsFile.py b/systemvm/debian/opt/cloud/bin/cs/CsFile.py index b33dbcff1c2..bb46bab350a 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsFile.py @@ -174,6 +174,6 @@ class CsFile: self.new_config = list(temp_config) def compare(self, o): - result = (isinstance(o, self.__class__) and set(self.config) == set(o.config)) + result = (isinstance(o, self.__class__) and self.config == o.config) logging.debug("Comparison of CsFiles content is ==> %s" % result) return result diff --git a/systemvm/pom.xml b/systemvm/pom.xml index 8fd0b59e703..ef12dfdfe6f 100644 --- a/systemvm/pom.xml +++ b/systemvm/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/test/integration/smoke/test_login.py b/test/integration/smoke/test_login.py index 40d8349a13d..acd6c4334ac 100644 --- a/test/integration/smoke/test_login.py +++ b/test/integration/smoke/test_login.py @@ -133,6 +133,7 @@ class TestLogin(cloudstackTestCase): args["command"] = 'listUsers' args["listall"] = 'true' args["response"] = "json" + args["sessionkey"] = response.json()['loginresponse']['sessionkey'] response = session.get(self.server_url, params=args) self.assertEqual( response.status_code, diff --git a/test/pom.xml b/test/pom.xml index 99efd5ef718..4e00092a905 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 diff --git a/tools/apidoc/pom.xml b/tools/apidoc/pom.xml index f2321df6788..1532ec35006 100644 --- a/tools/apidoc/pom.xml +++ b/tools/apidoc/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloud-tools - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/tools/build/build_asf.sh b/tools/build/build_asf.sh index a68a71c6153..44d41472b05 100755 --- a/tools/build/build_asf.sh +++ b/tools/build/build_asf.sh @@ -64,8 +64,8 @@ echo "Using version: $version" echo "Using source directory: $sourcedir" echo "Using output directory: $outputdir" echo "Using branch: $branch" -if [ "$tag" == "yes" ]; then - if [ "$certid" == "X" ]; then +if [ "$tag" = "yes" ]; then + if [ "$certid" = "X" ]; then echo "Tagging the branch with the version number, and signing the branch with your default certificate." else echo "Tagging the branch with the version number, and signing the branch with certificate ID $certid." @@ -143,7 +143,7 @@ bzip2 $outputdir/apache-cloudstack-$version-src.tar cd $outputdir echo 'armor' -if [ "$certid" == "X" ]; then +if [ "$certid" = "X" ]; then gpg -v --armor --output apache-cloudstack-$version-src.tar.bz2.asc --detach-sig apache-cloudstack-$version-src.tar.bz2 else gpg -v --default-key $certid --armor --output apache-cloudstack-$version-src.tar.bz2.asc --detach-sig apache-cloudstack-$version-src.tar.bz2 @@ -155,7 +155,7 @@ sha512sum apache-cloudstack-$version-src.tar.bz2 > apache-cloudstack-$version-sr echo 'verify' gpg -v --verify apache-cloudstack-$version-src.tar.bz2.asc apache-cloudstack-$version-src.tar.bz2 -if [ "$tag" == "yes" ]; then +if [ "$tag" = "yes" ]; then echo 'tag' cd $sourcedir if [ "$certid" == "X" ]; then @@ -165,7 +165,7 @@ if [ "$tag" == "yes" ]; then fi fi -if [ "$committosvn" == "yes" ]; then +if [ "$committosvn" = "yes" ]; then echo 'committing artifacts to svn' rm -Rf /tmp/cloudstack-dev-dist cd /tmp diff --git a/tools/checkstyle/pom.xml b/tools/checkstyle/pom.xml index b707cba4969..e86359ae968 100644 --- a/tools/checkstyle/pom.xml +++ b/tools/checkstyle/pom.xml @@ -22,7 +22,7 @@ Apache CloudStack Developer Tools - Checkstyle Configuration org.apache.cloudstack checkstyle - 4.20.0.0-SNAPSHOT + 4.20.0.0 UTF-8 diff --git a/tools/devcloud-kvm/pom.xml b/tools/devcloud-kvm/pom.xml index d6266fad7c2..8d201b49fe5 100644 --- a/tools/devcloud-kvm/pom.xml +++ b/tools/devcloud-kvm/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloud-tools - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/tools/devcloud4/pom.xml b/tools/devcloud4/pom.xml index d4f2251ba81..6cbea5ecc11 100644 --- a/tools/devcloud4/pom.xml +++ b/tools/devcloud4/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloud-tools - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 0494e9ae0da..860468f0ec2 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -19,7 +19,7 @@ FROM ubuntu:22.04 -LABEL Vendor="Apache.org" License="ApacheV2" Version="4.20.0.0-SNAPSHOT" Author="Apache CloudStack " +LABEL Vendor="Apache.org" License="ApacheV2" Version="4.20.0.0" Author="Apache CloudStack " ARG DEBIAN_FRONTEND=noninteractive diff --git a/tools/docker/Dockerfile.marvin b/tools/docker/Dockerfile.marvin index b227e92e608..38ff776f0cd 100644 --- a/tools/docker/Dockerfile.marvin +++ b/tools/docker/Dockerfile.marvin @@ -19,11 +19,11 @@ # build for cloudstack_home_dir not this folder FROM python:2 -LABEL Vendor="Apache.org" License="ApacheV2" Version="4.20.0.0-SNAPSHOT" Author="Apache CloudStack " +LABEL Vendor="Apache.org" License="ApacheV2" Version="4.20.0.0" Author="Apache CloudStack " ENV WORK_DIR=/marvin -ENV PKG_URL=https://builds.cloudstack.org/job/build-master-marvin/lastSuccessfulBuild/artifact/tools/marvin/dist/Marvin-4.20.0.0-SNAPSHOT.tar.gz +ENV PKG_URL=https://builds.cloudstack.org/job/build-master-marvin/lastSuccessfulBuild/artifact/tools/marvin/dist/Marvin-4.20.0.0.tar.gz RUN apt-get update && apt-get install -y vim RUN pip install --upgrade paramiko nose requests diff --git a/tools/marvin/pom.xml b/tools/marvin/pom.xml index ea963fc8da7..3ceaffb5eae 100644 --- a/tools/marvin/pom.xml +++ b/tools/marvin/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloud-tools - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/tools/marvin/setup.py b/tools/marvin/setup.py index 0618d84370a..679b1d5920d 100644 --- a/tools/marvin/setup.py +++ b/tools/marvin/setup.py @@ -27,7 +27,7 @@ except ImportError: raise RuntimeError("python setuptools is required to build Marvin") -VERSION = "4.20.0.0-SNAPSHOT" +VERSION = "4.20.0.0" setup(name="Marvin", version=VERSION, diff --git a/tools/pom.xml b/tools/pom.xml index aa736372aff..b2ad9e2aae4 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.20.0.0 ../pom.xml diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index f02aee747eb..6955d83a510 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -1482,8 +1482,8 @@ "label.setting": "Configura\u00e7\u00e3o", "label.settings": "Configura\u00e7\u00f5es", "label.setup": "Configura\u00e7\u00e3o", -"label.shared": "Compatilhado", -"label.sharedexecutable": "Compatilhado", +"label.shared": "Compartilhado", +"label.sharedexecutable": "Compartilhado", "label.sharedmountpoint": "SharedMountPoint", "label.sharedrouterip": "Endere\u00e7os IPv4 para o roteador dentro da rede compartilhada", "label.sharedrouteripv6": "Endere\u00e7os IPv6 para o roteador dentro da rede compartilhada", diff --git a/ui/src/api/index.js b/ui/src/api/index.js index 1db41661276..7ab87780a9d 100644 --- a/ui/src/api/index.js +++ b/ui/src/api/index.js @@ -15,8 +15,13 @@ // specific language governing permissions and limitations // under the License. +import Cookies from 'js-cookie' import { axios, sourceToken } from '@/utils/request' import { message, notification } from 'ant-design-vue' +import { vueProps } from '@/vue-app' +import { + ACCESS_TOKEN +} from '@/store/mutation-types' export function api (command, args = {}, method = 'GET', data = {}) { let params = {} @@ -30,6 +35,11 @@ export function api (command, args = {}, method = 'GET', data = {}) { }) } + const sessionkey = vueProps.$localStorage.get(ACCESS_TOKEN) || Cookies.get('sessionkey') + if (sessionkey) { + args.sessionkey = sessionkey + } + return axios({ params: { ...args @@ -65,7 +75,6 @@ export function login (arg) { } export function logout () { - sourceToken.cancel() message.destroy() notification.destroy() return api('logout') diff --git a/ui/src/components/view/DetailSettings.vue b/ui/src/components/view/DetailSettings.vue index 1581c2daef8..9f808b907c7 100644 --- a/ui/src/components/view/DetailSettings.vue +++ b/ui/src/components/view/DetailSettings.vue @@ -39,7 +39,7 @@ { @@ -188,7 +188,12 @@ export default { this.updateResource(this.resource) }, methods: { - filterOption (input, option) { + filterOption (input, option, filterType) { + if ((filterType === 'key' && !this.newKey) || + (filterType === 'value' && !this.newValue)) { + return true + } + return ( option.value.toUpperCase().indexOf(input.toUpperCase()) >= 0 ) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 5a82c1ce48a..5737126d463 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -542,7 +542,7 @@ export default { if (store.listAllProjects) { fields.push('project') } - if (store.apis.scaleKubernetesCluster.params.filter(x => x.name === 'autoscalingenabled').length > 0) { + if (store.apis.scaleKubernetesCluster?.params?.filter(x => x.name === 'autoscalingenabled').length > 0) { fields.splice(2, 0, 'autoscalingenabled') } fields.push('zonename') diff --git a/ui/src/store/modules/user.js b/ui/src/store/modules/user.js index 22a2fab959d..bd0afaa17bf 100644 --- a/ui/src/store/modules/user.js +++ b/ui/src/store/modules/user.js @@ -444,11 +444,6 @@ const user = { cloudianUrl = state.cloudian.url + 'logout.htm?redirect=' + encodeURIComponent(window.location.href) } - Object.keys(Cookies.get()).forEach(cookieName => { - Cookies.remove(cookieName) - Cookies.remove(cookieName, { path: '/client' }) - }) - commit('SET_TOKEN', '') commit('SET_APIS', {}) commit('SET_PROJECT', {}) @@ -476,6 +471,11 @@ const user = { } }).catch(() => { resolve() + }).finally(() => { + Object.keys(Cookies.get()).forEach(cookieName => { + Cookies.remove(cookieName) + Cookies.remove(cookieName, { path: '/client' }) + }) }) }) }, diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index d6f6880621b..1c2c83a35ce 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -284,7 +284,7 @@ export default { } }, mounted () { - if (this.$store.getters.apis.scaleKubernetesCluster.params.filter(x => x.name === 'nodeids').length > 0 && this.resource.clustertype === 'CloudManaged') { + if (this.$store.getters.apis.scaleKubernetesCluster?.params?.filter(x => x.name === 'nodeids').length > 0 && this.resource.clustertype === 'CloudManaged') { this.vmColumns.push({ key: 'actions', title: this.$t('label.actions'), diff --git a/ui/src/views/compute/StartVirtualMachine.vue b/ui/src/views/compute/StartVirtualMachine.vue index 7438ebc1beb..f229cb44f67 100644 --- a/ui/src/views/compute/StartVirtualMachine.vue +++ b/ui/src/views/compute/StartVirtualMachine.vue @@ -16,7 +16,7 @@ // under the License.