diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 3c9110595a0..5cd67ffe9ba 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -120,7 +120,7 @@ public interface QueryService { ConfigKey UserVMReadOnlyDetails = new ConfigKey<>(String.class, "user.vm.readonly.details", "Advanced", "dataDiskController, rootDiskController", - "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null); + "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null, ""); ConfigKey SortKeyAscending = new ConfigKey<>("Advanced", Boolean.class, "sortkey.algorithm", "true", "Sort algorithm - ascending or descending - to use. For entities that use sort key(template, disk offering, service offering, " + diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 44c0be1cb59..a03b94faa79 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -48,7 +48,7 @@ public interface VolumeDao extends GenericDao, StateDao findIncludingRemovedByInstanceAndType(long id, Volume.Type vType); - List findByInstanceIdAndPoolId(long instanceId, long poolId); + List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId); List findByInstanceIdDestroyed(long vmId); @@ -70,11 +70,11 @@ public interface VolumeDao extends GenericDao, StateDao findCreatedByInstance(long id); - List findByPoolId(long poolId); + List findNonDestroyedVolumesByPoolId(long poolId); VolumeVO findByPoolIdName(long poolId, String name); - List findByPoolId(long poolId, Volume.Type volumeType); + List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType); List findByPoolIdAndState(long poolid, Volume.State state); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 2e263caeea3..286d4ff7542 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -136,7 +136,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByPoolId(long poolId) { + public List findNonDestroyedVolumesByPoolId(long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); @@ -145,7 +145,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByInstanceIdAndPoolId(long instanceId, long poolId) { + public List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("instanceId", instanceId); sc.setParameters("poolId", poolId); @@ -162,7 +162,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByPoolId(long poolId, Volume.Type volumeType) { + public List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); diff --git a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 0b81a25b1cd..15febbe972c 100644 --- a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -231,9 +231,9 @@ public class ConfigDriveBuilder { throw new CloudRuntimeException("Cannot create ISO for config drive using any know tool. Known paths [/usr/bin/genisoimage, /usr/bin/mkisofs, /usr/local/bin/mkisofs]"); } if (!isoCreator.canExecute()) { - throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getCanonicalPath()); + throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getAbsolutePath()); } - return isoCreator.getCanonicalPath(); + return isoCreator.getAbsolutePath(); } /** diff --git a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java index c04ff0a1601..03ceac84399 100644 --- a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java +++ b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java @@ -435,7 +435,7 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(2)).exists(); Mockito.verify(genIsoFileMock).canExecute(); - Mockito.verify(genIsoFileMock).getCanonicalPath(); + Mockito.verify(genIsoFileMock).getAbsolutePath(); } } @@ -475,11 +475,11 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(2)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getAbsolutePath(); } } @@ -509,15 +509,15 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getAbsolutePath(); } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java index 641a2a40dcd..f739fecf9bf 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java @@ -280,7 +280,7 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService { private void updateDataObject(DataObject srcData, DataObject destData) { if (destData instanceof SnapshotInfo) { SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image); - SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcData.getDataStore().getId(), srcData.getId()); + SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, destData.getDataStore().getId(), destData.getId()); if (snapshotStore != null && destSnapshotStore != null) { destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize()); destSnapshotStore.setCreated(snapshotStore.getCreated()); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index c6430bcf9f9..3e1504beb3a 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -296,6 +296,9 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { @Override public boolean isTemplateMarkedForDirectDownload(long templateId) { VMTemplateVO templateVO = imageDataDao.findById(templateId); + if (templateVO == null) { + throw new CloudRuntimeException(String.format("Template not found with ID: %s", templateId)); + } return templateVO.isDirectDownload(); } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 4b82bedefeb..138d469005b 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -1320,9 +1320,10 @@ public class TemplateServiceImpl implements TemplateService { if (_vmTemplateStoreDao.isTemplateMarkedForDirectDownload(tmplt.getId())) { continue; } - tmpltStore = - new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, null, null, null, - TemplateConstants.DEFAULT_SYSTEM_VM_TEMPLATE_PATH + tmplt.getId() + '/', tmplt.getUrl()); + String templateDirectoryPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR; + String installPath = templateDirectoryPath + tmplt.getAccountId() + File.separator + tmplt.getId() + File.separator; + tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, + null, null, null, installPath, tmplt.getUrl()); tmpltStore.setSize(0L); tmpltStore.setPhysicalSize(0); // no size information for // pre-seeded system vm templates diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index d0184359c8b..b6029c27148 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -126,7 +126,7 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { @Override public List getVolumes() { - List volumes = volumeDao.findByPoolId(getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(getId()); List volumeInfos = new ArrayList(); for (VolumeVO volume : volumes) { volumeInfos.add(VolumeObject.getVolumeObject(this, volume)); diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 0ea910fcb6d..c19ec751153 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -267,10 +267,18 @@ public class ConfigKey { static ConfigDepotImpl s_depot = null; - static public void init(ConfigDepotImpl depot) { + private String _defaultValueIfEmpty = null; + + public static void init(ConfigDepotImpl depot) { s_depot = depot; } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, + String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options, String defaultValueIfEmpty) { + this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, displayText, parent, group, subGroup, kind, options); + this._defaultValueIfEmpty = defaultValueIfEmpty; + } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, Scope scope) { this(type, name, category, defaultValue, description, isDynamic, scope, null); } @@ -380,7 +388,19 @@ public class ConfigKey { public T value() { if (_value == null || isDynamic()) { String value = s_depot != null ? s_depot.getConfigStringValue(_name, Scope.Global, null) : null; - _value = valueOf((value == null) ? defaultValue() : value); + + String effective; + if (value != null) { + if (value.isEmpty() && _defaultValueIfEmpty != null) { + effective = _defaultValueIfEmpty; + } else { + effective = value; + } + } else { + effective = _defaultValueIfEmpty != null ? _defaultValueIfEmpty : defaultValue(); + } + + _value = valueOf(effective); } return _value; } @@ -409,6 +429,10 @@ public class ConfigKey { return valueInGlobalOrAvailableParentScope(scope, id); } logger.trace("Scope({}) value for config ({}): {}", scope, _name, _value); + + if (value.isEmpty() && _defaultValueIfEmpty != null) { + return valueOf(_defaultValueIfEmpty); + } return valueOf(value); } diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index c5079a8aa94..705959336f1 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -76,7 +76,7 @@ Requires: sudo Requires: /sbin/service Requires: /sbin/chkconfig Requires: /usr/bin/ssh-keygen -Requires: (genisoimage or mkisofs) +Requires: (genisoimage or mkisofs or xorrisofs) Requires: ipmitool Requires: %{name}-common = %{_ver} Requires: (iptables-services or iptables) diff --git a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java index e93b8df39e9..b01af35725f 100644 --- a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java +++ b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java @@ -123,7 +123,7 @@ public class SiocManagerImpl implements SiocManager { int limitIopsTotal = 0; - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null && volumes.size() > 0) { Set instanceIds = new HashSet<>(); diff --git a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java index 951741cbb59..51455429039 100644 --- a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java +++ b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java @@ -362,7 +362,7 @@ public class OvmResourceBase implements ServerResource, HypervisorResource { sshConnection = SSHCmdHelper.acquireAuthorizedConnection(_ip, _username, _password); if (sshConnection == null) { - throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s)", _ip, _username)); } if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "sh /usr/bin/configureOvm.sh postSetup")) { diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index dcf84525748..62393610499 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -563,7 +563,7 @@ public class DateraPrimaryDataStoreDriver implements PrimaryDataStoreDriver { private long getUsedBytes(StoragePool storagePool, long volumeIdToIgnore) { long usedSpaceBytes = 0; - List lstVolumes = _volumeDao.findByPoolId(storagePool.getId(), null); + List lstVolumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (lstVolumes != null) { for (VolumeVO volume : lstVolumes) { diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java index a0dc23da486..08bc89737f2 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java @@ -247,7 +247,7 @@ public class DateraHostListener implements HypervisorHostListener { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -317,7 +317,7 @@ public class DateraHostListener implements HypervisorHostListener { StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index 6cc76d99d9e..1e927e20168 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -433,7 +433,7 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { public long getUsedIops(StoragePool storagePool) { long usedIops = 0; - List volumes = volumeDao.findByPoolId(storagePool.getId(), null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java index 052191128f1..c961c926739 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java @@ -199,7 +199,7 @@ public class SolidFireHostListener implements HypervisorHostListener { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -230,7 +230,7 @@ public class SolidFireHostListener implements HypervisorHostListener { StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index c305c393c9b..f0398e91957 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -1355,7 +1355,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { return volumeStats; } } else { - List volumes = volumeDao.findByPoolId(storagePool.getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId()); for (VolumeVO volume : volumes) { if (volume.getPath() != null && volume.getPath().equals(volumeId)) { long size = volume.getSize(); diff --git a/scripts/storage/secondary/cloud-install-sys-tmplt b/scripts/storage/secondary/cloud-install-sys-tmplt index ad976c502c6..fc09dc968ff 100755 --- a/scripts/storage/secondary/cloud-install-sys-tmplt +++ b/scripts/storage/secondary/cloud-install-sys-tmplt @@ -44,6 +44,7 @@ failed() { } #set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments mflag= fflag= ext="vhd" diff --git a/scripts/storage/secondary/setup-sysvm-tmplt b/scripts/storage/secondary/setup-sysvm-tmplt index 06f0586fe34..63006cc4e4c 100755 --- a/scripts/storage/secondary/setup-sysvm-tmplt +++ b/scripts/storage/secondary/setup-sysvm-tmplt @@ -19,6 +19,7 @@ # Usage: e.g. failed $? "this is an error" set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments failed() { local returnval=$1 diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index e881bd7ac6e..37710c1f44f 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -3336,6 +3336,7 @@ Configurable, StateListener rootDisks = volumeDao.findByPoolId(poolId); - List dataVolumes = volumeDao.findByPoolId(poolId, Volume.Type.DATADISK); + List rootDisks = volumeDao.findNonDestroyedVolumesByPoolId(poolId); + List dataVolumes = volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK); List volumes = new ArrayList<>(); addVolumesToList(volumes, rootDisks); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index a0d9c09c306..b6637aa87fc 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1649,7 +1649,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc List pools = _storagePoolDao.listAll(); for (StoragePoolVO pool : pools) { - List volumes = _volsDao.findByPoolId(pool.getId(), null); + List volumes = _volsDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); for (VolumeVO volume : volumes) { if (!List.of(ImageFormat.QCOW2, ImageFormat.VHD, ImageFormat.OVA, ImageFormat.RAW).contains(volume.getFormat()) && !List.of(Storage.StoragePoolType.PowerFlex, Storage.StoragePoolType.FiberChannel).contains(pool.getPoolType())) { diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index e21f9650d4c..6f289f8e6bb 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1801,17 +1801,21 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C protected String getStoragePoolNonDestroyedVolumesLog(long storagePoolId) { StringBuilder sb = new StringBuilder(); - List nonDestroyedVols = volumeDao.findByPoolId(storagePoolId, null); + List nonDestroyedVols = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); VMInstanceVO volInstance; List logMessageInfo = new ArrayList<>(); sb.append("["); for (VolumeVO vol : nonDestroyedVols) { - volInstance = _vmInstanceDao.findById(vol.getInstanceId()); - if (volInstance != null) { - logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + if (vol.getInstanceId() != null) { + volInstance = _vmInstanceDao.findById(vol.getInstanceId()); + if (volInstance != null) { + logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + } else { + logMessageInfo.add(String.format("Volume [%s] (attached VM with ID [%d] doesn't exists)", vol.getUuid(), vol.getInstanceId())); + } } else { - logMessageInfo.add(String.format("Volume [%s]", vol.getUuid())); + logMessageInfo.add(String.format("Volume [%s] (not attached to any VM)", vol.getUuid())); } } sb.append(String.join(", ", logMessageInfo)); @@ -3024,7 +3028,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C for (String childDatastoreUUID : childDatastoreUUIDs) { StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childDatastoreUUID); - List allVolumes = volumeDao.findByPoolId(dataStoreVO.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(dataStoreVO.getId()); allVolumes.removeIf(volumeVO -> volumeVO.getInstanceId() == null); allVolumes.removeIf(volumeVO -> volumeVO.getState() != Volume.State.Ready); for (VolumeVO volume : allVolumes) { diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 594bf3f90bb..f03a591a261 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -128,7 +128,7 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { boolean restart = !CollectionUtils.isEmpty(upPools); // 2. Get a list of all the ROOT volumes within this storage pool - List allVolumes = volumeDao.findByPoolId(pool.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId()); // 3. Enqueue to the work queue enqueueMigrationsForVolumes(allVolumes, pool); // 4. Process the queue diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 17a893c4400..16aad891f4e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2309,7 +2309,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private List getVolumesByHost(HostVO host, StoragePool pool){ List vmsPerHost = _vmInstanceDao.listByHostId(host.getId()); return vmsPerHost.stream() - .flatMap(vm -> _volsDao.findByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> + .flatMap(vm -> _volsDao.findNonDestroyedVolumesByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> vol.getState() == Volume.State.Ready ? (vol.getFormat() == ImageFormat.OVA ? vol.getChainInfo() : vol.getPath()) : null).filter(Objects::nonNull)) .collect(Collectors.toList()); } diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 15d23775742..6ed0774a423 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -225,8 +225,8 @@ public class ResourceManagerImplTest { rootDisks = Arrays.asList(rootDisk1, rootDisk2); dataDisks = Collections.singletonList(dataDisk); - when(volumeDao.findByPoolId(poolId)).thenReturn(rootDisks); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(rootDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); } @After @@ -591,22 +591,22 @@ public class ResourceManagerImplTest { @Test public void testDestroyLocalStoragePoolVolumesOnlyRootDisks() { - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(rootDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesOnlyDataDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(dataDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesNoDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, never()).updateAndRemoveVolume(any(VolumeVO.class)); } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 576c32c08ba..8f88800d549 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -575,7 +575,7 @@ public class StorageManagerImplTest { } @Test - public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumesReturnLog() { + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMAttachedLogs() { Mockito.doReturn(1L).when(storagePoolVOMock).getId(); Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); @@ -583,7 +583,7 @@ public class StorageManagerImplTest { Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); - Mockito.when(_volumeDao.findByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); @@ -592,6 +592,58 @@ public class StorageManagerImplTest { Assert.assertEquals(expected, log); } + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMLogForOneVolume() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (attached to VM [%s])]", volume1VOMock.getUuid(), volume2VOMock.getUuid(), vmInstanceVOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_NotAttachedLogs() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(null).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (not attached to any VM)]", volume1VOMock.getUuid(), volume2VOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMNotExistsLog() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(null).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (attached VM with ID [%d] doesn't exists), Volume [%s] (attached VM with ID [%d] doesn't exists)]", + volume1VOMock.getUuid(), volume1VOMock.getInstanceId(), volume2VOMock.getUuid(), volume2VOMock.getInstanceId()); + + Assert.assertEquals(expected, log); + } + private ChangeStoragePoolScopeCmd mockChangeStoragePooolScopeCmd(String newScope) { ChangeStoragePoolScopeCmd cmd = new ChangeStoragePoolScopeCmd(); ReflectionTestUtils.setField(cmd, "id", 1L);