From b2db8979f2249b0b6c0d8d0dfe40e05ba9ef9649 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 31 Jul 2019 15:37:59 +0530 Subject: [PATCH] server: fix for respecting secondary storage threshold limit (#3480) Retrieval of an image store using ImageStoreProviderManager has been refactored by introducing three different methods, DataStore getRandomImageStore(List imageStores); To get an image store for reading purpose. Threshold capacity check will not be used here. DataStore getImageStoreWithFreeCapacity(List imageStores); To get an image store for reading purpose. Threshold capacity check will be used here and the store with max free space will be returned. If no store with filled storage less than the threshold is found, the NULL value will be returned. List listImageStoresWithFreeCapacity(List imageStores); To get a list of image stores for writing purpose which fulfills threshold capacity check. Correspondingly DataStoreManager methods have been refactored to return similar values for a given zone. Fixes #3287 - NULL value will be returned when secondary storage is needed for writing but there is not store with free space. Fixes #3041 - Rather than returning random secondary storage for writing, storage with max. free space will be returned. Fixes #3478 - For migration on VMware, all writable secondary storage will be mounted while preparation. Signed-off-by: Abhishek Kumar --- .../api/storage/DataStoreManager.java | 6 +- .../StorageCacheRandomAllocator.java | 4 +- .../motion/AncientDataMotionStrategy.java | 4 +- ...vmNonManagedStorageDataMotionStrategy.java | 30 ++++----- ...NonManagedStorageSystemDataMotionTest.java | 22 +++---- .../ImageStoreProviderManagerImpl.java | 58 ++++++++++++++---- .../storage/snapshot/SnapshotServiceImpl.java | 4 +- .../datastore/DataStoreManagerImpl.java | 24 +++++++- .../datastore/ImageStoreProviderManager.java | 35 ++++++++++- .../hyperv/manager/HypervManagerImpl.java | 2 +- .../vmware/manager/VmwareManager.java | 13 ++-- .../vmware/manager/VmwareManagerImpl.java | 31 +++++++++- .../vmware/resource/VmwareResource.java | 61 +++++++++++-------- .../element/ConfigDriveNetworkElement.java | 19 ++++-- .../java/com/cloud/server/StatsCollector.java | 12 ++++ .../cloud/storage/VolumeApiServiceImpl.java | 5 +- .../storage/upload/UploadMonitorImpl.java | 6 +- .../template/HypervisorTemplateAdapter.java | 4 +- .../cloud/template/TemplateManagerImpl.java | 8 +-- .../ConfigDriveNetworkElementTest.java | 2 +- .../SecondaryStorageManagerImpl.java | 9 ++- 21 files changed, 262 insertions(+), 97 deletions(-) diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 5ebef031c5c..ad5b1622cd2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -33,7 +33,11 @@ public interface DataStoreManager { List getImageStoresByScope(ZoneScope scope); - DataStore getImageStore(long zoneId); + DataStore getRandomImageStore(long zoneId); + + DataStore getImageStoreWithFreeCapacity(long zoneId); + + List listImageStoresWithFreeCapacity(long zoneId); List getImageCacheStores(Scope scope); diff --git a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java index c9832bf1169..22b3f46a946 100644 --- a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java +++ b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java @@ -62,7 +62,7 @@ public class StorageCacheRandomAllocator implements StorageCacheAllocator { return null; } - return imageStoreMgr.getImageStore(cacheStores); + return imageStoreMgr.getImageStoreWithFreeCapacity(cacheStores); } @Override @@ -88,6 +88,6 @@ public class StorageCacheRandomAllocator implements StorageCacheAllocator { } } } - return imageStoreMgr.getImageStore(cacheStores); + return imageStoreMgr.getImageStoreWithFreeCapacity(cacheStores); } } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 7b526458835..39851b47b62 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -328,8 +328,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { if (cacheStore == null) { // need to find a nfs or cifs image store, assuming that can't copy volume // directly to s3 - ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStore(destScope.getScopeId()); - if (!imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { + ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId()); + if (imageStore == null || !imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { s_logger.debug("can't find a nfs (or cifs) image store to satisfy the need for a staging store"); return null; } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java index e42715a1e6d..e6b5c85b924 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java @@ -24,18 +24,17 @@ import java.util.Set; import javax.inject.Inject; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.datastore.DataStoreManagerImpl; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; @@ -47,6 +46,8 @@ import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; @@ -56,7 +57,6 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; -import org.apache.commons.collections.MapUtils; /** * Extends {@link StorageSystemDataMotionStrategy}, allowing KVM hosts to migrate VMs with the ROOT volume on a non managed local storage pool. @@ -197,19 +197,21 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot Host destHost) { VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId()); if (sourceVolumeTemplateStoragePoolVO == null && destStoragePool.getPoolType() == StoragePoolType.Filesystem) { - DataStore sourceTemplateDataStore = dataStoreManagerImpl.getImageStore(srcVolumeInfo.getDataCenterId()); - TemplateInfo sourceTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), sourceTemplateDataStore); - TemplateObjectTO sourceTemplate = new TemplateObjectTO(sourceTemplateInfo); + DataStore sourceTemplateDataStore = dataStoreManagerImpl.getRandomImageStore(srcVolumeInfo.getDataCenterId()); + if (sourceTemplateDataStore != null) { + TemplateInfo sourceTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), sourceTemplateDataStore); + TemplateObjectTO sourceTemplate = new TemplateObjectTO(sourceTemplateInfo); - LOGGER.debug(String.format("Could not find template [id=%s, name=%s] on the storage pool [id=%s]; copying the template to the target storage pool.", - srcVolumeInfo.getTemplateId(), sourceTemplateInfo.getName(), destDataStore.getId())); + LOGGER.debug(String.format("Could not find template [id=%s, name=%s] on the storage pool [id=%s]; copying the template to the target storage pool.", + srcVolumeInfo.getTemplateId(), sourceTemplateInfo.getName(), destDataStore.getId())); - TemplateInfo destTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), destDataStore); - final TemplateObjectTO destTemplate = new TemplateObjectTO(destTemplateInfo); - Answer copyCommandAnswer = sendCopyCommand(destHost, sourceTemplate, destTemplate, destDataStore); + TemplateInfo destTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), destDataStore); + final TemplateObjectTO destTemplate = new TemplateObjectTO(destTemplateInfo); + Answer copyCommandAnswer = sendCopyCommand(destHost, sourceTemplate, destTemplate, destDataStore); - if (copyCommandAnswer != null && copyCommandAnswer.getResult()) { - updateTemplateReferenceIfSuccessfulCopy(srcVolumeInfo, srcStoragePool, destTemplateInfo, destDataStore); + if (copyCommandAnswer != null && copyCommandAnswer.getResult()) { + updateTemplateReferenceIfSuccessfulCopy(srcVolumeInfo, srcStoragePool, destTemplateInfo, destDataStore); + } } } } diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 5b8d3aff2b8..3dfc4af409f 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -18,13 +18,12 @@ */ package org.apache.cloudstack.storage.motion; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + import java.util.HashMap; import java.util.Map; -import com.cloud.host.Host; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -58,21 +57,22 @@ import com.cloud.agent.api.MigrateCommand; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.CloudException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; -import com.cloud.storage.StoragePool; -import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.StoragePool; +import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.when; - @RunWith(MockitoJUnitRunner.class) public class KvmNonManagedStorageSystemDataMotionTest { @@ -353,7 +353,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { Mockito.when(sourceTemplateInfo.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(vmTemplatePoolDao.findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong())).thenReturn(vmTemplateStoragePoolVO); - Mockito.when(dataStoreManagerImpl.getImageStore(Mockito.anyLong())).thenReturn(sourceTemplateDataStore); + Mockito.when(dataStoreManagerImpl.getRandomImageStore(Mockito.anyLong())).thenReturn(sourceTemplateDataStore); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore))).thenReturn(sourceTemplateInfo); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore))).thenReturn(sourceTemplateInfo); kvmNonManagedStorageDataMotionStrategy.copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, srcStoragePool, destDataStore, destStoragePool, destHost); @@ -362,7 +362,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { InOrder verifyInOrder = Mockito.inOrder(vmTemplatePoolDao, dataStoreManagerImpl, templateDataFactory, kvmNonManagedStorageDataMotionStrategy); verifyInOrder.verify(vmTemplatePoolDao, Mockito.times(1)).findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong()); - verifyInOrder.verify(dataStoreManagerImpl, Mockito.times(times)).getImageStore(Mockito.anyLong()); + verifyInOrder.verify(dataStoreManagerImpl, Mockito.times(times)).getRandomImageStore(Mockito.anyLong()); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore)); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore)); verifyInOrder.verify(kvmNonManagedStorageDataMotionStrategy, Mockito.times(times)).sendCopyCommand(Mockito.eq(destHost), Mockito.any(TemplateObjectTO.class), diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index cb9a97e5965..80e5b38f1f7 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -20,17 +20,14 @@ package org.apache.cloudstack.storage.image.manager; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import javax.annotation.PostConstruct; import javax.inject.Inject; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider; @@ -42,6 +39,8 @@ import org.apache.cloudstack.storage.image.ImageStoreDriver; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; import org.apache.cloudstack.storage.image.store.ImageStoreImpl; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.server.StatsCollector; import com.cloud.storage.ScopeType; @@ -144,19 +143,56 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager } @Override - public DataStore getImageStore(List imageStores) { + public DataStore getRandomImageStore(List imageStores) { if (imageStores.size() > 1) { - Collections.shuffle(imageStores); // Randomize image store list. - Iterator i = imageStores.iterator(); - DataStore imageStore = null; - while(i.hasNext()) { - imageStore = i.next(); + Collections.shuffle(imageStores); + } + return imageStores.get(0); + } + + @Override + public DataStore getImageStoreWithFreeCapacity(List imageStores) { + if (imageStores.size() > 1) { + imageStores.sort(new Comparator() { // Sort data stores based on free capacity + @Override + public int compare(DataStore store1, DataStore store2) { + return Long.compare(_statsCollector.imageStoreCurrentFreeCapacity(store1), + _statsCollector.imageStoreCurrentFreeCapacity(store2)); + } + }); + for (DataStore imageStore : imageStores) { // Return image store if used percentage is less then threshold value i.e. 90%. if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { return imageStore; } } + } else if (imageStores.size() == 1) { + if (_statsCollector.imageStoreHasEnoughCapacity(imageStores.get(0))) { + return imageStores.get(0); + } } - return imageStores.get(0); + + // No store with space found + s_logger.error(String.format("Can't find an image storage in zone with less than %d usage", + Math.round(_statsCollector.getImageStoreCapacityThreshold()*100))); + return null; + } + + @Override + public List listImageStoresWithFreeCapacity(List imageStores) { + List stores = new ArrayList<>(); + for (DataStore imageStore : imageStores) { + // Return image store if used percentage is less then threshold value i.e. 90%. + if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + stores.add(imageStore); + } + } + + // No store with space found + if (stores.isEmpty()) { + s_logger.error(String.format("Can't find image storage in zone with less than %d usage", + Math.round(_statsCollector.getImageStoreCapacityThreshold() * 100))); + } + return stores; } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 9c513709677..51a2741dddb 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -240,7 +240,7 @@ public class SnapshotServiceImpl implements SnapshotService { fullSnapshot = snapshotFullBackup; } if (fullSnapshot) { - return dataStoreMgr.getImageStore(snapshot.getDataCenterId()); + return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); } else { SnapshotInfo parentSnapshot = snapshot.getParent(); // Note that DataStore information in parentSnapshot is for primary @@ -251,7 +251,7 @@ public class SnapshotServiceImpl implements SnapshotService { parentSnapshotOnBackupStore = _snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), DataStoreRole.Image); } if (parentSnapshotOnBackupStore == null) { - return dataStoreMgr.getImageStore(snapshot.getDataCenterId()); + return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); } return dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole()); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index f64037619a8..51421e4cd3d 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -73,12 +73,30 @@ public class DataStoreManagerImpl implements DataStoreManager { } @Override - public DataStore getImageStore(long zoneId) { + public DataStore getRandomImageStore(long zoneId) { List stores = getImageStoresByScope(new ZoneScope(zoneId)); if (stores == null || stores.size() == 0) { return null; } - return imageDataStoreMgr.getImageStore(stores); + return imageDataStoreMgr.getRandomImageStore(stores); + } + + @Override + public DataStore getImageStoreWithFreeCapacity(long zoneId) { + List stores = getImageStoresByScope(new ZoneScope(zoneId)); + if (stores == null || stores.size() == 0) { + return null; + } + return imageDataStoreMgr.getImageStoreWithFreeCapacity(stores); + } + + @Override + public List listImageStoresWithFreeCapacity(long zoneId) { + List stores = getImageStoresByScope(new ZoneScope(zoneId)); + if (stores == null || stores.size() == 0) { + return null; + } + return imageDataStoreMgr.listImageStoresWithFreeCapacity(stores); } @Override @@ -110,7 +128,7 @@ public class DataStoreManagerImpl implements DataStoreManager { if (stores == null || stores.size() == 0) { return null; } - return imageDataStoreMgr.getImageStore(stores); + return imageDataStoreMgr.getImageStoreWithFreeCapacity(stores); } @Override diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 70b7a7c3c68..01f2100f77f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -42,5 +42,38 @@ public interface ImageStoreProviderManager { boolean registerDriver(String uuid, ImageStoreDriver driver); - DataStore getImageStore(List imageStores); + /** + * Return a random DataStore from the a list of DataStores. + * + * @param imageStores the list of image stores from which a random store + * to be returned + * @return random DataStore + */ + DataStore getRandomImageStore(List imageStores); + + /** + * Return a DataStore which has free capacity. Stores will be sorted + * based on their free space and capacity check will be done based on + * the predefined threshold value. If a store is full beyond the + * threshold it won't be considered for response. First store in the + * sorted list free capacity will be returned. When there is no store + * with free capacity in the list a null value will be returned. + * + * @param imageStores the list of image stores from which stores with free + * capacity stores to be returned + * @return the DataStore which has free capacity + */ + DataStore getImageStoreWithFreeCapacity(List imageStores); + + /** + * Return a list of DataStore which have free capacity. Free capacity check + * will be done based on the predefined threshold value. If a store is full + * beyond the threshold it won't be considered for response. An empty list + * will be returned when no store in the parameter list has free capacity. + * + * @param imageStores the list of image stores from which stores with free + * capacity stores to be returned + * @return the list of DataStore which have free capacity + */ + List listImageStoresWithFreeCapacity(List imageStores); } diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java index 9d63726d347..09e45447944 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java @@ -137,7 +137,7 @@ public class HypervManagerImpl implements HypervManager { private String getSecondaryStorageStoreUrl(long zoneId) { String secUrl = null; - DataStore secStore = _dataStoreMgr.getImageStore(zoneId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (secStore != null) { secUrl = secStore.getUri(); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java index efdbc724fbd..8cc328a7a38 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -16,16 +16,17 @@ // under the License. package com.cloud.hypervisor.vmware.manager; +import java.io.File; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.framework.config.ConfigKey; + import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.utils.Pair; import com.vmware.vim25.ManagedObjectReference; -import org.apache.cloudstack.framework.config.ConfigKey; - -import java.io.File; -import java.util.List; -import java.util.Map; public interface VmwareManager { public final String CONTEXT_STOCK_NAME = "vmwareMgr"; @@ -65,6 +66,8 @@ public interface VmwareManager { Pair getSecondaryStorageStoreUrlAndId(long dcId); + List> getSecondaryStorageStoresUrlAndIdList(long dcId); + File getSystemVMKeyFile(); VmwareStorageManager getStorageManager(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 23758d512ba..1d3c1ad9d69 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -48,10 +48,12 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; +import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.log4j.Logger; +import com.amazonaws.util.CollectionUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; @@ -62,7 +64,6 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.cluster.ClusterManager; -import org.apache.cloudstack.management.ManagementServerHost; import com.cloud.cluster.dao.ManagementServerHostPeerDao; import com.cloud.configuration.Config; import com.cloud.dc.ClusterDetailsDao; @@ -492,7 +493,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String secUrl = null; Long secId = null; - DataStore secStore = _dataStoreMgr.getImageStore(dcId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dcId); if (secStore != null) { secUrl = secStore.getUri(); secId = secStore.getId(); @@ -513,6 +514,32 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return new Pair(secUrl, secId); } + @Override + public List> getSecondaryStorageStoresUrlAndIdList(long dcId) { + List> urlIdList = new ArrayList<>(); + List secStores = _dataStoreMgr.listImageStoresWithFreeCapacity(dcId); + if (!CollectionUtils.isNullOrEmpty(secStores)) { + for (DataStore secStore : secStores) { + if (secStore != null) { + urlIdList.add(new Pair<>(secStore.getUri(), secStore.getId())); + } + } + } + + if (urlIdList.isEmpty()) { + // we are using non-NFS image store, then use cache storage instead + s_logger.info("Secondary storage is not NFS, we need to use staging storage"); + DataStore cacheStore = _dataStoreMgr.getImageCacheStore(dcId); + if (cacheStore != null) { + urlIdList.add(new Pair<>(cacheStore.getUri(), cacheStore.getId())); + } else { + s_logger.warn("No staging storage is found when non-NFS secondary storage is used"); + } + } + + return urlIdList; + } + @Override public String getServiceConsolePortGroupName() { return _serviceConsoleName; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 141f2f63555..834900f67d3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -3868,19 +3868,24 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa prepareNetworkFromNicInfo(new HostMO(getServiceContext(), _morHyperHost), nic, false, cmd.getVirtualMachine().getType()); } - Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); - String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); - if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; - throw new Exception(msg); - } - mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + List> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId)); + for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { + String secStoreUrl = secStoreUrlAndId.first(); + Long secStoreId = secStoreUrlAndId.second(); + if (secStoreUrl == null) { + String msg = String.format("Secondary storage for dc %s is not ready yet?", _dcId); + throw new Exception(msg); + } - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); - if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; - throw new Exception(msg); + if (vm.getType() != VirtualMachine.Type.User) { + mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + } + + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); + if (morSecDs == null) { + String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + throw new Exception(msg); + } } return new PrepareForMigrationAnswer(cmd); } catch (Throwable e) { @@ -4247,19 +4252,25 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa prepareNetworkFromNicInfo(new HostMO(getServiceContext(), morTgtHost), nic, false, vmTo.getType()); } - // Ensure secondary storage mounted on target host - Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); - String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); - if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; - throw new Exception(msg); - } - mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnSpecificHost(secStoreUrl, tgtHyperHost); - if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; - throw new Exception(msg); + // Ensure all secondary storage mounted on target host + List> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId)); + for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { + String secStoreUrl = secStoreUrlAndId.first(); + Long secStoreId = secStoreUrlAndId.second(); + if (secStoreUrl == null) { + String msg = String.format("Secondary storage for dc %s is not ready yet?", _dcId); + throw new Exception(msg); + } + + if (vmTo.getType() != VirtualMachine.Type.User) { + mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + } + + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnSpecificHost(secStoreUrl, tgtHyperHost); + if (morSecDs == null) { + String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + throw new Exception(msg); + } } if (srcHostApiVersion.compareTo("5.1") < 0) { diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 76e4fc03ce7..e2c3ca7b597 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -23,7 +23,6 @@ import java.util.Set; import javax.inject.Inject; -import com.cloud.storage.StoragePool; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -59,6 +58,7 @@ import com.cloud.offering.NetworkOffering; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; +import com.cloud.storage.StoragePool; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSCategoryDao; @@ -308,7 +308,12 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle if (nic.isDefaultNic() && _networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.ConfigDrive)) { LOG.trace(String.format("[prepareMigration] for vm: %s", vm.getInstanceName())); final DataStore dataStore = findDataStore(vm, dest); - addConfigDriveDisk(vm, dataStore); + + try { + addConfigDriveDisk(vm, dataStore); + } catch (ResourceUnavailableException e) { + LOG.error("Failed to add config disk drive due to: ", e); + } return false; } else return true; @@ -332,7 +337,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle dataStore = pickExistingRootVolumeFromDataStore(profile, dataStore); } } else { - dataStore = _dataStoreMgr.getImageStore(dest.getDataCenter().getId()); + dataStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dest.getDataCenter().getId()); } return dataStore; } @@ -444,7 +449,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } private boolean deleteConfigDriveIso(final VirtualMachine vm) throws ResourceUnavailableException { - DataStore dataStore = _dataStoreMgr.getImageStore(vm.getDataCenterId()); + DataStore dataStore = _dataStoreMgr.getImageStoreWithFreeCapacity(vm.getDataCenterId()); Long agentId = findAgentIdForImageStore(dataStore); if (VirtualMachineManager.VmConfigDriveOnPrimaryPool.value()) { @@ -473,7 +478,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle return true; } - private void addConfigDriveDisk(final VirtualMachineProfile profile, final DataStore dataStore) { + private void addConfigDriveDisk(final VirtualMachineProfile profile, final DataStore dataStore) throws ResourceUnavailableException { boolean isoAvailable = false; final String isoPath = ConfigDrive.createConfigDrivePath(profile.getInstanceName()); for (DiskTO dataTo : profile.getDisks()) { @@ -484,6 +489,10 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } if (!isoAvailable) { TemplateObjectTO dataTO = new TemplateObjectTO(); + if (dataStore == null) { + throw new ResourceUnavailableException("Config drive disk add failed, datastore not available", + ConfigDriveNetworkElement.class, 0L); + } dataTO.setDataStore(dataStore.getTO()); dataTO.setUuid(profile.getUuid()); dataTO.setPath(isoPath); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index b81507ad52b..b2ccfe274a5 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1362,6 +1362,9 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc } public boolean imageStoreHasEnoughCapacity(DataStore imageStore) { + if (!_storageStats.keySet().contains(imageStore.getId())) { // Stats not available for this store yet, can be a new store. Better to assume it has enough capacity? + return true; + } StorageStats imageStoreStats = _storageStats.get(imageStore.getId()); if (imageStoreStats != null && (imageStoreStats.getByteUsed() / (imageStoreStats.getCapacityBytes() * 1.0)) <= _imageStoreCapacityThreshold) { return true; @@ -1369,6 +1372,11 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc return false; } + public long imageStoreCurrentFreeCapacity(DataStore imageStore) { + StorageStats imageStoreStats = _storageStats.get(imageStore.getId()); + return imageStoreStats != null ? Math.max(0, imageStoreStats.getCapacityBytes() - imageStoreStats.getByteUsed()) : 0; + } + /** * Sends VMs metrics to the configured graphite host. */ @@ -1574,4 +1582,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc public ConfigKey[] getConfigKeys() { return new ConfigKey[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri}; } + + public double getImageStoreCapacityThreshold() { + return _imageStoreCapacityThreshold; + } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 2a3d3957828..ddd84137ad9 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2674,7 +2674,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Volume to be extracted has been removed or not in right state!"); } // perform extraction - ImageStoreEntity secStore = (ImageStoreEntity)dataStoreMgr.getImageStore(zoneId); + ImageStoreEntity secStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + if (secStore == null) { + throw new InvalidParameterValueException(String.format("Secondary storage to satisfy storage needs cannot be found for zone: %d", zoneId)); + } String value = _configDao.getValue(Config.CopyVolumeWait.toString()); NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); diff --git a/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java index e8f2980a082..64ada6dc309 100644 --- a/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java @@ -176,7 +176,11 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE; - DataStore secStore = storeMgr.getImageStore(dataCenterId); + DataStore secStore = storeMgr.getImageStoreWithFreeCapacity(dataCenterId); + if(secStore == null) { + s_logger.error("Unable to extract template, secondary storage to satisfy storage needs cannot be found!"); + return null; + } UploadVO uploadTemplateObj = new UploadVO(secStore.getId(), template.getId(), new Date(), Upload.Status.NOT_UPLOADED, type, url, Mode.FTP_UPLOAD); _uploadDao.persist(uploadTemplateObj); diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index e2db31a16c9..85c4a77774e 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -607,7 +607,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { throw new InvalidParameterValueException("The DomR template cannot be deleted."); } - if (zoneIdList != null && (storeMgr.getImageStore(zoneIdList.get(0)) == null)) { + if (zoneIdList != null && (storeMgr.getImageStoreWithFreeCapacity(zoneIdList.get(0)) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone."); } @@ -620,7 +620,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { List zoneIdList = profile.getZoneIdList(); if (zoneIdList != null && - (storeMgr.getImageStore(zoneIdList.get(0)) == null)) { + (storeMgr.getImageStoreWithFreeCapacity(zoneIdList.get(0)) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone."); } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 8d732cb2844..b81faeb0bd2 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -428,7 +428,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (storeUuid != null) { imageStore = _dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image); } else { - imageStore = _dataStoreMgr.getImageStore(zoneId); + imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (imageStore == null) { throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); } @@ -1356,7 +1356,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("Unable to delete iso, as it's used by other vms"); } - if (zoneId != null && (_dataStoreMgr.getImageStore(zoneId) == null)) { + if (zoneId != null && (_dataStoreMgr.getImageStoreWithFreeCapacity(zoneId) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage store in the specified zone."); } @@ -1631,7 +1631,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, volume = _volumeDao.findById(volumeId); zoneId = volume.getDataCenterId(); } - DataStore store = _dataStoreMgr.getImageStore(zoneId); + DataStore store = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (store == null) { throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); } @@ -1957,7 +1957,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override public String getSecondaryStorageURL(long zoneId) { - DataStore secStore = _dataStoreMgr.getImageStore(zoneId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (secStore == null) { return null; } diff --git a/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java b/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java index 01713de389d..d4f96c99e98 100644 --- a/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java +++ b/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java @@ -156,7 +156,7 @@ public class ConfigDriveNetworkElementTest { _configDrivesNetworkElement._networkModel = _networkModel; - when(_dataStoreMgr.getImageStore(DATACENTERID)).thenReturn(dataStore); + when(_dataStoreMgr.getImageStoreWithFreeCapacity(DATACENTERID)).thenReturn(dataStore); when(_ep.select(dataStore)).thenReturn(endpoint); when(_vmDao.findById(VMID)).thenReturn(virtualMachine); diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 1d3eba835ce..8b2ed40c15d 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -603,7 +603,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar } protected Map createSecStorageVmInstance(long dataCenterId, SecondaryStorageVm.Role role) { - DataStore secStore = _dataStoreMgr.getImageStore(dataCenterId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dataCenterId); if (secStore == null) { String msg = "No secondary storage available in zone " + dataCenterId + ", cannot create secondary storage vm"; s_logger.warn(msg); @@ -1117,8 +1117,11 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar Map details = _vmDetailsDao.listDetailsKeyPairs(vm.getId()); vm.setDetails(details); - DataStore secStore = _dataStoreMgr.getImageStore(dest.getDataCenter().getId()); - assert (secStore != null); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dest.getDataCenter().getId()); + if (secStore == null) { + s_logger.error(String.format("Unable to finalize virtual machine profile as no secondary storage available to satisfy storage needs for zone: %s", dest.getDataCenter().getUuid())); + return false; + } StringBuilder buf = profile.getBootArgsBuilder(); buf.append(" template=domP type=secstorage");