diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 5ebef031c5c..ad5b1622cd2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -33,7 +33,11 @@ public interface DataStoreManager { List getImageStoresByScope(ZoneScope scope); - DataStore getImageStore(long zoneId); + DataStore getRandomImageStore(long zoneId); + + DataStore getImageStoreWithFreeCapacity(long zoneId); + + List listImageStoresWithFreeCapacity(long zoneId); List getImageCacheStores(Scope scope); diff --git a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java index c9832bf1169..22b3f46a946 100644 --- a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java +++ b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/allocator/StorageCacheRandomAllocator.java @@ -62,7 +62,7 @@ public class StorageCacheRandomAllocator implements StorageCacheAllocator { return null; } - return imageStoreMgr.getImageStore(cacheStores); + return imageStoreMgr.getImageStoreWithFreeCapacity(cacheStores); } @Override @@ -88,6 +88,6 @@ public class StorageCacheRandomAllocator implements StorageCacheAllocator { } } } - return imageStoreMgr.getImageStore(cacheStores); + return imageStoreMgr.getImageStoreWithFreeCapacity(cacheStores); } } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 7b526458835..39851b47b62 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -328,8 +328,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { if (cacheStore == null) { // need to find a nfs or cifs image store, assuming that can't copy volume // directly to s3 - ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStore(destScope.getScopeId()); - if (!imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { + ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId()); + if (imageStore == null || !imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { s_logger.debug("can't find a nfs (or cifs) image store to satisfy the need for a staging store"); return null; } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java index e42715a1e6d..e6b5c85b924 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java @@ -24,18 +24,17 @@ import java.util.Set; import javax.inject.Inject; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.datastore.DataStoreManagerImpl; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; @@ -47,6 +46,8 @@ import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; @@ -56,7 +57,6 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; -import org.apache.commons.collections.MapUtils; /** * Extends {@link StorageSystemDataMotionStrategy}, allowing KVM hosts to migrate VMs with the ROOT volume on a non managed local storage pool. @@ -197,19 +197,21 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot Host destHost) { VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId()); if (sourceVolumeTemplateStoragePoolVO == null && destStoragePool.getPoolType() == StoragePoolType.Filesystem) { - DataStore sourceTemplateDataStore = dataStoreManagerImpl.getImageStore(srcVolumeInfo.getDataCenterId()); - TemplateInfo sourceTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), sourceTemplateDataStore); - TemplateObjectTO sourceTemplate = new TemplateObjectTO(sourceTemplateInfo); + DataStore sourceTemplateDataStore = dataStoreManagerImpl.getRandomImageStore(srcVolumeInfo.getDataCenterId()); + if (sourceTemplateDataStore != null) { + TemplateInfo sourceTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), sourceTemplateDataStore); + TemplateObjectTO sourceTemplate = new TemplateObjectTO(sourceTemplateInfo); - LOGGER.debug(String.format("Could not find template [id=%s, name=%s] on the storage pool [id=%s]; copying the template to the target storage pool.", - srcVolumeInfo.getTemplateId(), sourceTemplateInfo.getName(), destDataStore.getId())); + LOGGER.debug(String.format("Could not find template [id=%s, name=%s] on the storage pool [id=%s]; copying the template to the target storage pool.", + srcVolumeInfo.getTemplateId(), sourceTemplateInfo.getName(), destDataStore.getId())); - TemplateInfo destTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), destDataStore); - final TemplateObjectTO destTemplate = new TemplateObjectTO(destTemplateInfo); - Answer copyCommandAnswer = sendCopyCommand(destHost, sourceTemplate, destTemplate, destDataStore); + TemplateInfo destTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), destDataStore); + final TemplateObjectTO destTemplate = new TemplateObjectTO(destTemplateInfo); + Answer copyCommandAnswer = sendCopyCommand(destHost, sourceTemplate, destTemplate, destDataStore); - if (copyCommandAnswer != null && copyCommandAnswer.getResult()) { - updateTemplateReferenceIfSuccessfulCopy(srcVolumeInfo, srcStoragePool, destTemplateInfo, destDataStore); + if (copyCommandAnswer != null && copyCommandAnswer.getResult()) { + updateTemplateReferenceIfSuccessfulCopy(srcVolumeInfo, srcStoragePool, destTemplateInfo, destDataStore); + } } } } diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 5b8d3aff2b8..3dfc4af409f 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -18,13 +18,12 @@ */ package org.apache.cloudstack.storage.motion; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + import java.util.HashMap; import java.util.Map; -import com.cloud.host.Host; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -58,21 +57,22 @@ import com.cloud.agent.api.MigrateCommand; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.CloudException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; -import com.cloud.storage.StoragePool; -import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.StoragePool; +import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.when; - @RunWith(MockitoJUnitRunner.class) public class KvmNonManagedStorageSystemDataMotionTest { @@ -353,7 +353,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { Mockito.when(sourceTemplateInfo.getHypervisorType()).thenReturn(HypervisorType.KVM); Mockito.when(vmTemplatePoolDao.findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong())).thenReturn(vmTemplateStoragePoolVO); - Mockito.when(dataStoreManagerImpl.getImageStore(Mockito.anyLong())).thenReturn(sourceTemplateDataStore); + Mockito.when(dataStoreManagerImpl.getRandomImageStore(Mockito.anyLong())).thenReturn(sourceTemplateDataStore); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore))).thenReturn(sourceTemplateInfo); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore))).thenReturn(sourceTemplateInfo); kvmNonManagedStorageDataMotionStrategy.copyTemplateToTargetFilesystemStorageIfNeeded(srcVolumeInfo, srcStoragePool, destDataStore, destStoragePool, destHost); @@ -362,7 +362,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { InOrder verifyInOrder = Mockito.inOrder(vmTemplatePoolDao, dataStoreManagerImpl, templateDataFactory, kvmNonManagedStorageDataMotionStrategy); verifyInOrder.verify(vmTemplatePoolDao, Mockito.times(1)).findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong()); - verifyInOrder.verify(dataStoreManagerImpl, Mockito.times(times)).getImageStore(Mockito.anyLong()); + verifyInOrder.verify(dataStoreManagerImpl, Mockito.times(times)).getRandomImageStore(Mockito.anyLong()); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore)); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore)); verifyInOrder.verify(kvmNonManagedStorageDataMotionStrategy, Mockito.times(times)).sendCopyCommand(Mockito.eq(destHost), Mockito.any(TemplateObjectTO.class), diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index cb9a97e5965..80e5b38f1f7 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -20,17 +20,14 @@ package org.apache.cloudstack.storage.image.manager; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import javax.annotation.PostConstruct; import javax.inject.Inject; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider; @@ -42,6 +39,8 @@ import org.apache.cloudstack.storage.image.ImageStoreDriver; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; import org.apache.cloudstack.storage.image.store.ImageStoreImpl; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; import com.cloud.server.StatsCollector; import com.cloud.storage.ScopeType; @@ -144,19 +143,56 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager } @Override - public DataStore getImageStore(List imageStores) { + public DataStore getRandomImageStore(List imageStores) { if (imageStores.size() > 1) { - Collections.shuffle(imageStores); // Randomize image store list. - Iterator i = imageStores.iterator(); - DataStore imageStore = null; - while(i.hasNext()) { - imageStore = i.next(); + Collections.shuffle(imageStores); + } + return imageStores.get(0); + } + + @Override + public DataStore getImageStoreWithFreeCapacity(List imageStores) { + if (imageStores.size() > 1) { + imageStores.sort(new Comparator() { // Sort data stores based on free capacity + @Override + public int compare(DataStore store1, DataStore store2) { + return Long.compare(_statsCollector.imageStoreCurrentFreeCapacity(store1), + _statsCollector.imageStoreCurrentFreeCapacity(store2)); + } + }); + for (DataStore imageStore : imageStores) { // Return image store if used percentage is less then threshold value i.e. 90%. if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { return imageStore; } } + } else if (imageStores.size() == 1) { + if (_statsCollector.imageStoreHasEnoughCapacity(imageStores.get(0))) { + return imageStores.get(0); + } } - return imageStores.get(0); + + // No store with space found + s_logger.error(String.format("Can't find an image storage in zone with less than %d usage", + Math.round(_statsCollector.getImageStoreCapacityThreshold()*100))); + return null; + } + + @Override + public List listImageStoresWithFreeCapacity(List imageStores) { + List stores = new ArrayList<>(); + for (DataStore imageStore : imageStores) { + // Return image store if used percentage is less then threshold value i.e. 90%. + if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + stores.add(imageStore); + } + } + + // No store with space found + if (stores.isEmpty()) { + s_logger.error(String.format("Can't find image storage in zone with less than %d usage", + Math.round(_statsCollector.getImageStoreCapacityThreshold() * 100))); + } + return stores; } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 9c513709677..51a2741dddb 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -240,7 +240,7 @@ public class SnapshotServiceImpl implements SnapshotService { fullSnapshot = snapshotFullBackup; } if (fullSnapshot) { - return dataStoreMgr.getImageStore(snapshot.getDataCenterId()); + return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); } else { SnapshotInfo parentSnapshot = snapshot.getParent(); // Note that DataStore information in parentSnapshot is for primary @@ -251,7 +251,7 @@ public class SnapshotServiceImpl implements SnapshotService { parentSnapshotOnBackupStore = _snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), DataStoreRole.Image); } if (parentSnapshotOnBackupStore == null) { - return dataStoreMgr.getImageStore(snapshot.getDataCenterId()); + return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); } return dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole()); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index f64037619a8..51421e4cd3d 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -73,12 +73,30 @@ public class DataStoreManagerImpl implements DataStoreManager { } @Override - public DataStore getImageStore(long zoneId) { + public DataStore getRandomImageStore(long zoneId) { List stores = getImageStoresByScope(new ZoneScope(zoneId)); if (stores == null || stores.size() == 0) { return null; } - return imageDataStoreMgr.getImageStore(stores); + return imageDataStoreMgr.getRandomImageStore(stores); + } + + @Override + public DataStore getImageStoreWithFreeCapacity(long zoneId) { + List stores = getImageStoresByScope(new ZoneScope(zoneId)); + if (stores == null || stores.size() == 0) { + return null; + } + return imageDataStoreMgr.getImageStoreWithFreeCapacity(stores); + } + + @Override + public List listImageStoresWithFreeCapacity(long zoneId) { + List stores = getImageStoresByScope(new ZoneScope(zoneId)); + if (stores == null || stores.size() == 0) { + return null; + } + return imageDataStoreMgr.listImageStoresWithFreeCapacity(stores); } @Override @@ -110,7 +128,7 @@ public class DataStoreManagerImpl implements DataStoreManager { if (stores == null || stores.size() == 0) { return null; } - return imageDataStoreMgr.getImageStore(stores); + return imageDataStoreMgr.getImageStoreWithFreeCapacity(stores); } @Override diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 70b7a7c3c68..01f2100f77f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -42,5 +42,38 @@ public interface ImageStoreProviderManager { boolean registerDriver(String uuid, ImageStoreDriver driver); - DataStore getImageStore(List imageStores); + /** + * Return a random DataStore from the a list of DataStores. + * + * @param imageStores the list of image stores from which a random store + * to be returned + * @return random DataStore + */ + DataStore getRandomImageStore(List imageStores); + + /** + * Return a DataStore which has free capacity. Stores will be sorted + * based on their free space and capacity check will be done based on + * the predefined threshold value. If a store is full beyond the + * threshold it won't be considered for response. First store in the + * sorted list free capacity will be returned. When there is no store + * with free capacity in the list a null value will be returned. + * + * @param imageStores the list of image stores from which stores with free + * capacity stores to be returned + * @return the DataStore which has free capacity + */ + DataStore getImageStoreWithFreeCapacity(List imageStores); + + /** + * Return a list of DataStore which have free capacity. Free capacity check + * will be done based on the predefined threshold value. If a store is full + * beyond the threshold it won't be considered for response. An empty list + * will be returned when no store in the parameter list has free capacity. + * + * @param imageStores the list of image stores from which stores with free + * capacity stores to be returned + * @return the list of DataStore which have free capacity + */ + List listImageStoresWithFreeCapacity(List imageStores); } diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java index 9d63726d347..09e45447944 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/hypervisor/hyperv/manager/HypervManagerImpl.java @@ -137,7 +137,7 @@ public class HypervManagerImpl implements HypervManager { private String getSecondaryStorageStoreUrl(long zoneId) { String secUrl = null; - DataStore secStore = _dataStoreMgr.getImageStore(zoneId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (secStore != null) { secUrl = secStore.getUri(); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java index efdbc724fbd..8cc328a7a38 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -16,16 +16,17 @@ // under the License. package com.cloud.hypervisor.vmware.manager; +import java.io.File; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.framework.config.ConfigKey; + import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.utils.Pair; import com.vmware.vim25.ManagedObjectReference; -import org.apache.cloudstack.framework.config.ConfigKey; - -import java.io.File; -import java.util.List; -import java.util.Map; public interface VmwareManager { public final String CONTEXT_STOCK_NAME = "vmwareMgr"; @@ -65,6 +66,8 @@ public interface VmwareManager { Pair getSecondaryStorageStoreUrlAndId(long dcId); + List> getSecondaryStorageStoresUrlAndIdList(long dcId); + File getSystemVMKeyFile(); VmwareStorageManager getStorageManager(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 23758d512ba..1d3c1ad9d69 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -48,10 +48,12 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; +import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.log4j.Logger; +import com.amazonaws.util.CollectionUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.AgentControlAnswer; @@ -62,7 +64,6 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.cluster.ClusterManager; -import org.apache.cloudstack.management.ManagementServerHost; import com.cloud.cluster.dao.ManagementServerHostPeerDao; import com.cloud.configuration.Config; import com.cloud.dc.ClusterDetailsDao; @@ -492,7 +493,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String secUrl = null; Long secId = null; - DataStore secStore = _dataStoreMgr.getImageStore(dcId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dcId); if (secStore != null) { secUrl = secStore.getUri(); secId = secStore.getId(); @@ -513,6 +514,32 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return new Pair(secUrl, secId); } + @Override + public List> getSecondaryStorageStoresUrlAndIdList(long dcId) { + List> urlIdList = new ArrayList<>(); + List secStores = _dataStoreMgr.listImageStoresWithFreeCapacity(dcId); + if (!CollectionUtils.isNullOrEmpty(secStores)) { + for (DataStore secStore : secStores) { + if (secStore != null) { + urlIdList.add(new Pair<>(secStore.getUri(), secStore.getId())); + } + } + } + + if (urlIdList.isEmpty()) { + // we are using non-NFS image store, then use cache storage instead + s_logger.info("Secondary storage is not NFS, we need to use staging storage"); + DataStore cacheStore = _dataStoreMgr.getImageCacheStore(dcId); + if (cacheStore != null) { + urlIdList.add(new Pair<>(cacheStore.getUri(), cacheStore.getId())); + } else { + s_logger.warn("No staging storage is found when non-NFS secondary storage is used"); + } + } + + return urlIdList; + } + @Override public String getServiceConsolePortGroupName() { return _serviceConsoleName; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 141f2f63555..834900f67d3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -3868,19 +3868,24 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa prepareNetworkFromNicInfo(new HostMO(getServiceContext(), _morHyperHost), nic, false, cmd.getVirtualMachine().getType()); } - Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); - String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); - if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; - throw new Exception(msg); - } - mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + List> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId)); + for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { + String secStoreUrl = secStoreUrlAndId.first(); + Long secStoreId = secStoreUrlAndId.second(); + if (secStoreUrl == null) { + String msg = String.format("Secondary storage for dc %s is not ready yet?", _dcId); + throw new Exception(msg); + } - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); - if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; - throw new Exception(msg); + if (vm.getType() != VirtualMachine.Type.User) { + mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + } + + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); + if (morSecDs == null) { + String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + throw new Exception(msg); + } } return new PrepareForMigrationAnswer(cmd); } catch (Throwable e) { @@ -4247,19 +4252,25 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa prepareNetworkFromNicInfo(new HostMO(getServiceContext(), morTgtHost), nic, false, vmTo.getType()); } - // Ensure secondary storage mounted on target host - Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); - String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); - if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; - throw new Exception(msg); - } - mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnSpecificHost(secStoreUrl, tgtHyperHost); - if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; - throw new Exception(msg); + // Ensure all secondary storage mounted on target host + List> secStoreUrlAndIdList = mgr.getSecondaryStorageStoresUrlAndIdList(Long.parseLong(_dcId)); + for (Pair secStoreUrlAndId : secStoreUrlAndIdList) { + String secStoreUrl = secStoreUrlAndId.first(); + Long secStoreId = secStoreUrlAndId.second(); + if (secStoreUrl == null) { + String msg = String.format("Secondary storage for dc %s is not ready yet?", _dcId); + throw new Exception(msg); + } + + if (vmTo.getType() != VirtualMachine.Type.User) { + mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + } + + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnSpecificHost(secStoreUrl, tgtHyperHost); + if (morSecDs == null) { + String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + throw new Exception(msg); + } } if (srcHostApiVersion.compareTo("5.1") < 0) { diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 76e4fc03ce7..e2c3ca7b597 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -23,7 +23,6 @@ import java.util.Set; import javax.inject.Inject; -import com.cloud.storage.StoragePool; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -59,6 +58,7 @@ import com.cloud.offering.NetworkOffering; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; +import com.cloud.storage.StoragePool; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSCategoryDao; @@ -308,7 +308,12 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle if (nic.isDefaultNic() && _networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.ConfigDrive)) { LOG.trace(String.format("[prepareMigration] for vm: %s", vm.getInstanceName())); final DataStore dataStore = findDataStore(vm, dest); - addConfigDriveDisk(vm, dataStore); + + try { + addConfigDriveDisk(vm, dataStore); + } catch (ResourceUnavailableException e) { + LOG.error("Failed to add config disk drive due to: ", e); + } return false; } else return true; @@ -332,7 +337,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle dataStore = pickExistingRootVolumeFromDataStore(profile, dataStore); } } else { - dataStore = _dataStoreMgr.getImageStore(dest.getDataCenter().getId()); + dataStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dest.getDataCenter().getId()); } return dataStore; } @@ -444,7 +449,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } private boolean deleteConfigDriveIso(final VirtualMachine vm) throws ResourceUnavailableException { - DataStore dataStore = _dataStoreMgr.getImageStore(vm.getDataCenterId()); + DataStore dataStore = _dataStoreMgr.getImageStoreWithFreeCapacity(vm.getDataCenterId()); Long agentId = findAgentIdForImageStore(dataStore); if (VirtualMachineManager.VmConfigDriveOnPrimaryPool.value()) { @@ -473,7 +478,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle return true; } - private void addConfigDriveDisk(final VirtualMachineProfile profile, final DataStore dataStore) { + private void addConfigDriveDisk(final VirtualMachineProfile profile, final DataStore dataStore) throws ResourceUnavailableException { boolean isoAvailable = false; final String isoPath = ConfigDrive.createConfigDrivePath(profile.getInstanceName()); for (DiskTO dataTo : profile.getDisks()) { @@ -484,6 +489,10 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } if (!isoAvailable) { TemplateObjectTO dataTO = new TemplateObjectTO(); + if (dataStore == null) { + throw new ResourceUnavailableException("Config drive disk add failed, datastore not available", + ConfigDriveNetworkElement.class, 0L); + } dataTO.setDataStore(dataStore.getTO()); dataTO.setUuid(profile.getUuid()); dataTO.setPath(isoPath); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index b81507ad52b..b2ccfe274a5 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1362,6 +1362,9 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc } public boolean imageStoreHasEnoughCapacity(DataStore imageStore) { + if (!_storageStats.keySet().contains(imageStore.getId())) { // Stats not available for this store yet, can be a new store. Better to assume it has enough capacity? + return true; + } StorageStats imageStoreStats = _storageStats.get(imageStore.getId()); if (imageStoreStats != null && (imageStoreStats.getByteUsed() / (imageStoreStats.getCapacityBytes() * 1.0)) <= _imageStoreCapacityThreshold) { return true; @@ -1369,6 +1372,11 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc return false; } + public long imageStoreCurrentFreeCapacity(DataStore imageStore) { + StorageStats imageStoreStats = _storageStats.get(imageStore.getId()); + return imageStoreStats != null ? Math.max(0, imageStoreStats.getCapacityBytes() - imageStoreStats.getByteUsed()) : 0; + } + /** * Sends VMs metrics to the configured graphite host. */ @@ -1574,4 +1582,8 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc public ConfigKey[] getConfigKeys() { return new ConfigKey[] {vmDiskStatsInterval, vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin, StatsTimeout, statsOutputUri}; } + + public double getImageStoreCapacityThreshold() { + return _imageStoreCapacityThreshold; + } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 2a3d3957828..ddd84137ad9 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2674,7 +2674,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Volume to be extracted has been removed or not in right state!"); } // perform extraction - ImageStoreEntity secStore = (ImageStoreEntity)dataStoreMgr.getImageStore(zoneId); + ImageStoreEntity secStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + if (secStore == null) { + throw new InvalidParameterValueException(String.format("Secondary storage to satisfy storage needs cannot be found for zone: %d", zoneId)); + } String value = _configDao.getValue(Config.CopyVolumeWait.toString()); NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); diff --git a/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java index e8f2980a082..64ada6dc309 100644 --- a/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/upload/UploadMonitorImpl.java @@ -176,7 +176,11 @@ public class UploadMonitorImpl extends ManagerBase implements UploadMonitor { Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE; - DataStore secStore = storeMgr.getImageStore(dataCenterId); + DataStore secStore = storeMgr.getImageStoreWithFreeCapacity(dataCenterId); + if(secStore == null) { + s_logger.error("Unable to extract template, secondary storage to satisfy storage needs cannot be found!"); + return null; + } UploadVO uploadTemplateObj = new UploadVO(secStore.getId(), template.getId(), new Date(), Upload.Status.NOT_UPLOADED, type, url, Mode.FTP_UPLOAD); _uploadDao.persist(uploadTemplateObj); diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index e2db31a16c9..85c4a77774e 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -607,7 +607,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { throw new InvalidParameterValueException("The DomR template cannot be deleted."); } - if (zoneIdList != null && (storeMgr.getImageStore(zoneIdList.get(0)) == null)) { + if (zoneIdList != null && (storeMgr.getImageStoreWithFreeCapacity(zoneIdList.get(0)) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone."); } @@ -620,7 +620,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { List zoneIdList = profile.getZoneIdList(); if (zoneIdList != null && - (storeMgr.getImageStore(zoneIdList.get(0)) == null)) { + (storeMgr.getImageStoreWithFreeCapacity(zoneIdList.get(0)) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone."); } diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 8d732cb2844..b81faeb0bd2 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -428,7 +428,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (storeUuid != null) { imageStore = _dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image); } else { - imageStore = _dataStoreMgr.getImageStore(zoneId); + imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (imageStore == null) { throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); } @@ -1356,7 +1356,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("Unable to delete iso, as it's used by other vms"); } - if (zoneId != null && (_dataStoreMgr.getImageStore(zoneId) == null)) { + if (zoneId != null && (_dataStoreMgr.getImageStoreWithFreeCapacity(zoneId) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage store in the specified zone."); } @@ -1631,7 +1631,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, volume = _volumeDao.findById(volumeId); zoneId = volume.getDataCenterId(); } - DataStore store = _dataStoreMgr.getImageStore(zoneId); + DataStore store = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (store == null) { throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); } @@ -1957,7 +1957,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override public String getSecondaryStorageURL(long zoneId) { - DataStore secStore = _dataStoreMgr.getImageStore(zoneId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); if (secStore == null) { return null; } diff --git a/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java b/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java index 01713de389d..d4f96c99e98 100644 --- a/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java +++ b/server/src/test/java/com/cloud/network/element/ConfigDriveNetworkElementTest.java @@ -156,7 +156,7 @@ public class ConfigDriveNetworkElementTest { _configDrivesNetworkElement._networkModel = _networkModel; - when(_dataStoreMgr.getImageStore(DATACENTERID)).thenReturn(dataStore); + when(_dataStoreMgr.getImageStoreWithFreeCapacity(DATACENTERID)).thenReturn(dataStore); when(_ep.select(dataStore)).thenReturn(endpoint); when(_vmDao.findById(VMID)).thenReturn(virtualMachine); diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 1d3eba835ce..8b2ed40c15d 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -603,7 +603,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar } protected Map createSecStorageVmInstance(long dataCenterId, SecondaryStorageVm.Role role) { - DataStore secStore = _dataStoreMgr.getImageStore(dataCenterId); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dataCenterId); if (secStore == null) { String msg = "No secondary storage available in zone " + dataCenterId + ", cannot create secondary storage vm"; s_logger.warn(msg); @@ -1117,8 +1117,11 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar Map details = _vmDetailsDao.listDetailsKeyPairs(vm.getId()); vm.setDetails(details); - DataStore secStore = _dataStoreMgr.getImageStore(dest.getDataCenter().getId()); - assert (secStore != null); + DataStore secStore = _dataStoreMgr.getImageStoreWithFreeCapacity(dest.getDataCenter().getId()); + if (secStore == null) { + s_logger.error(String.format("Unable to finalize virtual machine profile as no secondary storage available to satisfy storage needs for zone: %s", dest.getDataCenter().getUuid())); + return false; + } StringBuilder buf = profile.getBootArgsBuilder(); buf.append(" template=domP type=secstorage");