diff --git a/engine/storage/src/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java b/engine/storage/src/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java index 0dd55d1d325..5b1f8cd699a 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java +++ b/engine/storage/src/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java @@ -50,7 +50,7 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat @Override protected List select(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { - + s_logger.debug("ClusterScopeStoragePoolAllocator looking for storage pool"); List suitablePools = new ArrayList(); @@ -65,6 +65,14 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat } List pools = _storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags()); + + // add remaining pools in cluster, that did not match tags, to avoid set + List allPools = _storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null); + allPools.removeAll(pools); + for (StoragePoolVO pool : allPools) { + avoid.addPool(pool.getId()); + } + if (pools.size() == 0) { if (s_logger.isDebugEnabled()) { String storageType = dskCh.useLocalStorage() ? ServiceOffering.StorageType.local.toString() : ServiceOffering.StorageType.shared.toString(); @@ -72,7 +80,7 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat } return suitablePools; } - + for (StoragePoolVO pool: pools) { if(suitablePools.size() == returnUpTo){ break; @@ -80,13 +88,15 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat StoragePool pol = (StoragePool)this.dataStoreMgr.getPrimaryDataStore(pool.getId()); if (filter(avoid, pol, dskCh, plan)) { suitablePools.add(pol); + } else { + avoid.addPool(pool.getId()); } } - + if (s_logger.isDebugEnabled()) { s_logger.debug("FirstFitStoragePoolAllocator returning "+suitablePools.size() +" suitable storage pools"); } - + return suitablePools; } diff --git a/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java b/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java index 7447d988a58..632ba439cb0 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java +++ b/engine/storage/src/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java @@ -74,7 +74,7 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { if (!dskCh.useLocalStorage()) { return suitablePools; } - + // data disk and host identified from deploying vm (attach volume case) if (dskCh.getType() == Volume.Type.DATADISK && plan.getHostId() != null) { List hostPools = _poolHostDao.listByHostId(plan.getHostId()); @@ -85,7 +85,9 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { if (filter(avoid, pol, dskCh, plan)) { s_logger.debug("Found suitable local storage pool " + pool.getId() + ", adding to list"); suitablePools.add(pol); - } + } else { + avoid.addPool(pool.getId()); + } } if (suitablePools.size() == returnUpTo) { @@ -101,8 +103,19 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { StoragePool pol = (StoragePool)this.dataStoreMgr.getPrimaryDataStore(pool.getId()); if (filter(avoid, pol, dskCh, plan)) { suitablePools.add(pol); - } + } else { + avoid.addPool(pool.getId()); + } } + + // add remaining pools in cluster, that did not match tags, to avoid + // set + List allPools = _storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), + plan.getPodId(), plan.getClusterId(), null); + allPools.removeAll(availablePools); + for (StoragePoolVO pool : allPools) { + avoid.addPool(pool.getId()); + } } if (s_logger.isDebugEnabled()) { @@ -111,7 +124,7 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { return suitablePools; } - + @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); diff --git a/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java b/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java index 1d3cd819d70..e9769802a37 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java +++ b/engine/storage/src/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java @@ -39,18 +39,18 @@ import com.cloud.vm.VirtualMachineProfile; @Component public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { private static final Logger s_logger = Logger.getLogger(ZoneWideStoragePoolAllocator.class); - @Inject PrimaryDataStoreDao _storagePoolDao; - @Inject DataStoreManager dataStoreMgr; - + @Inject PrimaryDataStoreDao _storagePoolDao; + @Inject DataStoreManager dataStoreMgr; + @Override - protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, + protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, DeploymentPlan plan) { Volume volume = _volumeDao.findById(dskCh.getVolumeId()); List requestVolumes = new ArrayList(); requestVolumes.add(volume); return storageMgr.storagePoolHasEnoughSpace(requestVolumes, pool); } - + @Override protected List select(DiskProfile dskCh, VirtualMachineProfile vmProfile, @@ -64,9 +64,16 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { return suitablePools; } } - + List storagePools = _storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags()); - + + // add remaining pools in zone, that did not match tags, to avoid set + List allPools = _storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null); + allPools.removeAll(storagePools); + for (StoragePoolVO pool : allPools) { + avoid.addPool(pool.getId()); + } + for (StoragePoolVO storage : storagePools) { if (suitablePools.size() == returnUpTo) { break; @@ -74,7 +81,9 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { StoragePool pol = (StoragePool)this.dataStoreMgr.getPrimaryDataStore(storage.getId()); if (filter(avoid, pol, dskCh, plan)) { suitablePools.add(pol); - } + } else { + avoid.addPool(pol.getId()); + } } return suitablePools; } diff --git a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index b54b1c1f527..b6286aab8da 100755 --- a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -78,7 +78,7 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { @Inject ConsoleProxyDao _consoleProxyDao = null; @Inject SecondaryStorageVmDao _secStorgaeVmDao = null; @Inject ConfigurationDao _configDao = null; - @Inject GuestOSDao _guestOSDao = null; + @Inject GuestOSDao _guestOSDao = null; @Inject GuestOSCategoryDao _guestOSCategoryDao = null; @Inject VMInstanceDao _vmInstanceDao = null; @Inject ResourceManager _resourceMgr; @@ -88,17 +88,17 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { boolean _checkHvm = true; protected String _allocationAlgorithm = "random"; @Inject CapacityManager _capacityMgr; - - + + @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); } - + @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - + long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); @@ -110,19 +110,19 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of routing or not return new ArrayList(); } - + if(s_logger.isDebugEnabled()){ s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId ); } - + String hostTagOnOffering = offering.getHostTag(); String hostTagOnTemplate = template.getTemplateTag(); - + boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; - + List clusterHosts = new ArrayList(); - + String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); if (haVmTag != null) { clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag); @@ -133,31 +133,31 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { List hostsMatchingOfferingTag = new ArrayList(); List hostsMatchingTemplateTag = new ArrayList(); if (hasSvcOfferingTag){ - if (s_logger.isDebugEnabled()){ + if (s_logger.isDebugEnabled()){ s_logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); } hostsMatchingOfferingTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - if (s_logger.isDebugEnabled()){ + if (s_logger.isDebugEnabled()){ s_logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsMatchingOfferingTag); - } + } } if (hasTemplateTag){ - if (s_logger.isDebugEnabled()){ + if (s_logger.isDebugEnabled()){ s_logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); } - hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - if (s_logger.isDebugEnabled()){ + hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + if (s_logger.isDebugEnabled()){ s_logger.debug("Hosts with tag '" + hostTagOnTemplate+"' are:" + hostsMatchingTemplateTag); - } + } } - + if (hasSvcOfferingTag && hasTemplateTag){ hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - if (s_logger.isDebugEnabled()){ + clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + if (s_logger.isDebugEnabled()){ s_logger.debug("Found "+ hostsMatchingOfferingTag.size() +" Hosts satisfying both tags, host ids are:" + hostsMatchingOfferingTag); } - + clusterHosts = hostsMatchingOfferingTag; } else { if (hasSvcOfferingTag){ @@ -168,7 +168,14 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { } } } - + + // add all hosts that we are not considering to the avoid list + List allhostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); + allhostsInCluster.removeAll(clusterHosts); + for (HostVO host : allhostsInCluster) { + avoid.addHost(host.getId()); + } + return allocateTo(plan, offering, template, avoid, clusterHosts, returnUpTo, considerReservedCapacity, account); } @@ -226,11 +233,11 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { }else if(_allocationAlgorithm.equals("userdispersing")){ hosts = reorderHostsByNumberOfVms(plan, hosts, account); } - + if (s_logger.isDebugEnabled()) { s_logger.debug("FirstFitAllocator has " + hosts.size() + " hosts to check for allocation: "+hosts); } - + // We will try to reorder the host lists such that we give priority to hosts that have // the minimums to support a VM's requirements hosts = prioritizeHosts(template, hosts); @@ -242,7 +249,7 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (s_logger.isDebugEnabled()) { s_logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize()); } - + List suitableHosts = new ArrayList(); for (HostVO host : hosts) { @@ -255,7 +262,7 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { } continue; } - + //find number of guest VMs occupying capacity on this host. if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)){ if (s_logger.isDebugEnabled()) { @@ -285,13 +292,14 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (s_logger.isDebugEnabled()) { s_logger.debug("Not using host " + host.getId() + "; numCpusGood: " + numCpusGood + "; cpuFreqGood: " + cpuFreqGood + ", host has capacity?" + hostHasCapacity); } + avoid.addHost(host.getId()); } } - + if (s_logger.isDebugEnabled()) { s_logger.debug("Host Allocator returning "+suitableHosts.size() +" suitable hosts"); } - + return suitableHosts; } @@ -302,26 +310,26 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); - + List hostIdsByVmCount = _vmInstanceDao.listHostIdsByVmCount(dcId, podId, clusterId, account.getAccountId()); if (s_logger.isDebugEnabled()) { s_logger.debug("List of hosts in ascending order of number of VMs: "+ hostIdsByVmCount); } - + //now filter the given list of Hosts by this ordered list - Map hostMap = new HashMap(); + Map hostMap = new HashMap(); for (HostVO host : hosts) { hostMap.put(host.getId(), host); } List matchingHostIds = new ArrayList(hostMap.keySet()); - + hostIdsByVmCount.retainAll(matchingHostIds); - + List reorderedHosts = new ArrayList(); for(Long id: hostIdsByVmCount){ reorderedHosts.add(hostMap.get(id)); } - + return reorderedHosts; } @@ -336,13 +344,13 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (template == null) { return hosts; } - + // Determine the guest OS category of the template String templateGuestOSCategory = getTemplateGuestOSCategory(template); - + List prioritizedHosts = new ArrayList(); List noHvmHosts = new ArrayList(); - + // If a template requires HVM and a host doesn't support HVM, remove it from consideration List hostsToCheck = new ArrayList(); if (template.isRequiresHvm()) { @@ -356,7 +364,7 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { } else { hostsToCheck.addAll(hosts); } - + if (s_logger.isDebugEnabled()) { if (noHvmHosts.size() > 0) { s_logger.debug("Not considering hosts: " + noHvmHosts + " to deploy template: " + template +" as they are not HVM enabled"); @@ -376,10 +384,10 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { lowPriorityHosts.add(host); } } - + hostsToCheck.removeAll(highPriorityHosts); hostsToCheck.removeAll(lowPriorityHosts); - + // Prioritize the remaining hosts by HVM capability for (HostVO host : hostsToCheck) { if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { @@ -390,21 +398,21 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { prioritizedHosts.add(host); } } - + // Merge the lists prioritizedHosts.addAll(0, highPriorityHosts); prioritizedHosts.addAll(lowPriorityHosts); - + return prioritizedHosts; } - + protected boolean hostSupportsHVM(HostVO host) { if ( !_checkHvm ) { return true; } // Determine host capabilities String caps = host.getCapabilities(); - + if (caps != null) { String[] tokens = caps.split(","); for (String token : tokens) { @@ -413,24 +421,24 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { } } } - + return false; } - + protected String getHostGuestOSCategory(HostVO host) { DetailVO hostDetail = _hostDetailsDao.findDetail(host.getId(), "guest.os.category.id"); if (hostDetail != null) { String guestOSCategoryIdString = hostDetail.getValue(); long guestOSCategoryId; - + try { guestOSCategoryId = Long.parseLong(guestOSCategoryIdString); } catch (Exception e) { return null; } - + GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); - + if (guestOSCategory != null) { return guestOSCategory.getName(); } else { @@ -440,7 +448,7 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { return null; } } - + protected String getTemplateGuestOSCategory(VMTemplateVO template) { long guestOSId = template.getGuestOSId(); GuestOSVO guestOS = _guestOSDao.findById(guestOSId); @@ -455,7 +463,7 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { Map configs = _configDao.getConfiguration(params); String opFactor = configs.get("cpu.overprovisioning.factor"); _factor = NumbersUtil.parseFloat(opFactor, 1); - + String allocationAlgorithm = configs.get("vm.allocation.algorithm"); if (allocationAlgorithm != null) { _allocationAlgorithm = allocationAlgorithm;