Merge remote-tracking branch 'origin/4.14' into 4.15

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -22,8 +22,10 @@ import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
 import com.cloud.host.Host;
+import com.cloud.offering.ServiceOffering;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.storage.VMTemplateVO;
+import com.cloud.utils.Pair;
 import com.cloud.vm.VirtualMachine;
 
 /**
@@ -141,4 +143,6 @@ public interface CapacityManager {
     long getUsedBytes(StoragePoolVO pool);
 
     long getUsedIops(StoragePoolVO pool);
+
+    Pair<Boolean, Boolean> checkIfHostHasCpuCapabilityAndCapacity(Host host, ServiceOffering offering, boolean considerReservedCapacity);
 }

plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java

@@ -22,10 +22,14 @@ import java.util.List;
 
 import javax.inject.Inject;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
 import com.cloud.agent.manager.allocator.HostAllocator;
+import com.cloud.capacity.CapacityManager;
+import com.cloud.dc.ClusterDetailsDao;
+import com.cloud.dc.dao.ClusterDao;
 import com.cloud.deploy.DeploymentPlan;
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.host.Host;
@@ -34,6 +38,7 @@ import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.resource.ResourceManager;
+import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
@@ -45,6 +50,81 @@ public class RandomAllocator extends AdapterBase implements HostAllocator {
     private HostDao _hostDao;
     @Inject
     private ResourceManager _resourceMgr;
+    @Inject
+    private ClusterDao clusterDao;
+    @Inject
+    private ClusterDetailsDao clusterDetailsDao;
+    @Inject
+    private CapacityManager capacityManager;
+
+    private List<Host> findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type,
+                                         ExcludeList avoid, List<? extends Host> hosts, int returnUpTo,
+                                         boolean considerReservedCapacity) {
+        long dcId = plan.getDataCenterId();
+        Long podId = plan.getPodId();
+        Long clusterId = plan.getClusterId();
+        ServiceOffering offering = vmProfile.getServiceOffering();
+        List<? extends Host> hostsCopy = null;
+        List<Host> suitableHosts = new ArrayList<Host>();
+
+        if (type == Host.Type.Storage) {
+            return suitableHosts;
+        }
+        String hostTag = offering.getHostTag();
+        if (hostTag != null) {
+            s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId + " having host tag:" + hostTag);
+        } else {
+            s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId);
+        }
+        if (hosts != null) {
+            // retain all computing hosts, regardless of whether they support routing...it's random after all
+            hostsCopy = new ArrayList<Host>(hosts);
+            if (hostTag != null) {
+                hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag));
+            } else {
+                hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId));
+            }
+        } else {
+            // list all computing hosts, regardless of whether they support routing...it's random after all
+            hostsCopy = new ArrayList<HostVO>();
+            if (hostTag != null) {
+                hostsCopy = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag);
+            } else {
+                hostsCopy = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId);
+            }
+        }
+        s_logger.debug("Random Allocator found " + hostsCopy.size() + "  hosts");
+        if (hostsCopy.size() == 0) {
+            return suitableHosts;
+        }
+        Collections.shuffle(hostsCopy);
+        for (Host host : hostsCopy) {
+            if (suitableHosts.size() == returnUpTo) {
+                break;
+            }
+            if (avoid.shouldAvoid(host)) {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("Host name: " + host.getName() + ", hostId: " + host.getId() + " is in avoid set, skipping this and trying other available hosts");
+                }
+                continue;
+            }
+            Pair<Boolean, Boolean> cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity);
+            if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) {
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug("Not using host " + host.getId() + "; host has cpu capability? " + cpuCapabilityAndCapacity.first() + ", host has capacity?" + cpuCapabilityAndCapacity.second());
+                }
+                continue;
+            }
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug("Found a suitable host, adding to list: " + host.getId());
+            }
+            suitableHosts.add(host);
+        }
+        if (s_logger.isDebugEnabled()) {
+            s_logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts");
+        }
+        return suitableHosts;
+    }
 
     @Override
     public List<Host> allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) {
@@ -52,113 +132,22 @@ public class RandomAllocator extends AdapterBase implements HostAllocator {
     }
 
     @Override
-    public List<Host> allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List<? extends Host> hosts, int returnUpTo,
+    public List<Host> allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type,
-        boolean considerReservedCapacity) {
+                                 ExcludeList avoid, List<? extends Host> hosts, int returnUpTo,
-        long dcId = plan.getDataCenterId();
+                                 boolean considerReservedCapacity) {
-        Long podId = plan.getPodId();
+        if (CollectionUtils.isEmpty(hosts)) {
-        Long clusterId = plan.getClusterId();
+            if (s_logger.isDebugEnabled()) {
-        ServiceOffering offering = vmProfile.getServiceOffering();
+                s_logger.debug("Random Allocator found 0 hosts as given host list is empty");
-        List<Host> suitableHosts = new ArrayList<Host>();
-        List<Host> hostsCopy = new ArrayList<Host>(hosts);
-
-        if (type == Host.Type.Storage) {
-            return suitableHosts;
-        }
-
-        String hostTag = offering.getHostTag();
-        if (hostTag != null) {
-            s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId + " having host tag:" + hostTag);
-        } else {
-            s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId);
-        }
-
-        // list all computing hosts, regardless of whether they support routing...it's random after all
-        if (hostTag != null) {
-            hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag));
-        } else {
-            hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId));
-        }
-
-        s_logger.debug("Random Allocator found " + hostsCopy.size() + "  hosts");
-        if (hostsCopy.size() == 0) {
-            return suitableHosts;
-        }
-
-        Collections.shuffle(hostsCopy);
-        for (Host host : hostsCopy) {
-            if (suitableHosts.size() == returnUpTo) {
-                break;
-            }
-
-            if (!avoid.shouldAvoid(host)) {
-                suitableHosts.add(host);
-            } else {
-                if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Host name: " + host.getName() + ", hostId: " + host.getId() + " is in avoid set, " + "skipping this and trying other available hosts");
-                }
             }
+            return new ArrayList<Host>();
         }
-
+        return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity);
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts");
-        }
-
-        return suitableHosts;
     }
 
     @Override
-    public List<Host> allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) {
+    public List<Host> allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan,
-
+                                 Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) {
-        long dcId = plan.getDataCenterId();
+        return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity);
-        Long podId = plan.getPodId();
-        Long clusterId = plan.getClusterId();
-        ServiceOffering offering = vmProfile.getServiceOffering();
-
-        List<Host> suitableHosts = new ArrayList<Host>();
-
-        if (type == Host.Type.Storage) {
-            return suitableHosts;
-        }
-
-        String hostTag = offering.getHostTag();
-        if (hostTag != null) {
-            s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId + " having host tag:" + hostTag);
-        } else {
-            s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId);
-        }
-
-        // list all computing hosts, regardless of whether they support routing...it's random after all
-        List<? extends Host> hosts = new ArrayList<HostVO>();
-        if (hostTag != null) {
-            hosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag);
-        } else {
-            hosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId);
-        }
-
-        s_logger.debug("Random Allocator found " + hosts.size() + "  hosts");
-
-        if (hosts.size() == 0) {
-            return suitableHosts;
-        }
-
-        Collections.shuffle(hosts);
-        for (Host host : hosts) {
-            if (suitableHosts.size() == returnUpTo) {
-                break;
-            }
-
-            if (!avoid.shouldAvoid(host)) {
-                suitableHosts.add(host);
-            } else {
-                if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Host name: " + host.getName() + ", hostId: " + host.getId() + " is in avoid set, skipping this and trying other available hosts");
-                }
-            }
-        }
-        if (s_logger.isDebugEnabled()) {
-            s_logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts");
-        }
-        return suitableHosts;
     }
 
     @Override

server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java

@@ -35,7 +35,6 @@ import com.cloud.capacity.CapacityVO;
 import com.cloud.capacity.dao.CapacityDao;
 import com.cloud.configuration.Config;
 import com.cloud.dc.ClusterDetailsDao;
-import com.cloud.dc.ClusterDetailsVO;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.deploy.DeploymentPlan;
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
@@ -47,7 +46,6 @@ import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
 import com.cloud.host.dao.HostDetailsDao;
 import com.cloud.offering.ServiceOffering;
-import com.cloud.org.Cluster;
 import com.cloud.resource.ResourceManager;
 import com.cloud.service.ServiceOfferingDetailsVO;
 import com.cloud.service.dao.ServiceOfferingDetailsDao;
@@ -57,12 +55,13 @@ import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.dao.GuestOSCategoryDao;
 import com.cloud.storage.dao.GuestOSDao;
 import com.cloud.user.Account;
+import com.cloud.utils.Pair;
 import com.cloud.utils.component.AdapterBase;
+import com.cloud.vm.UserVmDetailVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
-import com.cloud.vm.dao.VMInstanceDao;
-import com.cloud.vm.UserVmDetailVO;
 import com.cloud.vm.dao.UserVmDetailsDao;
+import com.cloud.vm.dao.VMInstanceDao;
 
 
 /**
@@ -336,27 +335,15 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator {
                     continue;
                 }
             }
-
+            Pair<Boolean, Boolean> cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity);
-            int cpu_requested = offering.getCpu() * offering.getSpeed();
+            if (cpuCapabilityAndCapacity.first() && cpuCapabilityAndCapacity.second()) {
-            long ram_requested = offering.getRamSize() * 1024L * 1024L;
-            Cluster cluster = _clusterDao.findById(host.getClusterId());
-            ClusterDetailsVO clusterDetailsCpuOvercommit = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
-            ClusterDetailsVO clusterDetailsRamOvercommmt = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
-            Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
-            Float memoryOvercommitRatio = Float.parseFloat(clusterDetailsRamOvercommmt.getValue());
-
-            boolean hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host.getId(), offering.getCpu(), offering.getSpeed());
-            boolean hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, false, cpuOvercommitRatio, memoryOvercommitRatio,
-                considerReservedCapacity);
-
-            if (hostHasCpuCapability && hostHasCapacity) {
                 if (s_logger.isDebugEnabled()) {
                     s_logger.debug("Found a suitable host, adding to list: " + host.getId());
                 }
                 suitableHosts.add(host);
             } else {
                 if (s_logger.isDebugEnabled()) {
-                    s_logger.debug("Not using host " + host.getId() + "; host has cpu capability? " + hostHasCpuCapability + ", host has capacity?" + hostHasCapacity);
+                    s_logger.debug("Not using host " + host.getId() + "; host has cpu capability? " + cpuCapabilityAndCapacity.first() + ", host has capacity?" + cpuCapabilityAndCapacity.second());
                 }
                 avoid.addHost(host.getId());
             }

server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java

@@ -60,6 +60,7 @@ import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao;
 import com.cloud.offering.ServiceOffering;
+import com.cloud.org.Cluster;
 import com.cloud.resource.ResourceListener;
 import com.cloud.resource.ResourceManager;
 import com.cloud.resource.ResourceState;
@@ -1091,6 +1092,23 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
 
     }
 
+    @Override
+    public Pair<Boolean, Boolean> checkIfHostHasCpuCapabilityAndCapacity(Host host, ServiceOffering offering, boolean considerReservedCapacity) {
+        int cpu_requested = offering.getCpu() * offering.getSpeed();
+        long ram_requested = offering.getRamSize() * 1024L * 1024L;
+        Cluster cluster = _clusterDao.findById(host.getClusterId());
+        ClusterDetailsVO clusterDetailsCpuOvercommit = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio");
+        ClusterDetailsVO clusterDetailsRamOvercommmt = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");
+        Float cpuOvercommitRatio = Float.parseFloat(clusterDetailsCpuOvercommit.getValue());
+        Float memoryOvercommitRatio = Float.parseFloat(clusterDetailsRamOvercommmt.getValue());
+
+        boolean hostHasCpuCapability = checkIfHostHasCpuCapability(host.getId(), offering.getCpu(), offering.getSpeed());
+        boolean hostHasCapacity = checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, false, cpuOvercommitRatio, memoryOvercommitRatio,
+                considerReservedCapacity);
+
+        return new Pair<>(hostHasCpuCapability, hostHasCapacity);
+    }
+
     @Override
     public boolean processAnswers(long agentId, long seq, Answer[] answers) {
         // TODO Auto-generated method stub