diff --git a/api/src/com/cloud/api/ApiConstants.java b/api/src/com/cloud/api/ApiConstants.java index 5d279970899..08fe4176ec6 100755 --- a/api/src/com/cloud/api/ApiConstants.java +++ b/api/src/com/cloud/api/ApiConstants.java @@ -193,5 +193,6 @@ public class ApiConstants { public static final String AVAILABILITY = "availability"; public static final String NETWORKRATE = "networkrate"; public static final String GUEST_IP_TYPE = "guestiptype"; + public static final String HOST_TAGS = "hosttags"; } diff --git a/api/src/com/cloud/api/commands/AddHostCmd.java b/api/src/com/cloud/api/commands/AddHostCmd.java index 6eb957d990e..63a41324ec6 100644 --- a/api/src/com/cloud/api/commands/AddHostCmd.java +++ b/api/src/com/cloud/api/commands/AddHostCmd.java @@ -15,8 +15,8 @@ * along with this program. If not, see . * */ -package com.cloud.api.commands; - +package com.cloud.api.commands; + import java.util.ArrayList; import java.util.List; @@ -34,8 +34,8 @@ import com.cloud.host.Host; import com.cloud.user.Account; @Implementation(description="Adds a new host.", responseObject=HostResponse.class) -public class AddHostCmd extends BaseCmd { - public static final Logger s_logger = Logger.getLogger(AddHostCmd.class.getName()); +public class AddHostCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(AddHostCmd.class.getName()); private static final String s_name = "addhostresponse"; @@ -67,6 +67,8 @@ public class AddHostCmd extends BaseCmd { @Parameter(name=ApiConstants.HYPERVISOR, type=CommandType.STRING, required=true, description="hypervisor type of the host") private String hypervisor; + @Parameter(name=ApiConstants.HOST_TAGS, type=CommandType.LIST, collectionType=CommandType.STRING, description="list of tags to be added to the host") + private List hostTags; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -103,6 +105,10 @@ public class AddHostCmd extends BaseCmd { public String getHypervisor() { return hypervisor; } + + public List getHostTags() { + return hostTags; + } ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// @@ -141,5 +147,5 @@ public class AddHostCmd extends BaseCmd { s_logger.warn("Exception: ", ex); throw new ServerApiException(BaseCmd.INTERNAL_ERROR, ex.getMessage()); } - } -} + } +} diff --git a/api/src/com/cloud/api/commands/CreateServiceOfferingCmd.java b/api/src/com/cloud/api/commands/CreateServiceOfferingCmd.java index 254021d2e36..abcb41aa366 100644 --- a/api/src/com/cloud/api/commands/CreateServiceOfferingCmd.java +++ b/api/src/com/cloud/api/commands/CreateServiceOfferingCmd.java @@ -65,6 +65,10 @@ public class CreateServiceOfferingCmd extends BaseCmd { @Parameter(name=ApiConstants.DOMAIN_ID, type=CommandType.LONG, description="the ID of the containing domain, null for public offerings") private Long domainId; + @Parameter(name=ApiConstants.HOST_TAGS, type=CommandType.STRING, description="the host tag for this service offering.") + private String hostTag; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -105,6 +109,9 @@ public class CreateServiceOfferingCmd extends BaseCmd { return domainId; } + public String getHostTag() { + return hostTag; + } ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/com/cloud/api/response/HostResponse.java b/api/src/com/cloud/api/response/HostResponse.java index 40982383e49..3c17966f675 100755 --- a/api/src/com/cloud/api/response/HostResponse.java +++ b/api/src/com/cloud/api/response/HostResponse.java @@ -141,6 +141,9 @@ public class HostResponse extends BaseResponse { @SerializedName("jobstatus") @Param(description="shows the current pending asynchronous job status") private Integer jobStatus; + @SerializedName("hosttags") @Param(description="comma-separated list of tags for the host") + private String hostTags; + @Override public Long getObjectId() { return getId(); @@ -453,4 +456,12 @@ public class HostResponse extends BaseResponse { public void setEvents(String events) { this.events = events; } + + public String getHostTags() { + return hostTags; + } + + public void setHostTags(String hostTags) { + this.hostTags = hostTags; + } } diff --git a/api/src/com/cloud/api/response/ServiceOfferingResponse.java b/api/src/com/cloud/api/response/ServiceOfferingResponse.java index 33bf6b70a61..30e8edc0247 100644 --- a/api/src/com/cloud/api/response/ServiceOfferingResponse.java +++ b/api/src/com/cloud/api/response/ServiceOfferingResponse.java @@ -60,6 +60,9 @@ public class ServiceOfferingResponse extends BaseResponse { @SerializedName(ApiConstants.DOMAIN) @Param(description="Domain name for the offering") private String domain; + @SerializedName(ApiConstants.HOST_TAGS) @Param(description="the host tag for the service offering") + private String hostTag; + public Long getId() { return id; } @@ -156,6 +159,12 @@ public class ServiceOfferingResponse extends BaseResponse { this.domain = domain; } - + public String getHostTag() { + return hostTag; + } + + public void setHostTag(String hostTag) { + this.hostTag = hostTag; + } } diff --git a/api/src/com/cloud/capacity/Capacity.java b/api/src/com/cloud/capacity/Capacity.java index e821a71a5fe..7d5f5aacb72 100644 --- a/api/src/com/cloud/capacity/Capacity.java +++ b/api/src/com/cloud/capacity/Capacity.java @@ -35,6 +35,7 @@ public interface Capacity { public Long getHostOrPoolId(); public long getDataCenterId(); public Long getPodId(); + public Long getClusterId(); public long getUsedCapacity(); public long getTotalCapacity(); public short getCapacityType(); diff --git a/api/src/com/cloud/deploy/DeployDestination.java b/api/src/com/cloud/deploy/DeployDestination.java index 01edf31ee16..32471f0cdc4 100644 --- a/api/src/com/cloud/deploy/DeployDestination.java +++ b/api/src/com/cloud/deploy/DeployDestination.java @@ -47,6 +47,11 @@ public class DeployDestination { _host = host; } + public DeployDestination(DataCenter dc, Pod pod, Cluster cluster, Host host, Map storage) { + this(dc,pod,cluster,host); + _storage = storage; + } + public DeployDestination() { } @@ -84,6 +89,22 @@ public class DeployDestination { @Override public String toString() { - return new StringBuilder("Dest[").append(_dc.getId()).append("-").append(_pod.getId()).append("-").append(_cluster.getId()).append("-").append(_host.getId()).append("]").toString(); + StringBuilder destination = new StringBuilder("Dest[Zone(Id)-Pod(Id)-Cluster(Id)-Host(Id)-Storage(Volume(Id|Type-->Pool(Id))] : Dest["); + destination.append("Zone(").append(_dc.getId()).append(")").append("-"); + destination.append("Pod(").append(_pod.getId()).append(")").append("-"); + destination.append("Cluster(").append(_cluster.getId()).append(")").append("-"); + destination.append("Host(").append(_host.getId()).append(")").append("-"); + destination.append("Storage("); + if(_storage != null){ + String storageStr = ""; + for(Volume vol : _storage.keySet()){ + if(!storageStr.equals("")){ + storageStr = storageStr + ", "; + } + storageStr = storageStr + "Volume(" + vol.getId() + "|"+ vol.getVolumeType().name() + "-->Pool("+_storage.get(vol).getId()+")"; + } + destination.append(storageStr); + } + return destination.append(")]").toString(); } } diff --git a/api/src/com/cloud/deploy/DeploymentPlanner.java b/api/src/com/cloud/deploy/DeploymentPlanner.java index a93b484f771..e37217d6420 100644 --- a/api/src/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/com/cloud/deploy/DeploymentPlanner.java @@ -45,11 +45,21 @@ public interface DeploymentPlanner extends Adapter { boolean check(VirtualMachineProfile vm, DeploymentPlan plan, DeployDestination dest, ExcludeList exclude); public static class ExcludeList { - Set _dcIds; - Set _podIds; - Set _clusterIds; - Set _hostIds; - Set _poolIds; + private Set _dcIds; + private Set _podIds; + private Set _clusterIds; + private Set _hostIds; + private Set _poolIds; + + public ExcludeList(){ + } + + public ExcludeList(Set _dcIds, Set _podIds, Set _clusterIds, Set _hostIds, Set _poolIds){ + this._dcIds = _dcIds; + this._podIds = _podIds; + this._clusterIds = _clusterIds; + this._poolIds = _poolIds; + } public boolean add(InsufficientCapacityException e) { Class scope = e.getScope(); @@ -180,5 +190,45 @@ public interface DeploymentPlanner extends Adapter { return false; } + + public boolean shouldAvoid(StoragePool pool) { + if (_dcIds != null && _dcIds.contains(pool.getDataCenterId())) { + return true; + } + + if (_podIds != null && _podIds.contains(pool.getPodId())) { + return true; + } + + if (_clusterIds != null && _clusterIds.contains(pool.getClusterId())) { + return true; + } + + if (_poolIds != null && _poolIds.contains(pool.getId())) { + return true; + } + + return false; + } + + public Set getDataCentersToAvoid(){ + return _dcIds; + } + + public Set getPodsToAvoid(){ + return _podIds; + } + + public Set getClustersToAvoid(){ + return _clusterIds; + } + + public Set getHostsToAvoid(){ + return _hostIds; + } + + public Set getPoolsToAvoid(){ + return _poolIds; + } } } diff --git a/api/src/com/cloud/offering/ServiceOffering.java b/api/src/com/cloud/offering/ServiceOffering.java index 57c5d0e49e5..6fa754ae4ed 100755 --- a/api/src/com/cloud/offering/ServiceOffering.java +++ b/api/src/com/cloud/offering/ServiceOffering.java @@ -72,6 +72,11 @@ public interface ServiceOffering { */ boolean getUseLocalStorage(); - Long getDomainId(); + Long getDomainId(); + + /** + * @return tag that should be present on the host needed, optional parameter + */ + String getHostTag(); } diff --git a/core/src/com/cloud/capacity/CapacityVO.java b/core/src/com/cloud/capacity/CapacityVO.java index 90e79add76d..086afbe69de 100644 --- a/core/src/com/cloud/capacity/CapacityVO.java +++ b/core/src/com/cloud/capacity/CapacityVO.java @@ -42,6 +42,9 @@ public class CapacityVO implements Capacity { @Column(name="pod_id") private Long podId; + @Column(name="cluster_id") + private Long clusterId; + @Column(name="used_capacity") private long usedCapacity; @@ -56,10 +59,11 @@ public class CapacityVO implements Capacity { public CapacityVO() {} - public CapacityVO(Long hostId, long dataCenterId, Long podId, long usedCapacity, long totalCapacity, short capacityType) { + public CapacityVO(Long hostId, long dataCenterId, Long podId, Long clusterId, long usedCapacity, long totalCapacity, short capacityType) { this.hostOrPoolId = hostId; this.dataCenterId = dataCenterId; - this.podId = podId; + this.podId = podId; + this.clusterId = clusterId; this.usedCapacity = usedCapacity; this.totalCapacity = totalCapacity; this.capacityType = capacityType; @@ -92,7 +96,16 @@ public class CapacityVO implements Capacity { } public void setPodId(long podId) { this.podId = new Long(podId); - } + } + + @Override + public Long getClusterId() { + return clusterId; + } + public void setClusterId(long clusterId) { + this.clusterId = new Long(clusterId); + } + @Override public long getUsedCapacity() { return usedCapacity; diff --git a/core/src/com/cloud/host/HostTagVO.java b/core/src/com/cloud/host/HostTagVO.java new file mode 100644 index 00000000000..97d9150f382 --- /dev/null +++ b/core/src/com/cloud/host/HostTagVO.java @@ -0,0 +1,47 @@ +package com.cloud.host; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name="host_tags") +public class HostTagVO { + @Id + @GeneratedValue(strategy=GenerationType.IDENTITY) + @Column(name="id") + private long id; + + @Column(name="host_id") + private long hostId; + + @Column(name="tag") + private String tag; + + protected HostTagVO() { + } + + public HostTagVO(long hostId, String tag) { + this.hostId = hostId; + this.tag = tag; + } + + public long getHostId() { + return hostId; + } + + public String getTag() { + return tag; + } + + public void setTag(String tag) { + this.tag = tag; + } + + public long getId() { + return id; + } +} diff --git a/core/src/com/cloud/host/HostVO.java b/core/src/com/cloud/host/HostVO.java index af1bea360a1..5fd16ef974b 100644 --- a/core/src/com/cloud/host/HostVO.java +++ b/core/src/com/cloud/host/HostVO.java @@ -18,6 +18,7 @@ package com.cloud.host; import java.util.Date; +import java.util.List; import java.util.Map; import javax.persistence.Column; @@ -131,7 +132,13 @@ public class HostVO implements Host { // Call host dao to load it. @Transient Map details; - + + // This is a delayed load value. If the value is null, + // then this field has not been loaded yet. + // Call host dao to load it. + @Transient + List hostTags; + @Override public String getStorageIpAddressDeux() { return storageIpAddressDeux; @@ -274,6 +281,14 @@ public class HostVO implements Host { public void setDetails(Map details) { this.details = details; } + + public List getHostTags() { + return hostTags; + } + + public void setHostTags(List hostTags) { + this.hostTags = hostTags; + } @Column(name="data_center_id", nullable=false) private long dataCenterId; diff --git a/core/src/com/cloud/storage/VolumeVO.java b/core/src/com/cloud/storage/VolumeVO.java index 267f2ddbaa4..d9ef406778f 100755 --- a/core/src/com/cloud/storage/VolumeVO.java +++ b/core/src/com/cloud/storage/VolumeVO.java @@ -33,6 +33,7 @@ import javax.persistence.TemporalType; import com.cloud.async.AsyncInstanceCreateStatus; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.GenericDao; @Entity @@ -450,4 +451,18 @@ public class VolumeVO implements Volume { public void setChainInfo(String chainInfo) { this.chainInfo = chainInfo; } + + @Override + public int hashCode() { + return NumbersUtil.hash(id); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof VolumeVO) { + return id == ((VolumeVO)obj).id; + } else { + return false; + } + } } diff --git a/server/src/com/cloud/agent/AgentManager.java b/server/src/com/cloud/agent/AgentManager.java index c7c2fe9cd5c..ffc29722d4d 100755 --- a/server/src/com/cloud/agent/AgentManager.java +++ b/server/src/com/cloud/agent/AgentManager.java @@ -153,6 +153,8 @@ public interface AgentManager extends Manager { Long getGuestOSCategoryId(long hostId); + String getHostTags(long hostId); + /** * Find a host based on the type needed, data center to deploy in, pod * to deploy in, service offering, template, and list of host to avoid. @@ -220,7 +222,7 @@ public interface AgentManager extends Manager { public boolean executeUserRequest(long hostId, Event event) throws AgentUnavailableException; public boolean reconnect(final long hostId) throws AgentUnavailableException; - public List discoverHosts(Long dcId, Long podId, Long clusterId, String clusterName, String url, String username, String password, String hypervisor) + public List discoverHosts(Long dcId, Long podId, Long clusterId, String clusterName, String url, String username, String password, String hypervisor, List hostTags) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; Answer easySend(Long hostId, Command cmd, int timeout); diff --git a/server/src/com/cloud/agent/manager/AgentManagerImpl.java b/server/src/com/cloud/agent/manager/AgentManagerImpl.java index cc0004b2a97..f32b836842c 100755 --- a/server/src/com/cloud/agent/manager/AgentManagerImpl.java +++ b/server/src/com/cloud/agent/manager/AgentManagerImpl.java @@ -109,6 +109,7 @@ import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.StorageUnavailableException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.exception.UnsupportedVersionException; import com.cloud.ha.HighAvailabilityManager; import com.cloud.ha.HighAvailabilityManager.WorkType; @@ -157,6 +158,7 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.ActionDelegate; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.UriUtils; import com.cloud.utils.component.Adapters; import com.cloud.utils.component.ComponentLocator; @@ -178,6 +180,7 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.host.dao.HostTagsDao; /** * Implementation of the Agent Manager. This class controls the connection to @@ -254,7 +257,9 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, protected ClusterDao _clusterDao = null; @Inject protected ClusterDetailsDao _clusterDetailsDao = null; - + @Inject + protected HostTagsDao _hostTagsDao = null; + @Inject(adapter = DeploymentPlanner.class) private Adapters _planners; @@ -516,8 +521,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, } } catch (InsufficientServerCapacityException e) { - } - + } } s_logger.warn("findHost() could not find a non-null host."); @@ -560,12 +564,12 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, } protected AgentAttache handleDirectConnect(ServerResource resource, - StartupCommand[] startup, Map details, boolean old) + StartupCommand[] startup, Map details, boolean old, List hostTags) throws ConnectionException { if (startup == null) { return null; } - HostVO server = createHost(startup, resource, details, old); + HostVO server = createHost(startup, resource, details, old, hostTags); if (server == null) { return null; } @@ -705,7 +709,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, if (resources != null) { for (Map.Entry> entry : resources.entrySet()) { ServerResource resource = entry.getKey(); - AgentAttache attache = simulateStart(resource, entry.getValue(), true); + AgentAttache attache = simulateStart(resource, entry.getValue(), true, null); if (attache != null) { hosts.add(_hostDao.findById(attache.getId())); } @@ -756,8 +760,10 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, if (clusterName == null && clusterId == null) { clusterName = "Standalone-" + url; } + List hostTags = cmd.getHostTags(); + return discoverHosts(dcId, podId, clusterId, clusterName, url, - username, password, cmd.getHypervisor()); + username, password, cmd.getHypervisor(), hostTags); } @Override @@ -767,13 +773,13 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, Long dcId = cmd.getZoneId(); String url = cmd.getUrl(); return discoverHosts(dcId, null, null, null, url, null, null, - "SecondaryStorage"); + "SecondaryStorage", null); } @Override public List discoverHosts(Long dcId, Long podId, Long clusterId, String clusterName, String url, String username, String password, - String hypervisorType) throws IllegalArgumentException, + String hypervisorType, List hostTags) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException { URI uri = null; @@ -904,7 +910,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, return null; } AgentAttache attache = simulateStart(resource, - entry.getValue(), true); + entry.getValue(), true, hostTags); if (attache != null) { hosts.add(_hostDao.findById(attache.getId())); } @@ -1703,7 +1709,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, } protected AgentAttache simulateStart(ServerResource resource, - Map details, boolean old) + Map details, boolean old, List hostTags) throws IllegalArgumentException { StartupCommand[] cmds = resource.initialize(); if (cmds == null) { @@ -1718,7 +1724,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, // .toString()); } try { - attache = handleDirectConnect(resource, cmds, details, old); + attache = handleDirectConnect(resource, cmds, details, old, hostTags); } catch (IllegalArgumentException ex) { s_logger.warn("Unable to connect due to ", ex); throw ex; @@ -1840,6 +1846,16 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, } } + @Override + public String getHostTags(long hostId){ + List hostTags = _hostTagsDao.gethostTags(hostId); + if (hostTags == null) { + return null; + } else { + return StringUtils.listToCsvTags(hostTags); + } + } + @Override public String getName() { return _name; @@ -2302,13 +2318,13 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, } } - AgentAttache attache = simulateStart(resource, hostDetails, true); + AgentAttache attache = simulateStart(resource, hostDetails, true, null); return _hostDao.findById(attache.getId()); } public HostVO createHost(final StartupCommand startup, ServerResource resource, Map details, - boolean directFirst) throws IllegalArgumentException { + boolean directFirst, List hostTags) throws IllegalArgumentException { Host.Type type = null; if (startup instanceof StartupStorageCommand) { @@ -2374,7 +2390,8 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, } server.setDetails(details); - + server.setHostTags(hostTags); + updateHost(server, startup, type, _nodeId); if (resource != null) { server.setResource(resource.getClass().getName()); @@ -2425,9 +2442,9 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, public HostVO createHost(final StartupCommand[] startup, ServerResource resource, Map details, - boolean directFirst) throws IllegalArgumentException { + boolean directFirst, List hostTags) throws IllegalArgumentException { StartupCommand firstCmd = startup[0]; - HostVO result = createHost(firstCmd, resource, details, directFirst); + HostVO result = createHost(firstCmd, resource, details, directFirst, hostTags); if (result == null) { return null; } @@ -2437,7 +2454,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, public AgentAttache handleConnect(final Link link, final StartupCommand[] startup) throws IllegalArgumentException, ConnectionException { - HostVO server = createHost(startup, null, null, false); + HostVO server = createHost(startup, null, null, false, null); if (server == null) { return null; } @@ -2776,12 +2793,12 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, StartupStorageCommand ssCmd = (StartupStorageCommand) startup; if (ssCmd.getResourceType() == Storage.StorageResourceType.STORAGE_HOST) { CapacityVO capacity = new CapacityVO(server.getId(), - server.getDataCenterId(), server.getPodId(), 0L, + server.getDataCenterId(), server.getPodId(), server.getClusterId(),0L, server.getTotalSize(), CapacityVO.CAPACITY_TYPE_STORAGE); _capacityDao.persist(capacity); capacity = new CapacityVO(server.getId(), - server.getDataCenterId(), server.getPodId(), 0L, + server.getDataCenterId(), server.getPodId(), server.getClusterId(), 0L, server.getTotalSize() * _overProvisioningFactor, CapacityVO.CAPACITY_TYPE_STORAGE_ALLOCATED); _capacityDao.persist(capacity); @@ -2824,7 +2841,8 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, CapacityVO capacity = new CapacityVO( server.getId(), server.getDataCenterId(), - server.getPodId(), + server.getPodId(), + server.getClusterId(), 0L, (long) (server.getCpus().longValue() * server.getSpeed().longValue() * _cpuOverProvisioningFactor), @@ -2866,7 +2884,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, _capacityDao.update(CapacityVOMem.getId(), CapacityVOMem); } else { CapacityVO capacity = new CapacityVO(server.getId(), - server.getDataCenterId(), server.getPodId(), 0L, + server.getDataCenterId(), server.getPodId(), server.getClusterId(), 0L, server.getTotalMemory(), CapacityVO.CAPACITY_TYPE_MEMORY); _capacityDao.persist(capacity); @@ -2927,7 +2945,7 @@ public class AgentManagerImpl implements AgentManager, HandlerFactory, s_logger.debug("Simulating start for resource " + resource.getName() + " id " + id); } - simulateStart(resource, details, false); + simulateStart(resource, details, false, null); } catch (Exception e) { s_logger.warn("Unable to simulate start on resource " + id + " name " + resource.getName(), e); diff --git a/server/src/com/cloud/agent/manager/allocator/HostAllocator.java b/server/src/com/cloud/agent/manager/allocator/HostAllocator.java index 18501b1c01b..77828c0a1ad 100755 --- a/server/src/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/HostAllocator.java @@ -17,21 +17,39 @@ */ package com.cloud.agent.manager.allocator; -import java.util.Set; - -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import java.util.List; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.offering.ServiceOffering; -import com.cloud.storage.VMTemplateVO; import com.cloud.uservm.UserVm; import com.cloud.utils.component.Adapter; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { + + /** + * Checks if the VM can be upgraded to the specified ServiceOffering + * @param UserVm vm + * @param ServiceOffering offering + * @return boolean true if the VM can be upgraded + **/ boolean isVirtualMachineUpgradable(final UserVm vm, final ServiceOffering offering); - Host allocateTo(VirtualMachineProfile vm, ServiceOffering offering, Type type, DataCenterVO dc, HostPodVO pod, Long clusterId, VMTemplateVO template, Set avoid); + + /** + * Determines which physical hosts are suitable to + * allocate the guest virtual machines on + * + * @param VirtualMachineProfile vmProfile + * @param DeploymentPlan plan + * @param Type type + * @param ExcludeList avoid + * @param int returnUpTo (use -1 to return all possible hosts) + * @return List List of hosts that are suitable for VM allocation + **/ + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); + } diff --git a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 707766b98da..a542e3b9ee0 100755 --- a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -33,6 +33,8 @@ import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.Host.Type; @@ -60,6 +62,7 @@ import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.SecondaryStorageVmDao; import com.cloud.vm.dao.UserVmDao; +import com.cloud.capacity.CapacityManager; /** * An allocator that tries to find a fit on a computing host. This allocator does not care whether or not the host supports routing. @@ -80,134 +83,97 @@ public class FirstFitAllocator implements HostAllocator { @Inject GuestOSCategoryDao _guestOSCategoryDao = null; float _factor = 1; protected String _allocationAlgorithm = "random"; + @Inject CapacityManager _capacityMgr; @Override - public Host allocateTo(VirtualMachineProfile vm, ServiceOffering offering, Type type, DataCenterVO dc, - HostPodVO pod, Long clusterId, VMTemplateVO template, - Set avoid) { + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, + ExcludeList avoid, int returnUpTo) { + + long dcId = plan.getDataCenterId(); + long podId = plan.getPodId(); + long clusterId = plan.getClusterId(); + ServiceOffering offering = vmProfile.getServiceOffering(); + VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); if (type == Host.Type.Storage) { // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of routing or not - return null; + return new ArrayList(); } - s_logger.debug("Looking for hosts in dc: " + dc.getId() + " pod:" + pod.getId() + " cluster:" + clusterId); + String hostTag = offering.getHostTag(); + if(hostTag != null){ + s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId + " having host tag:" + hostTag); + }else{ + s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); + } - List clusterHosts = _hostDao.listBy(type, clusterId, pod.getId(), dc.getId()); - Iterator it = clusterHosts.iterator(); - while (it.hasNext()) { - HostVO host = it.next(); - if (avoid.contains(host)) { - it.remove(); - } else { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Adding host " + host + " as possible pod host"); - } - } - } - return allocateTo(offering, template, avoid, clusterHosts); + List clusterHosts = new ArrayList(); + if(hostTag != null){ + clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag); + }else{ + clusterHosts = _hostDao.listBy(type, clusterId, podId, dcId); + } + + return allocateTo(offering, template, avoid, clusterHosts, returnUpTo); } - protected Host allocateTo(ServiceOffering offering, VMTemplateVO template, Set avoid, List hosts) { + protected List allocateTo(ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo) { if (_allocationAlgorithm.equals("random")) { // Shuffle this so that we don't check the hosts in the same order. Collections.shuffle(hosts); } if (s_logger.isDebugEnabled()) { - StringBuffer sb = new StringBuffer(); - for(Host h : avoid) { - sb.append(h.getName()).append(" "); - } - s_logger.debug("Found " + hosts.size() + " hosts for allocation and a avoid set of [" + sb + "]"); + s_logger.debug("FirstFitAllocator has " + hosts.size() + " hosts to check for allocation: "+hosts); } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize()); - } - // We will try to reorder the host lists such that we give priority to hosts that have // the minimums to support a VM's requirements hosts = prioritizeHosts(template, hosts); if (s_logger.isDebugEnabled()) { - s_logger.debug("Found " + hosts.size() + " hosts for allocation after prioritization"); + s_logger.debug("Found " + hosts.size() + " hosts for allocation after prioritization: "+ hosts); } + if (s_logger.isDebugEnabled()) { + s_logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize()); + } + + List suitableHosts = new ArrayList(); + for (HostVO host : hosts) { - if (avoid.contains(host)) { + if(suitableHosts.size() == returnUpTo){ + break; + } + if (avoid.shouldAvoid(host)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("host " + host.getName() + " is in avoid set, skip and try other available hosts"); + s_logger.debug("Host name: " + host.getName() + ", hostId: "+ host.getId() +" is in avoid set, skipping this and trying other available hosts"); } continue; } - long usedMemory = 0; - double totalSpeed = 0d; - - List domainRouters = _routerDao.listUpByHostId(host.getId()); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found " + domainRouters.size() + " router domains on host " + host.getId()); - } - for (DomainRouterVO router : domainRouters) { - usedMemory += 0;//FIXME or more like get rid of me router.getRamSize() * 1024L * 1024L; - } - - List proxys = _consoleProxyDao.listUpByHostId(host.getId()); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found " + proxys.size() + " console proxy on host " + host.getId()); - } - for(ConsoleProxyVO proxy : proxys) { - usedMemory += 0; // FIXME or get ird of me totally proxy.getRamSize() * 1024L * 1024L; - } - - List secStorageVms = _secStorgaeVmDao.listUpByHostId(host.getId()); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found " + secStorageVms.size() + " secondary storage VM on host " + host.getId()); - } - for(SecondaryStorageVmVO secStorageVm : secStorageVms) { - usedMemory += 0; // FIXME or get rid of me secStorageVm.getRamSize() * 1024L * 1024L; - } - - List vms = _vmDao.listUpByHostId(host.getId()); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found " + vms.size() + " user VM on host " + host.getId()); - } - - for (UserVmVO vm : vms) { - ServiceOffering so = _offeringDao.findById(vm.getServiceOfferingId()); - if (s_logger.isDebugEnabled()) { - s_logger.debug("vm " + vm.getId() + ": speed=" + (so.getCpu() * so.getSpeed()) + "Mhz, RAM=" + so.getRamSize() + "MB"); - } - usedMemory += so.getRamSize() * 1024L * 1024L; - totalSpeed += so.getCpu() * (so.getSpeed() * 0.99); - } - - if (s_logger.isDebugEnabled()) { - long availableSpeed = (long)(host.getCpus() * host.getSpeed() * _factor); - double desiredSpeed = offering.getCpu() * (offering.getSpeed() * 0.99); - long coreSpeed = host.getSpeed(); - s_logger.debug("Host " + host.getId() + ": available speed=" + availableSpeed + "Mhz, core speed=" + coreSpeed + "Mhz, used speed=" + totalSpeed + "Mhz, desired speed=" + desiredSpeed + - "Mhz, desired cores: " + offering.getCpu() + ", available cores: " + host.getCpus() + ", RAM=" + host.getTotalMemory() + - ", avail RAM=" + (host.getTotalMemory() - usedMemory) + ", desired RAM=" + (offering.getRamSize() * 1024L * 1024L)); - } - boolean numCpusGood = host.getCpus().intValue() >= offering.getCpu(); - boolean coreSpeedGood = host.getSpeed().doubleValue() >= (offering.getSpeed() * 0.99); - boolean totalSpeedGood = ((host.getCpus().doubleValue() * host.getSpeed().doubleValue() * _factor) - totalSpeed) >= (offering.getCpu() * (offering.getSpeed() * 0.99)); - boolean memoryGood = (host.getTotalMemory() - usedMemory) >= (offering.getRamSize() * 1024L * 1024L); - if (numCpusGood && totalSpeedGood && coreSpeedGood && memoryGood) { + int cpu_requested = offering.getCpu() * offering.getSpeed(); + long ram_requested = offering.getRamSize() * 1024L * 1024L; + boolean hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, false); + + if (numCpusGood && hostHasCapacity) { if (s_logger.isDebugEnabled()) { - s_logger.debug("found host " + host.getId()); + s_logger.debug("Found a suitable host, adding to list: " + host.getId()); } - return host; + suitableHosts.add(host); } else { if (s_logger.isDebugEnabled()) { - s_logger.debug("not using host " + host.getId() + "; numCpusGood: " + numCpusGood + ", coreSpeedGood: " + coreSpeedGood + ", totalSpeedGood: " + totalSpeedGood + ", memoryGood: " + memoryGood); + s_logger.debug("Not using host " + host.getId() + "; numCpusGood: " + numCpusGood + ", host has capacity?" + hostHasCapacity); } } } - return null; + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Host Allocator returning "+suitableHosts.size() +" suitable hosts"); + } + + return suitableHosts; } @Override diff --git a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitRoutingAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitRoutingAllocator.java index b13785fee4f..fc7295be6da 100644 --- a/server/src/com/cloud/agent/manager/allocator/impl/FirstFitRoutingAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/FirstFitRoutingAllocator.java @@ -18,34 +18,31 @@ package com.cloud.agent.manager.allocator.impl; -import java.util.Set; - +import java.util.ArrayList; +import java.util.List; import javax.ejb.Local; - import org.apache.log4j.NDC; - import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; -import com.cloud.offering.ServiceOffering; -import com.cloud.storage.VMTemplateVO; +import com.cloud.host.Host.Type; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @Local(value={HostAllocator.class}) public class FirstFitRoutingAllocator extends FirstFitAllocator { @Override - public Host allocateTo(VirtualMachineProfile vm, ServiceOffering offering, Host.Type type, DataCenterVO dc, HostPodVO pod, - Long clusterId, VMTemplateVO template, Set avoid) { + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, + ExcludeList avoid, int returnUpTo) { try { NDC.push("FirstFitRoutingAllocator"); if (type != Host.Type.Routing) { // FirstFitRoutingAllocator is to find space on routing capable hosts only - return null; + return new ArrayList(); } //all hosts should be of type routing anyway. - return super.allocateTo(vm, offering, type, dc, pod, clusterId, template, avoid); + return super.allocateTo(vmProfile, plan, type, avoid, returnUpTo); } finally { NDC.pop(); } diff --git a/server/src/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index d5b48d02668..c61d45963c2 100755 --- a/server/src/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -17,6 +17,7 @@ */ package com.cloud.agent.manager.allocator.impl; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; @@ -25,10 +26,13 @@ import java.util.Set; import javax.ejb.Local; import org.apache.log4j.Logger; - +import com.cloud.host.HostVO; +import com.cloud.host.Host.Type; import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; @@ -45,26 +49,60 @@ public class RandomAllocator implements HostAllocator { private HostDao _hostDao; @Override - public Host allocateTo(VirtualMachineProfile vm, ServiceOffering offering, Host.Type type, DataCenterVO dc, HostPodVO pod, - Long clusterId, VMTemplateVO template, Set avoid) { + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, + ExcludeList avoid, int returnUpTo) { + + long dcId = plan.getDataCenterId(); + long podId = plan.getPodId(); + long clusterId = plan.getClusterId(); + ServiceOffering offering = vmProfile.getServiceOffering(); + + List suitableHosts = new ArrayList(); + if (type == Host.Type.Storage) { - return null; + return suitableHosts; + } + + String hostTag = offering.getHostTag(); + if(hostTag != null){ + s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId + " having host tag:" + hostTag); + }else{ + s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); } // list all computing hosts, regardless of whether they support routing...it's random after all - List hosts = _hostDao.listBy(type, clusterId, pod.getId(), dc.getId()); - if (hosts.size() == 0) { - return null; + List hosts = new ArrayList(); + if(hostTag != null){ + hosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag); + }else{ + hosts = _hostDao.listBy(type, clusterId, podId, dcId); } + s_logger.debug("Random Allocator found " + hosts.size() + " hosts"); + + if (hosts.size() == 0) { + return suitableHosts; + } + Collections.shuffle(hosts); for (Host host : hosts) { - if (!avoid.contains(host)) { - return host; + if(suitableHosts.size() == returnUpTo){ + break; + } + + if (!avoid.shouldAvoid(host)) { + suitableHosts.add(host); + }else{ + if (s_logger.isDebugEnabled()) { + s_logger.debug("Host name: " + host.getName() + ", hostId: "+ host.getId() +" is in avoid set, skipping this and trying other available hosts"); + } } } - return null; + if (s_logger.isDebugEnabled()) { + s_logger.debug("Random Host Allocator returning "+suitableHosts.size() +" suitable hosts"); + } + return suitableHosts; } @Override diff --git a/server/src/com/cloud/agent/manager/allocator/impl/RecreateHostAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/RecreateHostAllocator.java index 19779476347..56b8e9136ad 100644 --- a/server/src/com/cloud/agent/manager/allocator/impl/RecreateHostAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/RecreateHostAllocator.java @@ -17,6 +17,7 @@ */ package com.cloud.agent.manager.allocator.impl; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -31,15 +32,19 @@ import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenterVO; +import com.cloud.dc.DataCenter; import com.cloud.dc.HostPodVO; import com.cloud.dc.PodCluster; import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; -import com.cloud.offering.ServiceOffering; -import com.cloud.storage.VMTemplateVO; +import com.cloud.host.Host.Type; +import com.cloud.host.dao.HostDao; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.VolumeDao; @@ -57,22 +62,26 @@ public class RecreateHostAllocator extends FirstFitRoutingAllocator { @Inject ClusterDao _clusterDao; @Inject AgentManager _agentMgr; @Inject VolumeDao _volsDao; + @Inject DataCenterDao _dcDao; + @Inject HostDao _hostDao; @Override - public Host allocateTo(VirtualMachineProfile vm, ServiceOffering offering, Host.Type type, DataCenterVO dc, HostPodVO pod, - Long clusterId, VMTemplateVO template, Set avoid) { - Host host = super.allocateTo(vm, offering, type, dc, pod, clusterId, template, avoid); - if (host != null) { - return host; + public List allocateTo(VirtualMachineProfile vm,DeploymentPlan plan, Type type, + ExcludeList avoid, int returnUpTo) { + + List hosts = super.allocateTo(vm, plan, type, avoid, returnUpTo); + if (hosts != null && !hosts.isEmpty()) { + return hosts; } s_logger.debug("First fit was unable to find a host"); VirtualMachine.Type vmType = vm.getType(); if (vmType == VirtualMachine.Type.User) { - s_logger.debug("vm is not a system vm so let's just return null"); - return null; + s_logger.debug("vm is not a system vm so let's just return empty list"); + return new ArrayList(); } + DataCenter dc = _dcDao.findById(plan.getDataCenterId()); List pcs = _agentMgr.listByDataCenter(dc.getId()); //getting rid of direct.attached.untagged.vlan.enabled config param: Bug 7204 //basic network type for zone maps to direct untagged case @@ -91,8 +100,14 @@ public class RecreateHostAllocator extends FirstFitRoutingAllocator { } } Set> avoidPcs = new HashSet>(); - for (Host h : avoid) { - avoidPcs.add(new Pair(h.getPodId(), h.getClusterId())); + Set hostIdsToAvoid = avoid.getHostsToAvoid(); + if(hostIdsToAvoid != null){ + for (Long hostId : hostIdsToAvoid) { + Host h = _hostDao.findById(hostId); + if(h != null){ + avoidPcs.add(new Pair(h.getPodId(), h.getClusterId())); + } + } } for (Pair pcId : avoidPcs) { @@ -101,16 +116,17 @@ public class RecreateHostAllocator extends FirstFitRoutingAllocator { } for (PodCluster p : pcs) { - clusterId = p.getCluster() == null ? null : p.getCluster().getId(); - host = super.allocateTo(vm, offering, type, dc, p.getPod(), - clusterId, template, avoid); - if (host != null) { - return host; + long clusterId = p.getCluster() == null ? null : p.getCluster().getId(); + DataCenterDeployment newPlan = new DataCenterDeployment(plan.getDataCenterId(), p.getPod().getId(), clusterId, null); + hosts = super.allocateTo(vm, newPlan, type, avoid, returnUpTo); + if (hosts != null && !hosts.isEmpty()) { + return hosts; } + } s_logger.debug("Unable to find any available pods at all!"); - return null; + return new ArrayList(); } @Override diff --git a/server/src/com/cloud/agent/manager/allocator/impl/TestingAllocator.java b/server/src/com/cloud/agent/manager/allocator/impl/TestingAllocator.java index 8f86ac84c51..4b0f8e22287 100755 --- a/server/src/com/cloud/agent/manager/allocator/impl/TestingAllocator.java +++ b/server/src/com/cloud/agent/manager/allocator/impl/TestingAllocator.java @@ -17,18 +17,19 @@ */ package com.cloud.agent.manager.allocator.impl; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -import java.util.Set; import javax.ejb.Local; import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; +import com.cloud.host.Host.Type; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; -import com.cloud.storage.VMTemplateVO; import com.cloud.uservm.UserVm; import com.cloud.utils.component.ComponentLocator; import com.cloud.vm.VirtualMachine; @@ -47,14 +48,19 @@ public class TestingAllocator implements HostAllocator { String _name; @Override - public Host allocateTo(VirtualMachineProfile vm, ServiceOffering offering, Host.Type type, DataCenterVO dc, HostPodVO pod, - Long clusterId, VMTemplateVO template, Set avoid) { + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, + ExcludeList avoid, int returnUpTo) { + List availableHosts = new ArrayList(); + Host host = null; if (type == Host.Type.Routing && _routingHost != null) { - return _hostDao.findById(_routingHost); + host = _hostDao.findById(_routingHost); } else if (type == Host.Type.Storage && _storageHost != null) { - return _hostDao.findById(_storageHost); + host = _hostDao.findById(_storageHost); } - return null; + if(host != null){ + availableHosts.add(host); + } + return availableHosts; } @Override diff --git a/server/src/com/cloud/alert/AlertManagerImpl.java b/server/src/com/cloud/alert/AlertManagerImpl.java index 7c412f57881..6cee40370ed 100644 --- a/server/src/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/com/cloud/alert/AlertManagerImpl.java @@ -283,7 +283,7 @@ public class AlertManagerImpl implements AlertManager { int totalPublicIPs = _publicIPAddressDao.countIPsForDashboard(dcId, false); int allocatedPublicIPs = _publicIPAddressDao.countIPsForDashboard(dcId, true); - CapacityVO newPublicIPCapacity = new CapacityVO(null, dcId, null, allocatedPublicIPs, totalPublicIPs, CapacityVO.CAPACITY_TYPE_PUBLIC_IP); + CapacityVO newPublicIPCapacity = new CapacityVO(null, dcId, null, null, allocatedPublicIPs, totalPublicIPs, CapacityVO.CAPACITY_TYPE_PUBLIC_IP); newCapacities.add(newPublicIPCapacity); } @@ -296,7 +296,7 @@ public class AlertManagerImpl implements AlertManager { int totalPrivateIPs = _privateIPAddressDao.countIPs(podId, dcId, false); int allocatedPrivateIPs = _privateIPAddressDao.countIPs(podId, dcId, true); - CapacityVO newPrivateIPCapacity = new CapacityVO(null, dcId, podId, allocatedPrivateIPs, totalPrivateIPs, CapacityVO.CAPACITY_TYPE_PRIVATE_IP); + CapacityVO newPrivateIPCapacity = new CapacityVO(null, dcId, podId, null, allocatedPrivateIPs, totalPrivateIPs, CapacityVO.CAPACITY_TYPE_PRIVATE_IP); newCapacities.add(newPrivateIPCapacity); } diff --git a/server/src/com/cloud/api/ApiDBUtils.java b/server/src/com/cloud/api/ApiDBUtils.java index d9cac8dc3ca..6c36671c312 100755 --- a/server/src/com/cloud/api/ApiDBUtils.java +++ b/server/src/com/cloud/api/ApiDBUtils.java @@ -347,6 +347,11 @@ public class ApiDBUtils { return null; } } + + public static String getHostTags(long hostId) { + return _agentMgr.getHostTags(hostId); + } + public static LoadBalancerVO findLoadBalancerById(Long loadBalancerId) { return _loadBalancerDao.findById(loadBalancerId); diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index 992f294b065..b3a3703ead5 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -414,6 +414,7 @@ public class ApiResponseHelper implements ResponseGenerator { offeringResponse.setDomain(ApiDBUtils.findDomainById(offering.getDomainId()).getName()); offeringResponse.setDomainId(offering.getDomainId()); } + offeringResponse.setHostTag(offering.getHostTag()); offeringResponse.setObjectName("serviceoffering"); return offeringResponse; @@ -538,6 +539,7 @@ public class ApiResponseHelper implements ResponseGenerator { Long mem = ApiDBUtils.getMemoryUsagebyHost(host.getId()); hostResponse.setMemoryAllocated(mem); hostResponse.setMemoryUsed(mem); + hostResponse.setHostTags(ApiDBUtils.getHostTags(host.getId())); } else if (host.getType().toString().equals("Storage")) { hostResponse.setDiskSizeTotal(host.getTotalSize()); hostResponse.setDiskSizeAllocated(0L); diff --git a/server/src/com/cloud/capacity/CapacityManager.java b/server/src/com/cloud/capacity/CapacityManager.java index bea487653fb..6dbb9de7ca1 100644 --- a/server/src/com/cloud/capacity/CapacityManager.java +++ b/server/src/com/cloud/capacity/CapacityManager.java @@ -11,5 +11,7 @@ import com.cloud.vm.VirtualMachine; public interface CapacityManager extends Manager { public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); - boolean allocateVmCapacity(long hostId, Integer cpu, long ram, boolean fromLastHost); + boolean allocateVmCapacity(VirtualMachine vm, boolean fromLastHost); + + boolean checkIfHostHasCapacity(long hostId, Integer cpu, long ram, boolean checkFromReservedCapacity); } diff --git a/server/src/com/cloud/capacity/CapacityManagerImpl.java b/server/src/com/cloud/capacity/CapacityManagerImpl.java index b5613874057..5eda3c15c69 100644 --- a/server/src/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/com/cloud/capacity/CapacityManagerImpl.java @@ -174,10 +174,22 @@ public class CapacityManagerImpl implements CapacityManager , StateListener= cpu){ + if(reservedMem >= ram) { + hasCapacity = true; + }else{ + failureReason = "Host does not have enough reserved RAM available"; + } + }else{ + failureReason = "Host does not have enough reserved CPU available"; + } + } else { + /*alloc from free resource*/ + if ((reservedCpu + usedCpu + cpu <= totalCpu)) { + if((reservedMem + usedMem + ram <= totalMem)){ + hasCapacity = true; + }else{ + failureReason = "Host does not have enough RAM available"; + } + }else{ + failureReason = "Host does not have enough CPU available"; + } + } + + if (hasCapacity) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Host has enough CPU and RAM available"); + } + + s_logger.debug("STATS: Can alloc CPU from host: " + hostId + ", used: " + usedCpu + ", reserved: " + + reservedCpu + ", total: " + totalCpu + + "; requested cpu:" + cpu + ",alloc_from_last_host?:" + checkFromReservedCapacity); + + s_logger.debug("STATS: Can alloc MEM from host: " + hostId + ", used: " + usedMem + ", reserved: " + + reservedMem + ", total: " + totalMem + "; requested mem: " + ram + ",alloc_from_last_host?:" + checkFromReservedCapacity); + } else { + + if (checkFromReservedCapacity) { + s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + " reservedCpu: " + reservedCpu + ", requested cpu: " + cpu + + ", reservedMem: " + reservedMem + ", requested mem: " + ram); + } else { + s_logger.debug("STATS: Failed to alloc resource from host: " + hostId + " reservedCpu: " + reservedCpu + ", used cpu: " + usedCpu + ", requested cpu: " + cpu + + ", total cpu: " + totalCpu + + ", reservedMem: " + reservedMem + ", used Mem: " + usedMem + ", requested mem: " + ram + ", total Mem:" + totalMem); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Free CPU: "+freeCpu + " , Requested CPU: "+cpu); + s_logger.debug("Free RAM: "+freeMem + " , Requested RAM: "+ram); + } + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug(failureReason + ", cannot allocate to this host."); + } + } + + return hasCapacity; + + } public class HostCapacityCollector implements Runnable { @@ -383,6 +479,16 @@ public class CapacityManagerImpl implements CapacityManager , StateListener { CapacityVO findByHostIdType(Long hostId, short capacityType); void clearNonStorageCapacities2(); List findByHostorPoolId(Long hostorPoolId); - + List orderClustersInZoneOrPodByHostCapacities(long id, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone); } diff --git a/server/src/com/cloud/capacity/dao/CapacityDaoImpl.java b/server/src/com/cloud/capacity/dao/CapacityDaoImpl.java index fda470afc5c..145a69b69ba 100755 --- a/server/src/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/server/src/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -19,6 +19,10 @@ package com.cloud.capacity.dao; import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import javax.ejb.Local; @@ -29,6 +33,7 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; +import com.cloud.utils.exception.CloudRuntimeException; @Local(value = { CapacityDao.class }) public class CapacityDaoImpl extends GenericDaoBase implements CapacityDao { @@ -39,6 +44,13 @@ public class CapacityDaoImpl extends GenericDaoBase implements private static final String CLEAR_STORAGE_CAPACITIES = "DELETE FROM `cloud`.`op_host_capacity` WHERE capacity_type=2 OR capacity_type=3 OR capacity_type=6"; //clear storage and secondary_storage capacities private static final String CLEAR_NON_STORAGE_CAPACITIES = "DELETE FROM `cloud`.`op_host_capacity` WHERE capacity_type<>2 AND capacity_type<>3 AND capacity_type<>6"; //clear non-storage and non-secondary_storage capacities private static final String CLEAR_NON_STORAGE_CAPACITIES2 = "DELETE FROM `cloud`.`op_host_capacity` WHERE capacity_type<>2 AND capacity_type<>3 AND capacity_type<>6 AND capacity_type<>0 AND capacity_type<>1"; //clear non-storage and non-secondary_storage capacities + + private static final String LIST_CLUSTERSINZONE_BY_HOST_CAPACITIES_PART1 = "SELECT DISTINCT cluster_id FROM `cloud`.`op_host_capacity` WHERE "; + private static final String LIST_CLUSTERSINZONE_BY_HOST_CAPACITIES_PART2 = " AND capacity_type = ? AND (total_capacity - used_capacity + reserved_capacity) >= ? " + + "AND cluster_id IN (SELECT distinct cluster_id FROM `cloud`.`op_host_capacity` WHERE "; + private static final String LIST_CLUSTERSINZONE_BY_HOST_CAPACITIES_PART3 = " AND capacity_type = ? AND (total_capacity - used_capacity + reserved_capacity) >= ?) " + + "ORDER BY (total_capacity - used_capacity + reserved_capacity) DESC"; + private SearchBuilder _hostIdTypeSearch; private SearchBuilder _hostOrPoolIdSearch; @@ -138,5 +150,55 @@ public class CapacityDaoImpl extends GenericDaoBase implements SearchCriteria sc = _hostOrPoolIdSearch.create(); sc.setParameters("hostId", hostorPoolId); return listBy(sc); - } + } + + + @Override + public List orderClustersInZoneOrPodByHostCapacities(long id, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone){ + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + List result = new ArrayList(); + + StringBuilder sql = new StringBuilder(LIST_CLUSTERSINZONE_BY_HOST_CAPACITIES_PART1); + if(isZone){ + sql.append("data_center_id = ?"); + }else{ + sql.append("pod_id = ?"); + } + sql.append(LIST_CLUSTERSINZONE_BY_HOST_CAPACITIES_PART2); + if(isZone){ + sql.append("data_center_id = ?"); + }else{ + sql.append("pod_id = ?"); + } + sql.append(LIST_CLUSTERSINZONE_BY_HOST_CAPACITIES_PART3); + + try { + pstmt = txn.prepareAutoCloseStatement(sql.toString()); + pstmt.setLong(1, id); + if(capacityTypeForOrdering == CapacityVO.CAPACITY_TYPE_CPU){ + pstmt.setShort(2, CapacityVO.CAPACITY_TYPE_CPU); + pstmt.setLong(3, requiredCpu); + pstmt.setLong(4, id); + pstmt.setShort(5, CapacityVO.CAPACITY_TYPE_MEMORY); + pstmt.setLong(6, requiredRam); + }else{ + pstmt.setShort(2, CapacityVO.CAPACITY_TYPE_MEMORY); + pstmt.setLong(3, requiredRam); + pstmt.setLong(4, id); + pstmt.setShort(5, CapacityVO.CAPACITY_TYPE_CPU); + pstmt.setLong(6, requiredCpu); + } + + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.add(rs.getLong(1)); + } + return result; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + sql, e); + } catch (Throwable e) { + throw new CloudRuntimeException("Caught: " + sql, e); + } + } } diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 9aa03d92633..aab6e315687 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -165,7 +165,9 @@ public enum Config { ControlCidr("Advanced", ManagementServer.class, String.class, "control.cidr", "169.254.0.0/16", "Changes the cidr for the control network traffic. Defaults to using link local. Must be unique within pods", null), ControlGateway("Advanced", ManagementServer.class, String.class, "control.gateway", "169.254.0.1", "gateway for the control network traffic", null), - + UseUserConcentratedPodAllocation("Advanced", ManagementServer.class, Boolean.class, "use.user.concentrated.pod.allocation", "true", "If true, deployment planner applies the user concentration heuristic during VM resource allocation", "true,false"), + HostCapacityTypeToOrderClusters("Advanced", ManagementServer.class, String.class, "host.capacityType.to.order.clusters", "CPU", "The host capacity type is used by deployment planner to order clusters during VM resource allocation", "CPU,RAM"), + // XenServer VmAllocationAlgorithm("Advanced", ManagementServer.class, String.class, "vm.allocation.algorithm", "random", "If 'random', hosts within a pod will be randomly considered for VM/volume allocation. If 'firstfit', they will be considered on a first-fit basis.", null), XenPublicNetwork("Network", ManagementServer.class, String.class, "xen.public.network.device", null, "[ONLY IF THE PUBLIC NETWORK IS ON A DEDICATED NIC]:The network name label of the physical device dedicated to the public network on a XenServer host", null), diff --git a/server/src/com/cloud/configuration/ConfigurationManager.java b/server/src/com/cloud/configuration/ConfigurationManager.java index 46b69647770..7e65c4162cf 100644 --- a/server/src/com/cloud/configuration/ConfigurationManager.java +++ b/server/src/com/cloud/configuration/ConfigurationManager.java @@ -68,9 +68,10 @@ public interface ConfigurationManager extends ConfigurationService, Manager { * @param offerHA * @param useVirtualNetwork * @param domainId + * @param hostTag * @return ID */ - ServiceOfferingVO createServiceOffering(long userId, String name, int cpu, int ramSize, int speed, String displayText, boolean localStorageRequired, boolean offerHA, String tags, Long domainId); + ServiceOfferingVO createServiceOffering(long userId, String name, int cpu, int ramSize, int speed, String displayText, boolean localStorageRequired, boolean offerHA, String tags, Long domainId, String hostTag); /** * Creates a new disk offering diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 672c7489556..4325c5736ae 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -1303,17 +1303,17 @@ public class ConfigurationManagerImpl implements ConfigurationManager, Configura } return createServiceOffering(userId, cmd.getServiceOfferingName(), cpuNumber.intValue(), memory.intValue(), cpuSpeed.intValue(), cmd.getDisplayText(), - localStorageRequired, offerHA, cmd.getTags(),cmd.getDomainId()); + localStorageRequired, offerHA, cmd.getTags(),cmd.getDomainId(), cmd.getHostTag()); } @Override - public ServiceOfferingVO createServiceOffering(long userId, String name, int cpu, int ramSize, int speed, String displayText, boolean localStorageRequired, boolean offerHA, String tags, Long domainId) { + public ServiceOfferingVO createServiceOffering(long userId, String name, int cpu, int ramSize, int speed, String displayText, boolean localStorageRequired, boolean offerHA, String tags, Long domainId, String hostTag) { String networkRateStr = _configDao.getValue("network.throttling.rate"); String multicastRateStr = _configDao.getValue("multicast.throttling.rate"); int networkRate = ((networkRateStr == null) ? 200 : Integer.parseInt(networkRateStr)); int multicastRate = ((multicastRateStr == null) ? 10 : Integer.parseInt(multicastRateStr)); tags = cleanupTags(tags); - ServiceOfferingVO offering = new ServiceOfferingVO(name, cpu, ramSize, speed, networkRate, multicastRate, offerHA, displayText, localStorageRequired, false, tags, false,domainId); + ServiceOfferingVO offering = new ServiceOfferingVO(name, cpu, ramSize, speed, networkRate, multicastRate, offerHA, displayText, localStorageRequired, false, tags, false,domainId, hostTag); if ((offering = _serviceOfferingDao.persist(offering)) != null) { return offering; diff --git a/server/src/com/cloud/dc/dao/ClusterDao.java b/server/src/com/cloud/dc/dao/ClusterDao.java index 3716b48341b..787219ca6e4 100644 --- a/server/src/com/cloud/dc/dao/ClusterDao.java +++ b/server/src/com/cloud/dc/dao/ClusterDao.java @@ -18,6 +18,7 @@ package com.cloud.dc.dao; import java.util.List; +import java.util.Map; import com.cloud.dc.ClusterVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; @@ -30,4 +31,5 @@ public interface ClusterDao extends GenericDao { List listByZoneId(long zoneId); List getAvailableHypervisorInZone(long zoneId); List listByDcHyType(long dcId, String hyType); + Map> getPodClusterIdMap(List clusterIds); } diff --git a/server/src/com/cloud/dc/dao/ClusterDaoImpl.java b/server/src/com/cloud/dc/dao/ClusterDaoImpl.java index ab9837cf181..a26906e000b 100644 --- a/server/src/com/cloud/dc/dao/ClusterDaoImpl.java +++ b/server/src/com/cloud/dc/dao/ClusterDaoImpl.java @@ -17,16 +17,24 @@ */ package com.cloud.dc.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.ejb.Local; +import com.cloud.capacity.CapacityVO; import com.cloud.dc.ClusterVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.exception.CloudRuntimeException; @Local(value=ClusterDao.class) public class ClusterDaoImpl extends GenericDaoBase implements ClusterDao { @@ -36,6 +44,10 @@ public class ClusterDaoImpl extends GenericDaoBase implements C protected final SearchBuilder AvailHyperSearch; protected final SearchBuilder ZoneSearch; protected final SearchBuilder ZoneHyTypeSearch; + + private static final String GET_POD_CLUSTER_MAP_PREFIX = "SELECT pod_id, id FROM cloud.cluster WHERE cluster.id IN( "; + private static final String GET_POD_CLUSTER_MAP_SUFFIX = " )"; + protected ClusterDaoImpl() { super(); @@ -116,4 +128,43 @@ public class ClusterDaoImpl extends GenericDaoBase implements C return hypers; } + + @Override + public Map> getPodClusterIdMap(List clusterIds){ + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + Map> result = new HashMap>(); + + try { + StringBuilder sql = new StringBuilder(GET_POD_CLUSTER_MAP_PREFIX); + if (clusterIds.size() > 0) { + for (Long clusterId : clusterIds) { + sql.append(clusterId).append(","); + } + sql.delete(sql.length()-1, sql.length()); + sql.append(GET_POD_CLUSTER_MAP_SUFFIX); + } + + pstmt = txn.prepareAutoCloseStatement(sql.toString()); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + Long podId = rs.getLong(1); + Long clusterIdInPod = rs.getLong(2); + if(result.containsKey(podId)){ + List clusterList = result.get(podId); + clusterList.add(clusterIdInPod); + result.put(podId, clusterList); + }else{ + List clusterList = new ArrayList(); + clusterList.add(clusterIdInPod); + result.put(podId, clusterList); + } + } + return result; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + GET_POD_CLUSTER_MAP_PREFIX, e); + } catch (Throwable e) { + throw new CloudRuntimeException("Caught: " + GET_POD_CLUSTER_MAP_PREFIX, e); + } + } } diff --git a/server/src/com/cloud/deploy/FirstFitPlanner.java b/server/src/com/cloud/deploy/FirstFitPlanner.java index 9a26d6404a9..6c07011fc21 100644 --- a/server/src/com/cloud/deploy/FirstFitPlanner.java +++ b/server/src/com/cloud/deploy/FirstFitPlanner.java @@ -2,12 +2,16 @@ package com.cloud.deploy; import java.util.ArrayList; import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.ejb.Local; import org.apache.log4j.Logger; +import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; @@ -21,7 +25,6 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -29,30 +32,54 @@ import com.cloud.host.dao.DetailsDao; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; -import com.cloud.storage.GuestOSCategoryVO; -import com.cloud.storage.GuestOSVO; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.allocator.StoragePoolAllocator; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; -import com.cloud.template.VirtualMachineTemplate; +import com.cloud.storage.dao.StoragePoolDao; +import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.Adapters; import com.cloud.utils.component.Inject; -import com.cloud.utils.db.DB; -import com.cloud.utils.db.Transaction; +import com.cloud.vm.DiskProfile; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; @Local(value=DeploymentPlanner.class) public class FirstFitPlanner extends PlannerBase implements DeploymentPlanner { private static final Logger s_logger = Logger.getLogger(FirstFitPlanner.class); @Inject private HostDao _hostDao; - @Inject private CapacityDao _capacityDao; @Inject private DataCenterDao _dcDao; @Inject private HostPodDao _podDao; @Inject private ClusterDao _clusterDao; @Inject DetailsDao _hostDetailsDao = null; @Inject GuestOSDao _guestOSDao = null; @Inject GuestOSCategoryDao _guestOSCategoryDao = null; + @Inject private DiskOfferingDao _diskOfferingDao; + @Inject private StoragePoolHostDao _poolHostDao; + @Inject private UserVmDao _vmDao; + @Inject VMInstanceDao _vmInstanceDao; + @Inject protected VolumeDao _volsDao; @Inject CapacityManager _capacityMgr; @Inject ConfigurationDao _configDao; + @Inject protected StoragePoolDao _storagePoolDao; + @Inject CapacityDao _capacityDao; + + @Inject(adapter=StoragePoolAllocator.class) + protected Adapters _storagePoolAllocators; + @Inject(adapter=HostAllocator.class) + protected Adapters _hostAllocators; + + private static int RETURN_UPTO_ALL = -1; @Override public DeployDestination plan(VirtualMachineProfile vmProfile, @@ -65,104 +92,365 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentPlanner { int cpu_requested = offering.getCpu() * offering.getSpeed(); long ram_requested = offering.getRamSize() * 1024L * 1024L; - s_logger.debug("try to allocate a host from dc:" + plan.getDataCenterId() + ", pod:" + plan.getPodId() + ",cluster:" + plan.getClusterId() + + s_logger.debug("In FirstFitPlanner:: plan"); + + s_logger.debug("Trying to allocate a host and storage pools from dc:" + plan.getDataCenterId() + ", pod:" + plan.getPodId() + ",cluster:" + plan.getClusterId() + ", requested cpu: " + cpu_requested + ", requested ram: " + ram_requested); + + s_logger.debug("Is ROOT volume READY (pool already allocated)?: " + (plan.getPoolId()!=null ? "Yes": "No")); + + if (vm.getLastHostId() != null) { - + s_logger.debug("This VM has last host_id specified, trying to choose the same host: " +vm.getLastHostId()); + HostVO host = _hostDao.findById(vm.getLastHostId()); - - if (host != null && host.getStatus() == Status.Up) { - boolean canDepployToLastHost = deployToHost(host, cpu_requested, ram_requested, true, avoid); - if (canDepployToLastHost) { - Pod pod = _podDao.findById(vm.getPodId()); - Cluster cluster = _clusterDao.findById(host.getClusterId()); - return new DeployDestination(dc, pod, cluster, host); + if(host == null){ + s_logger.debug("The last host of this VM cannot be found"); + }else{ + if (host.getStatus() == Status.Up) { + if(_capacityMgr.checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, true)){ + s_logger.debug("The last host of this VM is UP and has enough capacity"); + s_logger.debug("Now checking for suitable pools under zone: "+vm.getDataCenterId() +", pod: "+ vm.getPodId()+", cluster: "+ host.getClusterId()); + //search for storage under the zone, pod, cluster of the last host. + DataCenterDeployment lastPlan = new DataCenterDeployment(vm.getDataCenterId(), vm.getPodId(), host.getClusterId(), null); + Map> suitableVolumeStoragePools = findSuitablePoolsForVolumes(vmProfile, lastPlan, avoid, RETURN_UPTO_ALL); + //choose the potential pool for this VM for this host + if(!suitableVolumeStoragePools.isEmpty()){ + List suitableHosts = new ArrayList(); + suitableHosts.add(host); + + Pair> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools); + if(potentialResources != null){ + Pod pod = _podDao.findById(vm.getPodId()); + Cluster cluster = _clusterDao.findById(host.getClusterId()); + DeployDestination dest = new DeployDestination(dc, pod, cluster, host, potentialResources.second()); + s_logger.debug("Returning Deployment Destination: "+ dest); + return dest; + } + } + }else{ + s_logger.debug("The last host of this VM does not have enough capacity"); + } + }else{ + s_logger.debug("The last host of this VM is not UP, host status is: "+host.getStatus().name()); } } + + s_logger.debug("Cannot choose the last host to deploy this VM "); } - - /*Go through all the pods/clusters under zone*/ - List pods = null; - if (plan.getPodId() != null) { - HostPodVO pod = _podDao.findById(plan.getPodId()); - if (pod != null && dc.getId() == pod.getDataCenterId()) { - pods = new ArrayList(1); - pods.add(pod); - } else { - s_logger.debug("Can't enforce the pod selector"); + + List clusterList = new ArrayList(); + if (plan.getClusterId() != null) { + Long clusterIdSpecified = plan.getClusterId(); + s_logger.debug("Searching resources only under specified Cluster: "+ clusterIdSpecified); + ClusterVO cluster = _clusterDao.findById(plan.getClusterId()); + if (cluster != null ){ + clusterList.add(clusterIdSpecified); + return checkClustersforDestination(clusterList, vmProfile, plan, avoid, dc, _allocationAlgorithm); + }else{ + s_logger.debug("The specified cluster cannot be found, returning."); + avoid.addPod(plan.getClusterId()); return null; } - } - - if (pods == null) - pods = _podDao.listByDataCenterId(plan.getDataCenterId()); - - if (_allocationAlgorithm != null && _allocationAlgorithm.equalsIgnoreCase("random")) { - Collections.shuffle(pods); + }else if (plan.getPodId() != null) { + //consider clusters under this pod only + Long podIdSpecified = plan.getPodId(); + s_logger.debug("Searching resources only under specified Pod: "+ podIdSpecified); + + HostPodVO pod = _podDao.findById(podIdSpecified); + if (pod != null) { + //list clusters under this pod by cpu and ram capacity + clusterList = listClustersByCapacity(podIdSpecified, cpu_requested, ram_requested, avoid, false); + if(!clusterList.isEmpty()){ + if(avoid.getClustersToAvoid() != null){ + if (s_logger.isDebugEnabled()) { + s_logger.debug("Removing from the clusterId list these clusters from avoid set: "+ avoid.getClustersToAvoid()); + } + clusterList.removeAll(avoid.getClustersToAvoid()); + } + DeployDestination dest = checkClustersforDestination(clusterList, vmProfile, plan, avoid, dc, _allocationAlgorithm); + if(dest == null){ + avoid.addPod(plan.getPodId()); + } + return dest; + }else{ + if (s_logger.isDebugEnabled()) { + s_logger.debug("No clusters found under this pod, having a host with enough capacity, returning."); + } + avoid.addPod(plan.getPodId()); + return null; + } + } else { + s_logger.debug("The specified Pod cannot be found, returning."); + avoid.addPod(plan.getPodId()); + return null; + } + }else{ + //consider all clusters under this zone. + s_logger.debug("Searching all possible resources under this Zone: "+ plan.getDataCenterId()); + //list clusters under this zone by cpu and ram capacity + List prioritizedClusterIds = listClustersByCapacity(plan.getDataCenterId(), cpu_requested, ram_requested, avoid, true); + if(!prioritizedClusterIds.isEmpty()){ + if(avoid.getClustersToAvoid() != null){ + if (s_logger.isDebugEnabled()) { + s_logger.debug("Removing from the clusterId list these clusters from avoid set: "+ avoid.getClustersToAvoid()); + } + prioritizedClusterIds.removeAll(avoid.getClustersToAvoid()); + } + boolean applyUserConcentrationPodHeuristic = Boolean.parseBoolean(_configDao.getValue(Config.UseUserConcentratedPodAllocation.key())); + if(applyUserConcentrationPodHeuristic && vmProfile.getOwner() != null){ + //user has VMs in certain pods. - prioritize those pods first + //UserConcentratedPod strategy + long accountId = vmProfile.getOwner().getAccountId(); + List podIds = listPodsByUserConcentration(plan.getDataCenterId(), accountId); + if(!podIds.isEmpty()){ + if(avoid.getPodsToAvoid() != null){ + if (s_logger.isDebugEnabled()) { + s_logger.debug("Removing from the pod list these pods from avoid set: "+ avoid.getPodsToAvoid()); + } + podIds.removeAll(avoid.getPodsToAvoid()); + } + clusterList = reorderClustersByPods(prioritizedClusterIds, podIds); + }else{ + clusterList = prioritizedClusterIds; + } + }else{ + clusterList = prioritizedClusterIds; + } + return checkClustersforDestination(clusterList, vmProfile, plan, avoid, dc, _allocationAlgorithm); + }else{ + if (s_logger.isDebugEnabled()) { + s_logger.debug("No clusters found having a host with enough capacity, returning."); + } + return null; + } } + + } + + + private DeployDestination checkClustersforDestination(List clusterList, VirtualMachineProfile vmProfile, + DeploymentPlan plan, ExcludeList avoid, DataCenter dc, String _allocationAlgorithm){ - for (HostPodVO hostPod : pods) { - if (avoid.shouldAvoid(hostPod)) { + for(Long clusterId : clusterList){ + Cluster clusterVO = _clusterDao.findById(clusterId); + + if (clusterVO.getHypervisorType() != vmProfile.getHypervisorType()) { + s_logger.debug("Cluster: "+clusterId + " has HyperVisorType that does not match the VM, skipping this cluster"); + avoid.addCluster(clusterVO.getId()); continue; } - - - List clusters = null; - if (plan.getClusterId() != null) { - ClusterVO cluster = _clusterDao.findById(plan.getClusterId()); - if (cluster != null && hostPod.getId() == cluster.getPodId()) { - clusters = new ArrayList(1); - clusters.add(cluster); - } else { - s_logger.debug("Can't enforce the cluster selector"); - return null; - } - } - - if (clusters == null) { - clusters = _clusterDao.listByPodId(hostPod.getId()); - } - - if (_allocationAlgorithm != null && _allocationAlgorithm.equalsIgnoreCase("random")) { - Collections.shuffle(clusters); - } - - for (ClusterVO clusterVO : clusters) { - if (avoid.shouldAvoid(clusterVO)) { - continue; - } - - if (clusterVO.getHypervisorType() != vmProfile.getHypervisorType()) { - avoid.addCluster(clusterVO.getId()); - continue; - } - - List hosts = _hostDao.listBy(Host.Type.Routing, clusterVO.getId(), hostPod.getId(), dc.getId()); + s_logger.debug("Checking resources in Cluster: "+clusterId + " under Pod: "+clusterVO.getPodId()); + //search for resources(hosts and storage) under this zone, pod, cluster. + DataCenterDeployment potentialPlan = new DataCenterDeployment(plan.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), plan.getPoolId()); + + //find suitable hosts under this cluster, need as many hosts as we get. + List suitableHosts = findSuitableHosts(vmProfile, potentialPlan, avoid, RETURN_UPTO_ALL); + //if found suitable hosts in this cluster, find suitable storage pools for each volume of the VM + if(suitableHosts != null && !suitableHosts.isEmpty()){ if (_allocationAlgorithm != null && _allocationAlgorithm.equalsIgnoreCase("random")) { - Collections.shuffle(hosts); + Collections.shuffle(suitableHosts); } - - // We will try to reorder the host lists such that we give priority to hosts that have - // the minimums to support a VM's requirements - hosts = prioritizeHosts(vmProfile.getTemplate(), hosts); - - for (HostVO hostVO : hosts) { - boolean canDeployToHost = deployToHost(hostVO, cpu_requested, ram_requested, false, avoid); - if (canDeployToHost) { - Pod pod = _podDao.findById(hostPod.getId()); - Cluster cluster = _clusterDao.findById(clusterVO.getId()); - Host host = _hostDao.findById(hostVO.getId()); - return new DeployDestination(dc, pod, cluster, host); + Map> suitableVolumeStoragePools = findSuitablePoolsForVolumes(vmProfile, potentialPlan, avoid, RETURN_UPTO_ALL); + + //choose the potential host and pool for the VM + if(!suitableVolumeStoragePools.isEmpty()){ + Pair> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools); + + if(potentialResources != null){ + Pod pod = _podDao.findById(clusterVO.getPodId()); + Host host = _hostDao.findById(potentialResources.first().getId()); + DeployDestination dest = new DeployDestination(dc, pod, clusterVO, host, potentialResources.second() ); + s_logger.debug("Returning Deployment Destination: "+ dest); + return dest; } - avoid.addHost(hostVO.getId()); + }else{ + s_logger.debug("No suitable storagePools found under this Cluster: "+clusterId); } - avoid.addCluster(clusterVO.getId()); + }else{ + s_logger.debug("No suitable hosts found under this Cluster: "+clusterId); } - avoid.addPod(hostPod.getId()); + avoid.addCluster(clusterVO.getId()); } - + s_logger.debug("Could not find suitable Deployment Destination for this VM under any clusters, returning. "); return null; } + + private List reorderClustersByPods(List clusterIds, List podIds) { + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Reordering cluster list as per pods ordered by user concentration"); + } + + Map> podClusterMap = _clusterDao.getPodClusterIdMap(clusterIds); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Pod To cluster Map is: "+podClusterMap ); + } + + List reorderedClusters = new ArrayList(); + for (Long pod : podIds){ + if(podClusterMap.containsKey(pod)){ + List clustersOfThisPod = (List)podClusterMap.get(pod); + if(clustersOfThisPod != null){ + reorderedClusters.addAll(clustersOfThisPod); + clusterIds.removeAll(clustersOfThisPod); + } + } + } + reorderedClusters.addAll(clusterIds); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Reordered cluster list: "+reorderedClusters ); + } + return reorderedClusters; + } + + protected List listPodsByUserConcentration(long zoneId, long accountId){ + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Applying UserConcentratedPod heuristic for account: "+ accountId); + } + + List prioritizedPods = _vmDao.listPodIdsHavingVmsforAccount(zoneId, accountId); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("List of pods to be considered, after applying UserConcentratedPod heuristic: "+ prioritizedPods); + } + + return prioritizedPods; + } + + protected List listClustersByCapacity(long id, int requiredCpu, long requiredRam, ExcludeList avoid, boolean isZone){ + //look at the aggregate available cpu and ram per cluster + //although an aggregate value may be false indicator that a cluster can host a vm, it will at the least eliminate those clusters which definitely cannot + + //we need clusters having enough cpu AND RAM + if (s_logger.isDebugEnabled()) { + s_logger.debug("Listing clusters that have enough aggregate CPU and RAM capacity under this "+(isZone ? "Zone: " : "Pod: " )+id); + } + String capacityTypeToOrder = _configDao.getValue(Config.HostCapacityTypeToOrderClusters.key()); + short capacityType = CapacityVO.CAPACITY_TYPE_CPU; + if("RAM".equalsIgnoreCase(capacityTypeToOrder)){ + capacityType = CapacityVO.CAPACITY_TYPE_MEMORY; + } + + List clusterIdswithEnoughCapacity = _capacityDao.orderClustersInZoneOrPodByHostCapacities(id, requiredCpu, requiredRam, capacityType, isZone); + if (s_logger.isDebugEnabled()) { + s_logger.debug("ClusterId List having enough aggregate capacity: "+clusterIdswithEnoughCapacity ); + } + return clusterIdswithEnoughCapacity; + + } + + protected Pair> findPotentialDeploymentResources(List suitableHosts, Map> suitableVolumeStoragePools){ + s_logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM"); + + boolean hostCanAccessPool = false; + Map storage = new HashMap(); + for(Host potentialHost : suitableHosts){ + for(Volume vol : suitableVolumeStoragePools.keySet()){ + s_logger.debug("Checking if host: "+potentialHost.getId() +" can access any suitable storage pool for volume: "+ vol.getVolumeType()); + List volumePoolList = suitableVolumeStoragePools.get(vol); + hostCanAccessPool = false; + for(StoragePool potentialSPool : volumePoolList){ + if(hostCanAccessSPool(potentialHost, potentialSPool)){ + storage.put(vol, potentialSPool); + hostCanAccessPool = true; + break; + } + } + if(!hostCanAccessPool){ + break; + } + } + if(hostCanAccessPool){ + s_logger.debug("Found a potential host and associated storage pools for this VM"); + return new Pair>(potentialHost, storage); + } + } + s_logger.debug("Could not find a potential host that has associated storage pools from the suitable host/pool lists for this VM"); + return null; + } + + protected boolean hostCanAccessSPool(Host host, StoragePool pool){ + boolean hostCanAccessSPool = false; + + StoragePoolHostVO hostPoolLinkage = _poolHostDao.findByPoolHost(pool.getId(), host.getId()); + if(hostPoolLinkage != null){ + hostCanAccessSPool = true; + } + + s_logger.debug("Host: "+ host.getId() + (hostCanAccessSPool ?" can" : " cannot") + " access pool: "+ pool.getId()); + return hostCanAccessSPool; + } + + protected List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo){ + List suitableHosts = new ArrayList(); + Enumeration enHost = _hostAllocators.enumeration(); + s_logger.debug("Calling HostAllocators to find suitable hosts"); + + while (enHost.hasMoreElements()) { + final HostAllocator allocator = enHost.nextElement(); + suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, avoid, returnUpTo); + if (suitableHosts != null && !suitableHosts.isEmpty()) { + break; + } + } + + if(suitableHosts.isEmpty()){ + s_logger.debug("No suitable hosts found"); + } + return suitableHosts; + } + + protected Map> findSuitablePoolsForVolumes(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo){ + List volumesTobeCreated = _volsDao.findUsableVolumesForInstance(vmProfile.getId()); + Map> suitableVolumeStoragePools = new HashMap>(); + + s_logger.debug("Calling StoragePoolAllocators to find suitable pools"); + + //for each volume find list of suitable storage pools by calling the allocators + for (VolumeVO toBeCreated : volumesTobeCreated) { + s_logger.debug("Checking suitable pools for volume (Id, Type): ("+toBeCreated.getId() +"," +toBeCreated.getVolumeType().name() + ")"); + + //skip the volume if its already in READY state and has pool allocated + if(plan.getPoolId() != null){ + if (toBeCreated.getPoolId() != null && toBeCreated.getPoolId().longValue() == plan.getPoolId().longValue()) { + s_logger.debug("Volume is in READY state and has pool already allocated."); + List suitablePools = new ArrayList(); + StoragePoolVO pool = _storagePoolDao.findById(toBeCreated.getPoolId()); + suitablePools.add(pool); + suitableVolumeStoragePools.put(toBeCreated, suitablePools); + continue; + } + } + DiskOfferingVO diskOffering = _diskOfferingDao.findById(toBeCreated.getDiskOfferingId()); + DiskProfile diskProfile = new DiskProfile(toBeCreated, diskOffering, vmProfile.getHypervisorType()); + + boolean foundPotentialPools = false; + + Enumeration enPool = _storagePoolAllocators.enumeration(); + while (enPool.hasMoreElements()) { + final StoragePoolAllocator allocator = enPool.nextElement(); + final List suitablePools = allocator.allocateToPool(diskProfile, vmProfile, plan, avoid, returnUpTo); + if (suitablePools != null && !suitablePools.isEmpty()) { + suitableVolumeStoragePools.put(toBeCreated, suitablePools); + foundPotentialPools = true; + break; + } + } + + if(!foundPotentialPools){ + //No suitable storage pools found under this cluster for this volume. + break; + } + } + if(suitableVolumeStoragePools.isEmpty()){ + s_logger.debug("No suitable pools found"); + } + return suitableVolumeStoragePools; + } @Override @@ -171,119 +459,4 @@ public class FirstFitPlanner extends PlannerBase implements DeploymentPlanner { // TODO Auto-generated method stub return false; } - - @DB - protected boolean deployToHost(HostVO host, Integer cpu, long ram, boolean fromLastHost, ExcludeList avoid) { - if (avoid.shouldAvoid(host)) { - return false; - } - - return _capacityMgr.allocateVmCapacity(host.getId(), cpu, ram, fromLastHost); - } - - protected List prioritizeHosts(VirtualMachineTemplate template, List hosts) { - if (template == null) { - return hosts; - } - - // Determine the guest OS category of the template - String templateGuestOSCategory = getTemplateGuestOSCategory(template); - - List prioritizedHosts = new ArrayList(); - - // If a template requires HVM and a host doesn't support HVM, remove it from consideration - List hostsToCheck = new ArrayList(); - if (template.isRequiresHvm()) { - for (HostVO host : hosts) { - if (hostSupportsHVM(host)) { - hostsToCheck.add(host); - } - } - } else { - hostsToCheck.addAll(hosts); - } - - // If a host is tagged with the same guest OS category as the template, move it to a high priority list - // If a host is tagged with a different guest OS category than the template, move it to a low priority list - List highPriorityHosts = new ArrayList(); - List lowPriorityHosts = new ArrayList(); - for (HostVO host : hostsToCheck) { - String hostGuestOSCategory = getHostGuestOSCategory(host); - if (hostGuestOSCategory == null) { - continue; - } else if (templateGuestOSCategory.equals(hostGuestOSCategory)) { - highPriorityHosts.add(host); - } else { - lowPriorityHosts.add(host); - } - } - - hostsToCheck.removeAll(highPriorityHosts); - hostsToCheck.removeAll(lowPriorityHosts); - - // Prioritize the remaining hosts by HVM capability - for (HostVO host : hostsToCheck) { - if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { - // Host and template both do not support hvm, put it as first consideration - prioritizedHosts.add(0, host); - } else { - // Template doesn't require hvm, but the machine supports it, make it last for consideration - prioritizedHosts.add(host); - } - } - - // Merge the lists - prioritizedHosts.addAll(0, highPriorityHosts); - prioritizedHosts.addAll(lowPriorityHosts); - - return prioritizedHosts; - } - - protected boolean hostSupportsHVM(HostVO host) { - // Determine host capabilities - String caps = host.getCapabilities(); - - if (caps != null) { - String[] tokens = caps.split(","); - for (String token : tokens) { - if (token.contains("hvm")) { - return true; - } - } - } - - return false; - } - - protected String getHostGuestOSCategory(HostVO host) { - DetailVO hostDetail = _hostDetailsDao.findDetail(host.getId(), "guest.os.category.id"); - if (hostDetail != null) { - String guestOSCategoryIdString = hostDetail.getValue(); - long guestOSCategoryId; - - try { - guestOSCategoryId = Long.parseLong(guestOSCategoryIdString); - } catch (Exception e) { - return null; - } - - GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); - - if (guestOSCategory != null) { - return guestOSCategory.getName(); - } else { - return null; - } - } else { - return null; - } - } - - protected String getTemplateGuestOSCategory(VirtualMachineTemplate template) { - long guestOSId = template.getGuestOSId(); - GuestOSVO guestOS = _guestOSDao.findById(guestOSId); - long guestOSCategoryId = guestOS.getCategoryId(); - GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); - return guestOSCategory.getName(); - } } diff --git a/server/src/com/cloud/host/dao/HostDao.java b/server/src/com/cloud/host/dao/HostDao.java index 1d9695ef612..7018f1e58c4 100644 --- a/server/src/com/cloud/host/dao/HostDao.java +++ b/server/src/com/cloud/host/dao/HostDao.java @@ -153,5 +153,10 @@ public interface HostDao extends GenericDao { */ List listBy(Long dataCenterId, Long podId, Long clusterId, Type hostType, Status... statuses); - List listBy(Long clusterId, Long podId, long dcId); + List listBy(Long clusterId, Long podId, long dcId); + + void loadHostTags(HostVO host); + + List listByHostTag(Host.Type type, Long clusterId, Long podId, long dcId, String hostTag); + } diff --git a/server/src/com/cloud/host/dao/HostDaoImpl.java b/server/src/com/cloud/host/dao/HostDaoImpl.java index 6cfd466d261..3320ed537f9 100644 --- a/server/src/com/cloud/host/dao/HostDaoImpl.java +++ b/server/src/com/cloud/host/dao/HostDaoImpl.java @@ -33,6 +33,7 @@ import org.apache.log4j.Logger; import com.cloud.host.Host; import com.cloud.host.Host.Type; +import com.cloud.host.HostTagVO; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.Status.Event; @@ -45,6 +46,7 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; @@ -90,7 +92,8 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected final Attribute _pingTimeAttr; protected final DetailsDaoImpl _detailsDao = ComponentLocator.inject(DetailsDaoImpl.class); - + protected final HostTagsDaoImpl _hostTagsDao = ComponentLocator.inject(HostTagsDaoImpl.class); + public HostDaoImpl() { MaintenanceCountSearch = createSearchBuilder(); @@ -327,7 +330,39 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao sc.setParameters("dc", dcId); return listBy(sc); } - + + @Override + public List listByHostTag(Host.Type type, Long clusterId, Long podId, long dcId, String hostTag) { + + SearchBuilder hostTagSearch = _hostTagsDao.createSearchBuilder(); + HostTagVO tagEntity = hostTagSearch.entity(); + hostTagSearch.and("tag",tagEntity.getTag(), SearchCriteria.Op.EQ); + + SearchBuilder hostSearch = createSearchBuilder(); + HostVO entity = hostSearch.entity(); + hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ); + hostSearch.and("pod", entity.getPodId(), SearchCriteria.Op.EQ); + hostSearch.and("dc", entity.getDataCenterId(), SearchCriteria.Op.EQ); + hostSearch.and("cluster", entity.getClusterId(), SearchCriteria.Op.EQ); + hostSearch.and("status", entity.getStatus(), SearchCriteria.Op.EQ); + hostSearch.join("hostTagSearch", hostTagSearch, entity.getId(), tagEntity.getHostId(), JoinBuilder.JoinType.INNER); + + + SearchCriteria sc = hostSearch.create(); + sc.setJoinParameters("hostTagSearch", "tag", hostTag); + sc.setParameters("type", type.toString()); + if (podId != null) { + sc.setParameters("pod", podId); + } + if (clusterId != null) { + sc.setParameters("cluster", clusterId); + } + sc.setParameters("dc", dcId); + sc.setParameters("status", Status.Up.toString()); + + return listBy(sc); + } + @Override public List listByCluster(long clusterId) { SearchCriteria sc = ClusterSearch.create(); @@ -379,7 +414,13 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao Map details =_detailsDao.findDetails(host.getId()); host.setDetails(details); } - + + @Override + public void loadHostTags(HostVO host){ + List hostTags = _hostTagsDao.gethostTags(host.getId()); + host.setHostTags(hostTags); + } + @Override public boolean updateStatus(HostVO host, Event event, long msId) { Status oldStatus = host.getStatus(); @@ -553,7 +594,15 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return; } _detailsDao.persist(host.getId(), details); - } + } + + protected void saveHostTags(HostVO host) { + List hostTags = host.getHostTags(); + if (hostTags == null || (hostTags != null && hostTags.isEmpty())) { + return; + } + _hostTagsDao.persist(host.getId(), hostTags); + } @Override @DB public HostVO persist(HostVO host) { @@ -574,6 +623,8 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao saveDetails(host); loadDetails(dbHost); + saveHostTags(host); + loadHostTags(dbHost); txn.commit(); @@ -591,6 +642,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao } saveDetails(host); + saveHostTags(host); txn.commit(); diff --git a/server/src/com/cloud/host/dao/HostTagsDao.java b/server/src/com/cloud/host/dao/HostTagsDao.java new file mode 100644 index 00000000000..d7ce813ee96 --- /dev/null +++ b/server/src/com/cloud/host/dao/HostTagsDao.java @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2010 Cloud.com, Inc. All rights reserved. + * + * This software is licensed under the GNU General Public License v3 or later. + * + * It is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package com.cloud.host.dao; + +import java.util.List; +import com.cloud.host.HostTagVO; +import com.cloud.utils.db.GenericDao; + +public interface HostTagsDao extends GenericDao { + + void persist(long hostId, List hostTags); + + List gethostTags(long hostId); + +} + diff --git a/server/src/com/cloud/host/dao/HostTagsDaoImpl.java b/server/src/com/cloud/host/dao/HostTagsDaoImpl.java new file mode 100644 index 00000000000..4d1738c6f31 --- /dev/null +++ b/server/src/com/cloud/host/dao/HostTagsDaoImpl.java @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2010 Cloud.com, Inc. All rights reserved. + * + * This software is licensed under the GNU General Public License v3 or later. + * + * It is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package com.cloud.host.dao; + +import java.util.ArrayList; +import java.util.List; +import javax.ejb.Local; +import com.cloud.host.HostTagVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; + + +@Local(value=HostTagsDao.class) +public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao { + protected final SearchBuilder HostSearch; + + protected HostTagsDaoImpl() { + HostSearch = createSearchBuilder(); + HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ); + HostSearch.done(); + } + + @Override + public List gethostTags(long hostId) { + SearchCriteria sc = HostSearch.create(); + sc.setParameters("hostId", hostId); + + List results = search(sc, null); + List hostTags = new ArrayList(results.size()); + for (HostTagVO result : results) { + hostTags.add(result.getTag()); + } + + return hostTags; + } + + @Override + public void persist(long hostId, List hostTags) { + Transaction txn = Transaction.currentTxn(); + + txn.start(); + for (String tag : hostTags) { + HostTagVO vo = new HostTagVO(hostId, tag); + persist(vo); + } + txn.commit(); + } +} diff --git a/server/src/com/cloud/migration/ServiceOffering21VO.java b/server/src/com/cloud/migration/ServiceOffering21VO.java index b8ec2dee368..3a779df8996 100644 --- a/server/src/com/cloud/migration/ServiceOffering21VO.java +++ b/server/src/com/cloud/migration/ServiceOffering21VO.java @@ -39,6 +39,9 @@ public class ServiceOffering21VO extends DiskOffering21VO implements ServiceOffe @Enumerated(EnumType.STRING) private Network.GuestIpType guestIpType; + @Column(name="host_tag") + private String hostTag; + protected ServiceOffering21VO() { super(); } @@ -53,6 +56,11 @@ public class ServiceOffering21VO extends DiskOffering21VO implements ServiceOffe this.offerHA = offerHA; this.guestIpType = guestIpType; } + + public ServiceOffering21VO(String name, int cpu, int ramSize, int speed, int rateMbps, int multicastRateMbps, boolean offerHA, String displayText, Network.GuestIpType guestIpType, boolean useLocalStorage, boolean recreatable, String tags, String hostTag) { + this(name, cpu, ramSize, speed, rateMbps, multicastRateMbps, offerHA, displayText, guestIpType, useLocalStorage, recreatable, tags); + this.hostTag = hostTag; + } @Override public boolean getOfferHA() { @@ -126,4 +134,12 @@ public class ServiceOffering21VO extends DiskOffering21VO implements ServiceOffe public String gethypervisorType() { return null; } + + public void setHostTag(String hostTag) { + this.hostTag = hostTag; + } + + public String getHostTag() { + return hostTag; + } } diff --git a/server/src/com/cloud/server/StatsCollector.java b/server/src/com/cloud/server/StatsCollector.java index 3c16644796e..18e214a6b3b 100755 --- a/server/src/com/cloud/server/StatsCollector.java +++ b/server/src/com/cloud/server/StatsCollector.java @@ -319,7 +319,7 @@ public class StatsCollector { capacitySC.addAnd("capacityType", SearchCriteria.Op.EQ, capacityType); List capacities = _capacityDao.search(capacitySC, null); if (capacities.size() == 0){ // Create a new one - CapacityVO capacity = new CapacityVO(host.getId(), host.getDataCenterId(), host.getPodId(), stats.getByteUsed(), stats.getCapacityBytes(), capacityType); + CapacityVO capacity = new CapacityVO(host.getId(), host.getDataCenterId(), host.getPodId(), host.getClusterId(), stats.getByteUsed(), stats.getCapacityBytes(), capacityType); _capacityDao.persist(capacity); }else{ //Update if it already exists. CapacityVO capacity = capacities.get(0); diff --git a/server/src/com/cloud/service/ServiceOfferingVO.java b/server/src/com/cloud/service/ServiceOfferingVO.java index 7c4b0d0626a..a95e0e54c5c 100644 --- a/server/src/com/cloud/service/ServiceOfferingVO.java +++ b/server/src/com/cloud/service/ServiceOfferingVO.java @@ -50,6 +50,9 @@ public class ServiceOfferingVO extends DiskOfferingVO implements ServiceOffering @Column(name="ha_enabled") private boolean offerHA; + + @Column(name="host_tag") + private String hostTag; protected ServiceOfferingVO() { super(); @@ -74,8 +77,13 @@ public class ServiceOfferingVO extends DiskOfferingVO implements ServiceOffering this.multicastRateMbps = multicastRateMbps; this.offerHA = offerHA; } - - @Override + + public ServiceOfferingVO(String name, int cpu, int ramSize, int speed, int rateMbps, int multicastRateMbps, boolean offerHA, String displayText, boolean useLocalStorage, boolean recreatable, String tags, boolean systemUse, Long domainId, String hostTag) { + this(name, cpu, ramSize, speed, rateMbps, multicastRateMbps, offerHA, displayText, useLocalStorage, recreatable, tags, systemUse, domainId); + this.hostTag = hostTag; + } + + @Override public boolean getOfferHA() { return offerHA; } @@ -138,6 +146,13 @@ public class ServiceOfferingVO extends DiskOfferingVO implements ServiceOffering @Override public int getMulticastRateMbps() { return multicastRateMbps; - } - + } + + public void setHostTag(String hostTag) { + this.hostTag = hostTag; + } + + public String getHostTag() { + return hostTag; + } } diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index 72e6cd5a43f..dc1cee9efdf 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -1,187 +1,187 @@ -/** - * Copyright (C) 2010 Cloud.com, Inc. All rights reserved. - * - * This software is licensed under the GNU General Public License v3 or later. - * - * It is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or any later version. - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - */ -package com.cloud.storage; - -import java.util.List; - -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.Command; -import com.cloud.agent.manager.Commands; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; -import com.cloud.deploy.DeployDestination; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientStorageCapacityException; -import com.cloud.exception.StorageUnavailableException; -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.storage.Volume.VolumeType; -import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.utils.component.Manager; -import com.cloud.vm.DiskProfile; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; - -public interface StorageManager extends Manager { - boolean canVmRestartOnAnotherServer(long vmId); - - /** Returns the absolute path of the specified ISO - * @param templateId - the ID of the template that represents the ISO - * @param datacenterId - * @return absolute ISO path - */ - public Pair getAbsoluteIsoPath(long templateId, long dataCenterId); - - /** - * Returns the URL of the secondary storage host - * @param zoneId - * @return URL - */ - public String getSecondaryStorageURL(long zoneId); - - /** - * Returns a comma separated list of tags for the specified storage pool - * @param poolId - * @return comma separated list of tags - */ - public String getStoragePoolTags(long poolId); - - /** - * Returns the secondary storage host - * @param zoneId - * @return secondary storage host - */ - public HostVO getSecondaryStorageHost(long zoneId); - - /** - * Add a pool to a host - * @param hostId - * @param pool - */ - boolean addPoolToHost(long hostId, StoragePoolVO pool); - - /** - * Moves a volume from its current storage pool to a storage pool with enough capacity in the specified zone, pod, or cluster - * @param volume - * @param destPoolDcId - * @param destPoolPodId - * @param destPoolClusterId - * @return VolumeVO - */ - VolumeVO moveVolume(VolumeVO volume, long destPoolDcId, Long destPoolPodId, Long destPoolClusterId, HypervisorType dataDiskHyperType); - - /** - * Create a volume based on the given criteria - * @param volume - * @param vm - * @param template - * @param dc - * @param pod - * @param clusterId - * @param offering - * @param diskOffering - * @param avoids - * @param size - * @param hyperType - * @return volume VO if success, null otherwise - */ - VolumeVO createVolume(VolumeVO volume, VMInstanceVO vm, VMTemplateVO template, DataCenterVO dc, HostPodVO pod, Long clusterId, - ServiceOfferingVO offering, DiskOfferingVO diskOffering, List avoids, long size, HypervisorType hyperType); - - /** - * Marks the specified volume as destroyed in the management server database. The expunge thread will delete the volume from its storage pool. - * @param volume - */ - void destroyVolume(VolumeVO volume) throws ConcurrentOperationException; - - /** Create capacity entries in the op capacity table - * @param storagePool - */ - public void createCapacityEntry(StoragePoolVO storagePool); - - /** - * Checks that the volume is stored on a shared storage pool - * @param volume - * @return true if the volume is on a shared storage pool, false otherwise - */ - boolean volumeOnSharedStoragePool(VolumeVO volume); - - Answer sendToPool(long poolId, Command cmd) throws StorageUnavailableException; - Answer sendToPool(StoragePool pool, Command cmd) throws StorageUnavailableException; - Answer[] sendToPool(long poolId, Commands cmd) throws StorageUnavailableException; - Answer[] sendToPool(StoragePool pool, Commands cmds) throws StorageUnavailableException; - Pair sendToPool(StoragePool pool, long[] hostIdsToTryFirst, List hostIdsToAvoid, Commands cmds) throws StorageUnavailableException; - Pair sendToPool(StoragePool pool, long[] hostIdsToTryFirst, List hostIdsToAvoid, Command cmd) throws StorageUnavailableException; - - /** - * Checks that one of the following is true: - * 1. The volume is not attached to any VM - * 2. The volume is attached to a VM that is running on a host with the KVM hypervisor, and the VM is stopped - * 3. The volume is attached to a VM that is running on a host with the XenServer hypervisor (the VM can be stopped or running) - * @return true if one of the above conditions is true - */ - boolean volumeInactive(VolumeVO volume); - - String getVmNameOnVolume(VolumeVO volume); - - /** - * Checks if a host has running VMs that are using its local storage pool. - * @return true if local storage is active on the host - */ - boolean isLocalStorageActiveOnHost(Host host); - - /** - * Cleans up storage pools by removing unused templates. - * @param recurring - true if this cleanup is part of a recurring garbage collection thread - */ - void cleanupStorage(boolean recurring); - - String getPrimaryStorageNameLabel(VolumeVO volume); - - /** - * Allocates one volume. - * @param - * @param type - * @param offering - * @param name - * @param size - * @param template - * @param vm - * @param account - * @return VolumeVO a persisted volume. - */ - DiskProfile allocateRawVolume(VolumeType type, String name, DiskOfferingVO offering, Long size, T vm, Account owner); - DiskProfile allocateTemplatedVolume(VolumeType type, String name, DiskOfferingVO offering, VMTemplateVO template, T vm, Account owner); - - void createCapacityEntry(StoragePoolVO storagePool, long allocated); - - - void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; - - void release(VirtualMachineProfile profile); - - void cleanupVolumes(long vmId) throws ConcurrentOperationException; - - void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest); - - Answer sendToPool(StoragePool pool, long[] hostIdsToTryFirst, Command cmd) throws StorageUnavailableException; -} +/** + * Copyright (C) 2010 Cloud.com, Inc. All rights reserved. + * + * This software is licensed under the GNU General Public License v3 or later. + * + * It is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package com.cloud.storage; + +import java.util.List; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.manager.Commands; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientStorageCapacityException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.Volume.VolumeType; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.component.Manager; +import com.cloud.vm.DiskProfile; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; + +public interface StorageManager extends Manager { + boolean canVmRestartOnAnotherServer(long vmId); + + /** Returns the absolute path of the specified ISO + * @param templateId - the ID of the template that represents the ISO + * @param datacenterId + * @return absolute ISO path + */ + public Pair getAbsoluteIsoPath(long templateId, long dataCenterId); + + /** + * Returns the URL of the secondary storage host + * @param zoneId + * @return URL + */ + public String getSecondaryStorageURL(long zoneId); + + /** + * Returns a comma separated list of tags for the specified storage pool + * @param poolId + * @return comma separated list of tags + */ + public String getStoragePoolTags(long poolId); + + /** + * Returns the secondary storage host + * @param zoneId + * @return secondary storage host + */ + public HostVO getSecondaryStorageHost(long zoneId); + + /** + * Add a pool to a host + * @param hostId + * @param pool + */ + boolean addPoolToHost(long hostId, StoragePoolVO pool); + + /** + * Moves a volume from its current storage pool to a storage pool with enough capacity in the specified zone, pod, or cluster + * @param volume + * @param destPoolDcId + * @param destPoolPodId + * @param destPoolClusterId + * @return VolumeVO + */ + VolumeVO moveVolume(VolumeVO volume, long destPoolDcId, Long destPoolPodId, Long destPoolClusterId, HypervisorType dataDiskHyperType); + + /** + * Create a volume based on the given criteria + * @param volume + * @param vm + * @param template + * @param dc + * @param pod + * @param clusterId + * @param offering + * @param diskOffering + * @param avoids + * @param size + * @param hyperType + * @return volume VO if success, null otherwise + */ + VolumeVO createVolume(VolumeVO volume, VMInstanceVO vm, VMTemplateVO template, DataCenterVO dc, HostPodVO pod, Long clusterId, + ServiceOfferingVO offering, DiskOfferingVO diskOffering, List avoids, long size, HypervisorType hyperType); + + /** + * Marks the specified volume as destroyed in the management server database. The expunge thread will delete the volume from its storage pool. + * @param volume + */ + void destroyVolume(VolumeVO volume) throws ConcurrentOperationException; + + /** Create capacity entries in the op capacity table + * @param storagePool + */ + public void createCapacityEntry(StoragePoolVO storagePool); + + /** + * Checks that the volume is stored on a shared storage pool + * @param volume + * @return true if the volume is on a shared storage pool, false otherwise + */ + boolean volumeOnSharedStoragePool(VolumeVO volume); + + Answer sendToPool(long poolId, Command cmd) throws StorageUnavailableException; + Answer sendToPool(StoragePool pool, Command cmd) throws StorageUnavailableException; + Answer[] sendToPool(long poolId, Commands cmd) throws StorageUnavailableException; + Answer[] sendToPool(StoragePool pool, Commands cmds) throws StorageUnavailableException; + Pair sendToPool(StoragePool pool, long[] hostIdsToTryFirst, List hostIdsToAvoid, Commands cmds) throws StorageUnavailableException; + Pair sendToPool(StoragePool pool, long[] hostIdsToTryFirst, List hostIdsToAvoid, Command cmd) throws StorageUnavailableException; + + /** + * Checks that one of the following is true: + * 1. The volume is not attached to any VM + * 2. The volume is attached to a VM that is running on a host with the KVM hypervisor, and the VM is stopped + * 3. The volume is attached to a VM that is running on a host with the XenServer hypervisor (the VM can be stopped or running) + * @return true if one of the above conditions is true + */ + boolean volumeInactive(VolumeVO volume); + + String getVmNameOnVolume(VolumeVO volume); + + /** + * Checks if a host has running VMs that are using its local storage pool. + * @return true if local storage is active on the host + */ + boolean isLocalStorageActiveOnHost(Host host); + + /** + * Cleans up storage pools by removing unused templates. + * @param recurring - true if this cleanup is part of a recurring garbage collection thread + */ + void cleanupStorage(boolean recurring); + + String getPrimaryStorageNameLabel(VolumeVO volume); + + /** + * Allocates one volume. + * @param + * @param type + * @param offering + * @param name + * @param size + * @param template + * @param vm + * @param account + * @return VolumeVO a persisted volume. + */ + DiskProfile allocateRawVolume(VolumeType type, String name, DiskOfferingVO offering, Long size, T vm, Account owner); + DiskProfile allocateTemplatedVolume(VolumeType type, String name, DiskOfferingVO offering, VMTemplateVO template, T vm, Account owner); + + void createCapacityEntry(StoragePoolVO storagePool, long allocated); + + + void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException, ConcurrentOperationException; + + void release(VirtualMachineProfile profile); + + void cleanupVolumes(long vmId) throws ConcurrentOperationException; + + void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest); + + Answer sendToPool(StoragePool pool, long[] hostIdsToTryFirst, Command cmd) throws StorageUnavailableException; +} diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 71f8b95a71e..ea0e894693b 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -175,6 +175,7 @@ import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachine.Type; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.ConsoleProxyDao; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.SecondaryStorageVmDao; @@ -330,12 +331,13 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag protected StoragePoolVO findStoragePool(DiskProfile dskCh, final DataCenterVO dc, HostPodVO pod, Long clusterId, final ServiceOffering offering, final VMInstanceVO vm, final VMTemplateVO template, final Set avoid) { + VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vm, template, (ServiceOfferingVO)offering, null, null); Enumeration en = _storagePoolAllocators.enumeration(); while (en.hasMoreElements()) { final StoragePoolAllocator allocator = en.nextElement(); - final StoragePool pool = allocator.allocateToPool(dskCh, dc, pod, clusterId, vm, template, avoid); - if (pool != null) { - return (StoragePoolVO) pool; + final List poolList = allocator.allocateToPool(dskCh, vmProfile, dc.getId(), pod.getId(), clusterId, avoid, 1); + if (poolList != null && !poolList.isEmpty()) { + return (StoragePoolVO) poolList.get(0); } } return null; @@ -1560,7 +1562,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag List capacities = _capacityDao.search(capacitySC, null); if (capacities.size() == 0) { - CapacityVO capacity = new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), + CapacityVO capacity = new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), storagePool.getClusterId(), storagePool.getAvailableBytes(), storagePool.getCapacityBytes(), CapacityVO.CAPACITY_TYPE_STORAGE); _capacityDao.persist(capacity); } else { @@ -1587,7 +1589,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag provFactor = _overProvisioningFactor; } if (capacities.size() == 0) { - CapacityVO capacity = new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), allocated, + CapacityVO capacity = new CapacityVO(storagePool.getId(), storagePool.getDataCenterId(), storagePool.getPodId(), storagePool.getClusterId(), allocated, storagePool.getCapacityBytes() * provFactor, CapacityVO.CAPACITY_TYPE_STORAGE_ALLOCATED); _capacityDao.persist(capacity); } else { @@ -2346,17 +2348,6 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag return toDiskProfile(vol, offering); } - protected StoragePool findStorage(DiskProfile dskCh, DeployDestination dest, VirtualMachineProfile vm, - List alreadyAllocated, Set avoid) { - for (StoragePoolAllocator allocator : _storagePoolAllocators) { - StoragePool pool = allocator.allocateTo(dskCh, vm, dest, alreadyAllocated, avoid); - if (pool != null) { - return pool; - } - } - return null; - } - @Override public void prepareForMigration(VirtualMachineProfile vm, DeployDestination dest) { List vols = _volsDao.findUsableVolumesForInstance(vm.getId()); @@ -2383,16 +2374,17 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } } + @Override public void prepare(VirtualMachineProfile vm, DeployDestination dest) throws StorageUnavailableException, InsufficientStorageCapacityException { - List vols = _volsDao.findUsableVolumesForInstance(vm.getId()); + List vols = _volsDao.findUsableVolumesForInstance(vm.getId()); if (s_logger.isDebugEnabled()) { s_logger.debug("Preparing " + vols.size() + " volumes for " + vm); } - + List recreateVols = new ArrayList(vols.size()); - + for (VolumeVO vol : vols) { Volume.State state = vol.getState(); if (state == Volume.State.Ready) { @@ -2436,6 +2428,12 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } else { newVol = switchVolume(vol); newVol.setRecreatable(true); + //update the volume->storagePool map since volumeId has changed + if(dest.getStorageForDisks() != null && dest.getStorageForDisks().containsKey(vol)){ + StoragePool poolWithOldVol = dest.getStorageForDisks().get(vol); + dest.getStorageForDisks().put(newVol, poolWithOldVol); + dest.getStorageForDisks().remove(vol); + } if (s_logger.isDebugEnabled()) { s_logger.debug("Created new volume " + newVol + " for old volume " + vol); } @@ -2503,13 +2501,11 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag template = _templateDao.findById(toBeCreated.getTemplateId()); } - Set avoids = new HashSet(); - StoragePool pool = null; - while ((pool = findStorage(diskProfile, dest, vm, alreadyCreated, avoids)) != null) { + if(dest.getStorageForDisks() != null){ + StoragePool pool = dest.getStorageForDisks().get(toBeCreated); if (s_logger.isDebugEnabled()) { s_logger.debug("Trying to create in " + pool); } - avoids.add(pool); toBeCreated.setPoolId(pool.getId()); try { _volsDao.update(toBeCreated, Volume.Event.OperationRetry); @@ -2522,8 +2518,8 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag VMTemplateStoragePoolVO tmpltStoredOn = null; tmpltStoredOn = _tmpltMgr.prepareTemplateForCreate(template, pool); if (tmpltStoredOn == null) { - s_logger.debug("Skipping " + pool + " because we can't propragate template " + template); - continue; + s_logger.debug("Cannot use this pool " + pool + " because we can't propagate template " + template); + return null; } cmd = new CreateCommand(diskProfile, tmpltStoredOn.getLocalDownloadPath(), new StorageFilerTO(pool)); } else { diff --git a/server/src/com/cloud/storage/allocator/AbstractStoragePoolAllocator.java b/server/src/com/cloud/storage/allocator/AbstractStoragePoolAllocator.java index de9a1a87d09..740656cd5b0 100644 --- a/server/src/com/cloud/storage/allocator/AbstractStoragePoolAllocator.java +++ b/server/src/com/cloud/storage/allocator/AbstractStoragePoolAllocator.java @@ -29,10 +29,9 @@ import org.apache.log4j.Logger; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.server.StatsCollector; @@ -47,7 +46,6 @@ import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.Volume; import com.cloud.storage.Volume.VolumeType; import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.StoragePoolHostDao; @@ -126,25 +124,48 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement protected boolean poolIsCorrectType(DiskProfile dskCh, StoragePool pool, VMInstanceVO vm) { boolean localStorageAllocationNeeded = localStorageAllocationNeeded(dskCh, vm); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Is localStorageAllocationNeeded? "+ localStorageAllocationNeeded); + s_logger.debug("Is storage pool shared? "+ pool.getPoolType().isShared()); + } + return ((!localStorageAllocationNeeded && pool.getPoolType().isShared()) || (localStorageAllocationNeeded && !pool.getPoolType().isShared())); } - protected boolean checkPool(Set avoid, StoragePoolVO pool, DiskProfile dskCh, VMTemplateVO template, List templatesInPool, + protected boolean checkPool(ExcludeList avoid, StoragePoolVO pool, DiskProfile dskCh, VMTemplateVO template, List templatesInPool, VMInstanceVO vm, StatsCollector sc) { - if (avoid.contains(pool)) { + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Checking if storage pool is suitable, name: " + pool.getName()+ " ,poolId: "+ pool.getId()); + } + + if (avoid.shouldAvoid(pool)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("StoragePool is in avoid set, skipping this pool"); + } return false; } if(dskCh.getType().equals(VolumeType.ROOT) && pool.getPoolType().equals(StoragePoolType.Iscsi)){ + if (s_logger.isDebugEnabled()) { + s_logger.debug("Disk needed for ROOT volume, but StoragePoolType is Iscsi, skipping this and trying other available pools"); + } return false; } //by default, all pools are up when successfully added //don't return the pool if not up (if in maintenance/prepareformaintenance/errorinmaintenance) - if(!pool.getStatus().equals(StoragePoolStatus.Up)) + if(!pool.getStatus().equals(StoragePoolStatus.Up)){ + if (s_logger.isDebugEnabled()) { + s_logger.debug("StoragePool status is not UP, status is: "+pool.getStatus().name()+", skipping this pool"); + } return false; + } // Check that the pool type is correct if (!poolIsCorrectType(dskCh, pool, vm)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("StoragePool is not of correct type, skipping this pool"); + } return false; } @@ -154,6 +175,9 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement Long clusterId = pool.getClusterId(); ClusterVO cluster = _clusterDao.findById(clusterId); if (!(cluster.getHypervisorType() == dskCh.getHypersorType())) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("StoragePool's Cluster does not have required hypervisorType, skipping this pool"); + } return false; } @@ -229,7 +253,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement if ((pool.getCapacityBytes() * storageOverprovisioningFactor) < (totalAllocatedSize + askingSize)) { if (s_logger.isDebugEnabled()) { - s_logger.debug("Found pool " + pool.getId() + " for storage, maxSize : " + (pool.getCapacityBytes() * storageOverprovisioningFactor) + ", totalSize : " + totalAllocatedSize + ", askingSize : " + askingSize); + s_logger.debug("Cannot allocate this pool " + pool.getId() + " for storage, not enough storage, maxSize : " + (pool.getCapacityBytes() * storageOverprovisioningFactor) + ", totalSize : " + totalAllocatedSize + ", askingSize : " + askingSize); } return false; @@ -243,12 +267,17 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement return storage.getStorageIpAddress(); } + @Override - public StoragePool allocateTo(DiskProfile dskCh, VirtualMachineProfile vm, DeployDestination dest, List disks, Set avoids) { + public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vm, long dcId, long podId, Long clusterId, Set avoids, int returnUpTo) { - VMInstanceVO instance = (VMInstanceVO)(vm.getVirtualMachine()); + ExcludeList avoid = new ExcludeList(); + for(StoragePool pool : avoids){ + avoid.addPool(pool.getId()); + } - VMTemplateVO template = _templateDao.findById(instance.getTemplateId()); - return allocateToPool(dskCh, (DataCenterVO)dest.getDataCenter(), (HostPodVO)dest.getPod(), dest.getCluster().getId(), instance, template, avoids); + DataCenterDeployment plan = new DataCenterDeployment(dcId, podId, clusterId, null); + return allocateToPool(dskCh, vm, plan, avoid, returnUpTo); } + } diff --git a/server/src/com/cloud/storage/allocator/FirstFitStoragePoolAllocator.java b/server/src/com/cloud/storage/allocator/FirstFitStoragePoolAllocator.java index 731bb2ec0b3..e24f605c395 100644 --- a/server/src/com/cloud/storage/allocator/FirstFitStoragePoolAllocator.java +++ b/server/src/com/cloud/storage/allocator/FirstFitStoragePoolAllocator.java @@ -17,22 +17,23 @@ */ package com.cloud.storage.allocator; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Set; - import javax.ejb.Local; import org.apache.log4j.Logger; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.server.StatsCollector; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolVO; import com.cloud.storage.VMTemplateVO; import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; @Local(value=StoragePoolAllocator.class) public class FirstFitStoragePoolAllocator extends AbstractStoragePoolAllocator { @@ -44,35 +45,57 @@ public class FirstFitStoragePoolAllocator extends AbstractStoragePoolAllocator { } @Override - public StoragePool allocateToPool(DiskProfile dskCh, DataCenterVO dc, HostPodVO pod, Long clusterId, - VMInstanceVO vm, VMTemplateVO template, Set avoid) { - // Check that the allocator type is correct + public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { + + VMInstanceVO vm = (VMInstanceVO)(vmProfile.getVirtualMachine()); + + VMTemplateVO template = _templateDao.findById(vm.getTemplateId()); + + List suitablePools = new ArrayList(); + + // Check that the allocator type is correct if (!allocatorIsCorrectType(dskCh, vm)) { - return null; + return suitablePools; + } + long dcId = plan.getDataCenterId(); + long podId = plan.getPodId(); + long clusterId = plan.getClusterId(); + + if(dskCh.getTags() != null && dskCh.getTags().length != 0){ + s_logger.debug("Looking for pools in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId + " having tags:" + dskCh.getTags()); + }else{ + s_logger.debug("Looking for pools in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); } - List pools = _storagePoolDao.findPoolsByTags(dc.getId(), pod.getId(), clusterId, dskCh.getTags(), null); + List pools = _storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags(), null); if (pools.size() == 0) { if (s_logger.isDebugEnabled()) { - s_logger.debug("No storage pools available for pod id : " + pod.getId()); + s_logger.debug("No storage pools available for allocation, returning"); } - return null; + return suitablePools; } StatsCollector sc = StatsCollector.getInstance(); Collections.shuffle(pools); + if (s_logger.isDebugEnabled()) { + s_logger.debug("FirstFitStoragePoolAllocator has " + pools.size() + " pools to check for allocation"); + } + for (StoragePoolVO pool: pools) { + if(suitablePools.size() == returnUpTo){ + break; + } if (checkPool(avoid, pool, dskCh, template, null, vm, sc)) { - return pool; + suitablePools.add(pool); } } if (s_logger.isDebugEnabled()) { - s_logger.debug("Unable to find any storage pool"); - } + s_logger.debug("FirstFitStoragePoolAllocator returning "+suitablePools.size() +" suitable storage pools"); + } - return null; + return suitablePools; } } diff --git a/server/src/com/cloud/storage/allocator/GarbageCollectingStoragePoolAllocator.java b/server/src/com/cloud/storage/allocator/GarbageCollectingStoragePoolAllocator.java index 2d77c74c136..01d0c4d2ed4 100644 --- a/server/src/com/cloud/storage/allocator/GarbageCollectingStoragePoolAllocator.java +++ b/server/src/com/cloud/storage/allocator/GarbageCollectingStoragePoolAllocator.java @@ -17,9 +17,8 @@ */ package com.cloud.storage.allocator; -import java.util.HashSet; +import java.util.List; import java.util.Map; -import java.util.Set; import javax.ejb.Local; import javax.naming.ConfigurationException; @@ -27,14 +26,15 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import com.cloud.configuration.dao.ConfigurationDao; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; -import com.cloud.storage.VMTemplateVO; import com.cloud.utils.component.ComponentLocator; import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; @Local(value=StoragePoolAllocator.class) public class GarbageCollectingStoragePoolAllocator extends AbstractStoragePoolAllocator { @@ -60,13 +60,7 @@ public class GarbageCollectingStoragePoolAllocator extends AbstractStoragePoolAl } @Override - public StoragePool allocateToPool(DiskProfile dskCh, - DataCenterVO dc, - HostPodVO pod, - Long clusterId, - VMInstanceVO vm, - VMTemplateVO template, - Set avoid) { + public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { if (!_storagePoolCleanupEnabled) { s_logger.debug("Storage pool cleanup is not enabled, so GarbageCollectingStoragePoolAllocator is being skipped."); @@ -75,7 +69,7 @@ public class GarbageCollectingStoragePoolAllocator extends AbstractStoragePoolAl // Clean up all storage pools _storageMgr.cleanupStorage(false); - + VMInstanceVO vm = (VMInstanceVO)(vmProfile.getVirtualMachine()); // Determine what allocator to use StoragePoolAllocator allocator; if (localStorageAllocationNeeded(dskCh, vm)) { @@ -85,12 +79,9 @@ public class GarbageCollectingStoragePoolAllocator extends AbstractStoragePoolAl } // Try to find a storage pool after cleanup - Set myAvoids = new HashSet(); - for (StoragePool pool : avoid) { - myAvoids.add(pool); - } + ExcludeList myAvoids = new ExcludeList(avoid.getDataCentersToAvoid(), avoid.getPodsToAvoid(), avoid.getClustersToAvoid(), avoid.getHostsToAvoid(), avoid.getPoolsToAvoid()); - return allocator.allocateToPool(dskCh, dc, pod, clusterId, vm, template, myAvoids); + return allocator.allocateToPool(dskCh, vmProfile, plan, myAvoids, returnUpTo); } @Override diff --git a/server/src/com/cloud/storage/allocator/LocalStoragePoolAllocator.java b/server/src/com/cloud/storage/allocator/LocalStoragePoolAllocator.java index cea57238299..b593e91dc59 100644 --- a/server/src/com/cloud/storage/allocator/LocalStoragePoolAllocator.java +++ b/server/src/com/cloud/storage/allocator/LocalStoragePoolAllocator.java @@ -17,10 +17,9 @@ */ package com.cloud.storage.allocator; -import java.util.HashSet; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Set; import javax.ejb.Local; import javax.naming.ConfigurationException; @@ -30,14 +29,13 @@ import org.apache.log4j.Logger; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.dao.ConfigurationDao; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.offering.ServiceOffering; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; -import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.utils.DateUtil; @@ -54,7 +52,6 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; @@ -89,54 +86,69 @@ public class LocalStoragePoolAllocator extends FirstFitStoragePoolAllocator { } @Override - public StoragePool allocateToPool(DiskProfile dskCh, - DataCenterVO dc, - HostPodVO pod, - Long clusterId, - VMInstanceVO vm, - VMTemplateVO template, - Set avoid) { + public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { + + VMInstanceVO vm = (VMInstanceVO)(vmProfile.getVirtualMachine()); + List suitablePools = new ArrayList(); // Check that the allocator type is correct if (!allocatorIsCorrectType(dskCh, vm)) { - return null; + return suitablePools; } - Set myAvoids = new HashSet(avoid); - VirtualMachineProfile vmc = new VirtualMachineProfileImpl(vm.getType()); - StoragePool pool = null; - while ((pool = super.allocateToPool(dskCh, dc, pod, clusterId, vm, template, myAvoids)) != null) { - myAvoids.add(pool); + ExcludeList myAvoids = new ExcludeList(avoid.getDataCentersToAvoid(), avoid.getPodsToAvoid(), avoid.getClustersToAvoid(), avoid.getHostsToAvoid(), avoid.getPoolsToAvoid()); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("LocalStoragePoolAllocator trying to find storage pool to fit the vm"); + } + + List availablePool; + while (!(availablePool = super.allocateToPool(dskCh, vmProfile, plan, myAvoids, 1)).isEmpty()) { + StoragePool pool = availablePool.get(0); + myAvoids.addPool(pool.getId()); if (pool.getPoolType().isShared()) { - return pool; + suitablePools.add(pool); + }else{ + List hostsInSPool = _poolHostDao.listByPoolId(pool.getId()); + assert(hostsInSPool.size() == 1) : "Local storage pool should be one host per pool"; + + StoragePoolHostVO spHost = hostsInSPool.get(0); + + SearchCriteria sc = VmsOnPoolSearch.create(); + sc.setJoinParameters("volumeJoin", "poolId", pool.getId()); + sc.setParameters("state", State.Expunging); + List vmsOnHost = _vmInstanceDao.customSearchIncludingRemoved(sc, null); + + if(s_logger.isDebugEnabled()) { + s_logger.debug("Found " + vmsOnHost.size() + " VM instances are alloacated at host " + spHost.getHostId() + " with local storage pool " + pool.getName()); + for(Long vmId : vmsOnHost) { + s_logger.debug("VM " + vmId + " is allocated on host " + spHost.getHostId() + " with local storage pool " + pool.getName()); + } + } + + if(hostHasCpuMemoryCapacity(spHost.getHostId(), vmsOnHost, vm)) { + s_logger.debug("Found suitable local storage pool " + pool.getId() + ", adding to list"); + suitablePools.add(pool); + }else{ + s_logger.debug("Found pool " + pool.getId() + " but host doesn't fit, skipping this pool"); + } } - List hostsInSPool = _poolHostDao.listByPoolId(pool.getId()); - assert(hostsInSPool.size() == 1) : "Local storage pool should be one host per pool"; - - StoragePoolHostVO spHost = hostsInSPool.get(0); - - SearchCriteria sc = VmsOnPoolSearch.create(); - sc.setJoinParameters("volumeJoin", "poolId", pool.getId()); - sc.setParameters("state", State.Expunging); - List vmsOnHost = _vmInstanceDao.customSearchIncludingRemoved(sc, null); - - if(s_logger.isDebugEnabled()) { - s_logger.debug("Found " + vmsOnHost.size() + " VM instances are alloacated at host " + spHost.getHostId() + " with local storage pool " + pool.getName()); - for(Long vmId : vmsOnHost) { - s_logger.debug("VM " + vmId + " is allocated on host " + spHost.getHostId() + " with local storage pool " + pool.getName()); - } + if(suitablePools.size() == returnUpTo){ + break; } - - if(hostHasCpuMemoryCapacity(spHost.getHostId(), vmsOnHost, vm)) { - return pool; - } - - s_logger.debug("Found pool " + pool.getId() + " but host doesn't fit."); } - s_logger.debug("Unable to find storage pool to fit the vm"); - return null; + if (s_logger.isDebugEnabled()) { + s_logger.debug("LocalStoragePoolAllocator returning "+suitablePools.size() +" suitable storage pools"); + } + + if(suitablePools.isEmpty()){ + if (s_logger.isDebugEnabled()) { + s_logger.debug("Unable to find storage pool to fit the vm"); + } + } + return suitablePools; } private boolean hostHasCpuMemoryCapacity(long hostId, List vmOnHost, VMInstanceVO vm) { diff --git a/server/src/com/cloud/storage/allocator/RandomStoragePoolAllocator.java b/server/src/com/cloud/storage/allocator/RandomStoragePoolAllocator.java index 3501eba9e64..0c3ba7887a7 100644 --- a/server/src/com/cloud/storage/allocator/RandomStoragePoolAllocator.java +++ b/server/src/com/cloud/storage/allocator/RandomStoragePoolAllocator.java @@ -17,22 +17,24 @@ */ package com.cloud.storage.allocator; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Set; import javax.ejb.Local; import org.apache.log4j.Logger; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.server.StatsCollector; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolVO; import com.cloud.storage.VMTemplateVO; import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; @Local(value=StoragePoolAllocator.class) public class RandomStoragePoolAllocator extends AbstractStoragePoolAllocator { @@ -44,33 +46,47 @@ public class RandomStoragePoolAllocator extends AbstractStoragePoolAllocator { } @Override - public StoragePool allocateToPool(DiskProfile dskCh, - DataCenterVO dc, HostPodVO pod, Long clusterId, VMInstanceVO vm, - VMTemplateVO template, Set avoid) { + public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { + + List suitablePools = new ArrayList(); + VMInstanceVO vm = (VMInstanceVO)(vmProfile.getVirtualMachine()); + VMTemplateVO template = _templateDao.findById(vm.getTemplateId()); // Check that the allocator type is correct if (!allocatorIsCorrectType(dskCh, vm)) { - return null; + return suitablePools; } - - List pools = _storagePoolDao.listBy(dc.getId(), pod.getId(), clusterId); + long dcId = plan.getDataCenterId(); + long podId = plan.getPodId(); + long clusterId = plan.getClusterId(); + s_logger.debug("Looking for pools in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); + List pools = _storagePoolDao.listBy(dcId, podId, clusterId); if (pools.size() == 0) { if (s_logger.isDebugEnabled()) { - s_logger.debug("No storage pools available for pod id : " + pod.getId()); + s_logger.debug("No storage pools available for allocation, returning"); } - return null; + return suitablePools; } StatsCollector sc = StatsCollector.getInstance(); Collections.shuffle(pools); - + if (s_logger.isDebugEnabled()) { + s_logger.debug("RandomStoragePoolAllocator has " + pools.size() + " pools to check for allocation"); + } for (StoragePoolVO pool: pools) { + if(suitablePools.size() == returnUpTo){ + break; + } if (checkPool(avoid, pool, dskCh, template, null, vm, sc)) { - return pool; + suitablePools.add(pool); } } - - return null; + + if (s_logger.isDebugEnabled()) { + s_logger.debug("RandomStoragePoolAllocator returning "+suitablePools.size() +" suitable storage pools"); + } + + return suitablePools; } } diff --git a/server/src/com/cloud/storage/allocator/StoragePoolAllocator.java b/server/src/com/cloud/storage/allocator/StoragePoolAllocator.java index 44c538ef637..f28f0ea42a8 100644 --- a/server/src/com/cloud/storage/allocator/StoragePoolAllocator.java +++ b/server/src/com/cloud/storage/allocator/StoragePoolAllocator.java @@ -19,17 +19,12 @@ package com.cloud.storage.allocator; import java.util.List; import java.util.Set; - -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; -import com.cloud.deploy.DeployDestination; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.storage.StoragePool; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.Volume; import com.cloud.utils.component.Adapter; import com.cloud.vm.DiskProfile; -import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; @@ -39,9 +34,20 @@ import com.cloud.vm.VirtualMachineProfile; */ public interface StoragePoolAllocator extends Adapter { - StoragePool allocateToPool(DiskProfile dskCh, DataCenterVO dc, HostPodVO pod, Long cluster, VMInstanceVO vm, VMTemplateVO template, Set avoids); + //keeping since storageMgr is using this API for some existing functionalities + List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vm, long dcId, long podId, Long clusterId, Set avoids, int returnUpTo); String chooseStorageIp(VirtualMachine vm, Host host, Host storage); - - StoragePool allocateTo(DiskProfile dskCh, VirtualMachineProfile vm, DeployDestination dest, List disks, Set avoids); + + /** + * Determines which storage pools are suitable for the guest virtual machine + * + * @param DiskProfile dskCh + * @param VirtualMachineProfile vmProfile + * @param DeploymentPlan plan + * @param ExcludeList avoid + * @param int returnUpTo (use -1 to return all possible pools) + * @return List List of storage pools that are suitable for the VM + **/ + List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid, int returnUpTo); } diff --git a/server/src/com/cloud/storage/allocator/UseLocalForRootAllocator.java b/server/src/com/cloud/storage/allocator/UseLocalForRootAllocator.java index 2875e1dd10e..93c313bd38e 100644 --- a/server/src/com/cloud/storage/allocator/UseLocalForRootAllocator.java +++ b/server/src/com/cloud/storage/allocator/UseLocalForRootAllocator.java @@ -17,36 +17,36 @@ */ package com.cloud.storage.allocator; +import java.util.List; import java.util.Map; -import java.util.Set; import javax.ejb.Local; import javax.naming.ConfigurationException; import com.cloud.configuration.Config; import com.cloud.configuration.dao.ConfigurationDao; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.HostPodVO; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.storage.StoragePool; -import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume.VolumeType; import com.cloud.utils.component.ComponentLocator; import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; @Local(value=StoragePoolAllocator.class) public class UseLocalForRootAllocator extends LocalStoragePoolAllocator implements StoragePoolAllocator { boolean _useLocalStorage; @Override - public StoragePool allocateToPool(DiskProfile dskCh, DataCenterVO dc, HostPodVO pod, Long clusterId, VMInstanceVO vm, VMTemplateVO template, Set avoids) { + public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { if (!_useLocalStorage) { return null; } - return super.allocateToPool(dskCh, dc, pod, clusterId, vm, template, avoids); + return super.allocateToPool(dskCh, vmProfile, plan, avoid, returnUpTo); } @Override diff --git a/server/src/com/cloud/storage/dao/VolumeDao.java b/server/src/com/cloud/storage/dao/VolumeDao.java index 03ff2a47156..9d70edd2cb4 100755 --- a/server/src/com/cloud/storage/dao/VolumeDao.java +++ b/server/src/com/cloud/storage/dao/VolumeDao.java @@ -57,4 +57,6 @@ public interface VolumeDao extends GenericDao { List listVolumesToBeDestroyed(); ImageFormat getImageFormat(Long volumeId); + + List findReadyRootVolumesByInstance(long instanceId); } diff --git a/server/src/com/cloud/storage/dao/VolumeDaoImpl.java b/server/src/com/cloud/storage/dao/VolumeDaoImpl.java index 9d6a3214110..3e9e09c5391 100755 --- a/server/src/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/server/src/com/cloud/storage/dao/VolumeDaoImpl.java @@ -58,7 +58,6 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol protected final GenericSearchBuilder TotalSizeByPoolSearch; protected final GenericSearchBuilder ActiveTemplateSearch; protected final SearchBuilder InstanceStatesSearch; - protected final SearchBuilder AllFieldsSearch; protected final Attribute _stateAttr; @@ -163,6 +162,15 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol return listIncludingRemovedBy(sc); } + @Override + public List findReadyRootVolumesByInstance(long instanceId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("instanceId", instanceId); + sc.setParameters("state", Volume.State.Ready); + sc.setParameters("vType", Volume.VolumeType.ROOT.toString()); + return listIncludingRemovedBy(sc); + } + @Override public List findByAccountAndPod(long accountId, long podId) { SearchCriteria sc = AllFieldsSearch.create(); @@ -326,7 +334,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol InstanceStatesSearch.and("instance", InstanceStatesSearch.entity().getInstanceId(), Op.EQ); InstanceStatesSearch.and("states", InstanceStatesSearch.entity().getState(), Op.IN); InstanceStatesSearch.done(); - + _stateAttr = _allAttributes.get("state"); assert _stateAttr != null : "Couldn't get the state attribute"; } diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index d544eb01673..0b6759968e6 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -33,6 +33,7 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; + import com.cloud.agent.AgentManager; import com.cloud.agent.AgentManager.OnError; import com.cloud.agent.Listener; @@ -102,11 +103,14 @@ import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePoolVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.Volume.VolumeType; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; @@ -171,6 +175,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene @Inject protected HighAvailabilityManager _haMgr; @Inject protected HostPodDao _podDao; @Inject protected DataCenterDao _dcDao; + @Inject protected StoragePoolDao _storagePoolDao; @Inject protected HypervisorGuruManager _hvGuruMgr; @Inject(adapter=DeploymentPlanner.class) @@ -542,8 +547,27 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene ExcludeList avoids = new ExcludeList(); int retry = _retry; while (retry-- != 0) { // It's != so that it can match -1. + + //edit plan if this vm's ROOT volume is in READY state already + List vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); - VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vm, template, offering, null, params); + for (VolumeVO vol : vols) { + Volume.State state = vol.getState(); + if (state == Volume.State.Ready) { + StoragePoolVO pool = _storagePoolDao.findById(vol.getPoolId()); + if (!pool.isInMaintenance()) { + long rootVolDcId = pool.getDataCenterId(); + Long rootVolPodId = pool.getPodId(); + Long rootVolClusterId = pool.getClusterId(); + plan = new DataCenterDeployment(rootVolDcId, rootVolPodId, rootVolClusterId, vol.getPoolId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Root Volume " + vol + " is ready, changing deployment plan to use this pool's datacenterId: "+rootVolDcId +" , podId: "+rootVolPodId +" , and clusterId: "+rootVolClusterId); + } + } + } + } + + VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vm, template, offering, account, params); DeployDestination dest = null; for (DeploymentPlanner planner : _planners) { dest = planner.plan(vmProfile, plan, avoids); @@ -566,7 +590,6 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene } try { - _storageMgr.prepare(vmProfile, dest); _networkMgr.prepare(vmProfile, dest, ctx); @@ -579,6 +602,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx); vm.setPodId(dest.getPod().getId()); + work = _workDao.findById(work.getId()); if (work == null || work.getStep() != Step.Prepare) { throw new ConcurrentOperationException("Work steps have been changed: " + work); diff --git a/server/src/com/cloud/vm/dao/UserVmDao.java b/server/src/com/cloud/vm/dao/UserVmDao.java index 2a6e2f6332a..4ab64106165 100755 --- a/server/src/com/cloud/vm/dao/UserVmDao.java +++ b/server/src/com/cloud/vm/dao/UserVmDao.java @@ -68,4 +68,6 @@ public interface UserVmDao extends GenericDao { void saveDetails(UserVmVO vm); + List listPodIdsHavingVmsforAccount(long zoneId, long accountId); + } diff --git a/server/src/com/cloud/vm/dao/UserVmDaoImpl.java b/server/src/com/cloud/vm/dao/UserVmDaoImpl.java index 4202f81de75..db9ae67bc64 100755 --- a/server/src/com/cloud/vm/dao/UserVmDaoImpl.java +++ b/server/src/com/cloud/vm/dao/UserVmDaoImpl.java @@ -17,6 +17,10 @@ */ package com.cloud.vm.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; @@ -25,13 +29,21 @@ import javax.ejb.Local; import org.apache.log4j.Logger; +import com.cloud.capacity.CapacityVO; +import com.cloud.host.HostVO; import com.cloud.uservm.UserVm; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.db.Attribute; +import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.SearchCriteria.Func; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.NicVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine.State; @@ -54,8 +66,14 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use protected final SearchBuilder DestroySearch; protected SearchBuilder AccountDataCenterVirtualSearch; + protected GenericSearchBuilder CountByAccountPod; + protected GenericSearchBuilder PodsHavingVmsForAccount; + protected SearchBuilder UserVmSearch; protected final Attribute _updateTimeAttr; + + private static final String LIST_PODS_HAVING_VMS_FOR_ACCOUNT = "SELECT pod_id FROM cloud.vm_instance WHERE data_center_id = ? AND account_id = ? AND pod_id IS NOT NULL GROUP BY pod_id HAVING count(id) > 0 ORDER BY count(id) DESC"; + protected final UserVmDetailsDaoImpl _detailsDao = ComponentLocator.inject(UserVmDetailsDaoImpl.class); @@ -119,6 +137,12 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use AccountHostSearch.and("hostId", AccountHostSearch.entity().getHostId(), SearchCriteria.Op.EQ); AccountHostSearch.done(); + CountByAccountPod = createSearchBuilder(Long.class); + CountByAccountPod.select(null, Func.COUNT, null); + CountByAccountPod.and("account", CountByAccountPod.entity().getAccountId(), SearchCriteria.Op.EQ); + CountByAccountPod.and("pod", CountByAccountPod.entity().getPodId(), SearchCriteria.Op.EQ); + CountByAccountPod.done(); + _updateTimeAttr = _allAttributes.get("updateTime"); assert _updateTimeAttr != null : "Couldn't get this updateTime attribute"; } @@ -278,4 +302,28 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use } _detailsDao.persist(vm.getId(), details); } + + @Override + public List listPodIdsHavingVmsforAccount(long zoneId, long accountId){ + Transaction txn = Transaction.currentTxn(); + PreparedStatement pstmt = null; + List result = new ArrayList(); + + try { + String sql = LIST_PODS_HAVING_VMS_FOR_ACCOUNT; + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, zoneId); + pstmt.setLong(2, accountId); + + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.add(rs.getLong(1)); + } + return result; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + LIST_PODS_HAVING_VMS_FOR_ACCOUNT, e); + } catch (Throwable e) { + throw new CloudRuntimeException("Caught: " + LIST_PODS_HAVING_VMS_FOR_ACCOUNT, e); + } + } } diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 339ef6f8933..b99663e672e 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -987,6 +987,7 @@ CREATE TABLE `cloud`.`op_host_capacity` ( `host_id` bigint unsigned, `data_center_id` bigint unsigned NOT NULL, `pod_id` bigint unsigned, + `cluster_id` bigint unsigned COMMENT 'foreign key to cluster', `used_capacity` bigint unsigned NOT NULL, `reserved_capacity` bigint unsigned NOT NULL, `total_capacity` bigint unsigned NOT NULL, @@ -996,7 +997,9 @@ CREATE TABLE `cloud`.`op_host_capacity` ( CONSTRAINT `fk_op_host_capacity__pod_id` FOREIGN KEY (`pod_id`) REFERENCES `host_pod_ref` (`id`) ON DELETE CASCADE, INDEX `i_op_host_capacity__pod_id`(`pod_id`), CONSTRAINT `fk_op_host_capacity__data_center_id` FOREIGN KEY (`data_center_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE, - INDEX `i_op_host_capacity__data_center_id`(`data_center_id`) + INDEX `i_op_host_capacity__data_center_id`(`data_center_id`), + CONSTRAINT `fk_op_host_capacity__cluster_id` FOREIGN KEY `fk_op_host_capacity__cluster_id` (`cluster_id`) REFERENCES `cloud`.`cluster`(`id`) ON DELETE CASCADE, + INDEX `i_op_host_capacity__cluster_id`(`cluster_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`alert` ( diff --git a/utils/src/com/cloud/utils/StringUtils.java b/utils/src/com/cloud/utils/StringUtils.java index b055fd6d834..653d33dd8ae 100644 --- a/utils/src/com/cloud/utils/StringUtils.java +++ b/utils/src/com/cloud/utils/StringUtils.java @@ -18,7 +18,9 @@ package com.cloud.utils; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; // StringUtils exists in Apache Commons Lang, but rather than import the entire JAR to our system, for now // just implement the method needed @@ -37,5 +39,47 @@ public class StringUtils { } } return sb.toString(); - } + } + + + /** + * Converts a comma separated list of tags to a List + * @param tags + * @return List of tags + */ + + public static List csvTagsToList(String tags) { + List tagsList = new ArrayList(); + + if (tags != null) { + String[] tokens = tags.split(","); + for (int i = 0; i < tokens.length; i++) { + tagsList.add(tokens[i].trim()); + } + } + + return tagsList; + } + + + /** + * Converts a List of tags to a comma separated list + * @param tags + * @return String containing a comma separated list of tags + */ + + public static String listToCsvTags(List tagsList) { + String tags = ""; + if (tagsList.size() > 0) { + for (int i = 0; i < tagsList.size(); i++) { + tags += tagsList.get(i); + if (i != tagsList.size() - 1) { + tags += ","; + } + } + } + + return tags; + } + }