diff --git a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java index 604720aaa29..5d028d31d5b 100644 --- a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java @@ -22,19 +22,11 @@ import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.Adapter; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { - /** - * @param UserVm vm - * @param ServiceOffering offering - **/ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - /** * Determines which physical hosts are suitable to * allocate the guest virtual machines on @@ -49,31 +41,6 @@ public interface HostAllocator extends Adapter { public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); - /** - * Determines which physical hosts are suitable to allocate the guest - * virtual machines on - * - * Allocators must set any other hosts not considered for allocation in the - * ExcludeList avoid. Thus the avoid set and the list of hosts suitable, - * together must cover the entire host set in the cluster. - * - * @param VirtualMachineProfile - * vmProfile - * @param DeploymentPlan - * plan - * @param GuestType - * type - * @param ExcludeList - * avoid - * @param int returnUpTo (use -1 to return all possible hosts) - * @param boolean considerReservedCapacity (default should be true, set to - * false if host capacity calculation should not look at reserved - * capacity) - * @return List List of hosts that are suitable for VM allocation - **/ - - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity); - /** * Determines which physical hosts are suitable to allocate the guest * virtual machines on diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 8f7e773070f..22d796d4a77 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing; + random, firstfit, userdispersing, firstfitleastconsumed; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 28f13ea86c3..3d75c79066a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -157,6 +157,7 @@ public class ApiConstants { public static final String CUSTOM_ID = "customid"; public static final String CUSTOM_ACTION_ID = "customactionid"; public static final String CUSTOM_JOB_ID = "customjobid"; + public static final String CURRENCY = "currency"; public static final String CURRENT_START_IP = "currentstartip"; public static final String CURRENT_END_IP = "currentendip"; public static final String ENCRYPT = "encrypt"; @@ -542,6 +543,7 @@ public class ApiConstants { public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; public static final String SHOW_REMOVED = "showremoved"; + public static final String SHOW_RESOURCES = "showresources"; public static final String SHOW_RESOURCE_ICON = "showicon"; public static final String SHOW_INACTIVE = "showinactive"; public static final String SHOW_UNIQUE = "showunique"; @@ -607,9 +609,11 @@ public class ApiConstants { public static final String TENANT_NAME = "tenantname"; public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; + public static final String TOTAL_QUOTA = "totalquota"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; public static final String TRUST_STORE_PASSWORD = "truststorepass"; + public static final String UNIT = "unit"; public static final String URL = "url"; public static final String USAGE_INTERFACE = "usageinterface"; public static final String USED = "used"; @@ -1308,6 +1312,8 @@ public class ApiConstants { public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; public static final String QUOTA = "quota"; + public static final String QUOTA_CONSUMED = "quotaconsumed"; + public static final String QUOTA_USAGE = "quotausage"; public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java index abca619f82a..4d6ef741961 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java @@ -78,7 +78,7 @@ public class FindHostsForMigrationCmd extends BaseListCmd { for (Host host : result.first()) { HostForMigrationResponse hostResponse = _responseGenerator.createHostForMigrationResponse(host); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index e202dfad77b..8f5e6c784d6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -252,7 +252,7 @@ public class ListHostsCmd extends BaseListCmd { for (Host host : result.first()) { HostResponse hostResponse = _responseGenerator.createHostResponse(host, getDetails()); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/client/pom.xml b/client/pom.xml index 0d36df7a472..c7fc6321d65 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -387,11 +387,6 @@ cloud-plugin-explicit-dedication ${project.version} - - org.apache.cloudstack - cloud-plugin-host-allocator-random - ${project.version} - org.apache.cloudstack cloud-plugin-outofbandmanagement-driver-ipmitool diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 57dc1b7bf72..e871bd8672f 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -181,15 +181,6 @@ public interface VirtualMachineManager extends Manager { void advanceReboot(String vmUuid, Map params) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; - /** - * Check to see if a virtual machine can be upgraded to the given service offering - * - * @param vm - * @param offering - * @return true if the host can handle the upgrade, false otherwise - */ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - VirtualMachine findById(long vmId); void storageMigration(String vmUuid, Map volumeToPool); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index baf8cdcbbfd..685a5e7e20d 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -4017,19 +4017,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - @Override - public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { - boolean isMachineUpgradable = true; - for (final HostAllocator allocator : hostAllocators) { - isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (!isMachineUpgradable) { - break; - } - } - - return isMachineUpgradable; - } - @Override public void reboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ResourceUnavailableException { try { @@ -4469,11 +4456,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { - throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + - newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); - } - final List currentTags = StringUtils.csvTagsToList(currentDiskOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newDiskOffering.getTags()); if (VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(vmInstance.getDataCenterId())) { diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 090b019334f..5f5b2affee0 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -218,7 +218,7 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); - List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags); List findClustersThatMatchHostTagRule(String computeOfferingTags); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 2d8fcca6cdb..99c9a979c3b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -1520,7 +1520,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao } } - public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + public List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags) { List hostTagVOList = _hostTagsDao.findHostRuleTags(); List result = new ArrayList<>(); for (HostTagVO rule: hostTagVOList) { @@ -1534,7 +1534,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao public List findClustersThatMatchHostTagRule(String computeOfferingTags) { Set result = new HashSet<>(); - List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + List hosts = findHostsWithTagRuleThatMatchComputeOfferingTags(computeOfferingTags); for (HostVO host: hosts) { result.add(host.getClusterId()); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 6ffa7cd5962..019d152ce5c 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -358,7 +358,8 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem IdsPowerStateSelectSearch.entity().getPowerHostId(), IdsPowerStateSelectSearch.entity().getPowerState(), IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(), - IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime()); + IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime(), + IdsPowerStateSelectSearch.entity().getState()); IdsPowerStateSelectSearch.done(); CountByOfferingId = createSearchBuilder(Integer.class); @@ -1105,10 +1106,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) { State instanceState = instance.getState(); + if (instanceState == null) { + logger.warn("VM {} has null instance state during power state sync check, treating as out of sync", instance); + return false; + } if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running) || (powerState == VirtualMachine.PowerState.PowerOn && instanceState == State.Stopped)) { HostVO instanceHost = hostDao.findById(instance.getHostId()); - HostVO powerHost = powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); + HostVO powerHost = instance.getHostId() != null && powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); logger.debug("VM: {} on host: {} and power host : {} is in {} state, but power state is {}", instance, instanceHost, powerHost, instanceState, powerState); return false; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 54baf226ac4..2f104568c14 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,3 +18,9 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; + +-- Entries remaining on `cloud`.`resource_reservation` during the upgrade process are stale, so delete them. +-- This script was added to normalize volume/primary storage reservations that got stuck due to a bug on VM deployment, +-- but it is more interesting to introduce a smarter logic to clean these stale reservations in the future without the need +-- for upgrades (for instance, by having a heartbeat_time column for the reservations and automatically cleaning old entries). +DELETE FROM `cloud`.`resource_reservation`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 5425e7ea181..f1a120f3097 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -282,3 +282,13 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kms_database_kek_objects` ( --- Disable/enable NICs CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); + +--- Quota tariff/usage mapping +CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_tariff_usage` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `tariff_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the tariff of the Quota usage detail calculated, foreign key to quota_tariff table', + `quota_usage_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the aggregation of Quota usage details, foreign key to quota_usage table', + `quota_used` decimal(20,8) NOT NULL COMMENT 'Amount of quota used', + PRIMARY KEY (`id`), + CONSTRAINT `fk_quota_tariff_usage__tariff_id` FOREIGN KEY (`tariff_id`) REFERENCES `cloud_usage`.`quota_tariff` (`id`), + CONSTRAINT `fk_quota_tariff_usage__quota_usage_id` FOREIGN KEY (`quota_usage_id`) REFERENCES `cloud_usage`.`quota_usage` (`id`)); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql new file mode 100644 index 00000000000..7ac001384e8 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud_usage`.`quota_usage_view`; + +DROP VIEW IF EXISTS `cloud_usage`.`quota_usage_view`; +CREATE VIEW `cloud_usage`.`quota_usage_view` AS +SELECT qu.id, + qu.usage_item_id, + qu.zone_id, + qu.account_id, + qu.domain_id, + qu.usage_type, + qu.quota_used, + qu.start_date, + qu.end_date, + cu.usage_id AS resource_id, + cu.network_id as network_id, + cu.offering_id as offering_id +FROM `cloud_usage`.`quota_usage` qu +INNER JOIN `cloud_usage`.`cloud_usage` cu ON (cu.id = qu.usage_item_id); diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 15807eb26d4..3323d5c4d82 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -205,6 +205,12 @@ public class SearchCriteria { } + public void setJoinParametersIfNotNull(String joinName, String conditionName, Object... params) { + if (ArrayUtils.isNotEmpty(params) && (params.length > 1 || params[0] != null)) { + setJoinParameters(joinName, conditionName, params); + } + } + public SearchCriteria getJoin(String joinName) { return _joins.get(joinName).getT(); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index cd87052e878..23020292027 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -35,6 +35,7 @@ import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -783,10 +784,21 @@ public class PresetVariableHelper { value.setId(network.getUuid()); value.setName(network.getName()); value.setState(usageRecord.getState()); - + value.setResourceCounting(getPresetVariableValueNetworkResourceCounting(networkId)); value.setNetworkOffering(getPresetVariableValueNetworkOffering(network.getNetworkOfferingId())); } + protected ResourceCounting getPresetVariableValueNetworkResourceCounting(Long networkId) { + ResourceCounting resourceCounting = new ResourceCounting(); + List vmInstancesVO = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(networkId, VirtualMachine.Type.User); + int runningVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Running)).count(); + int stoppedVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Stopped)).count(); + + resourceCounting.setRunningVms(runningVms); + resourceCounting.setStoppedVms(stoppedVms); + return resourceCounting; + } + protected GenericPresetVariable getPresetVariableValueNetworkOffering(Long networkOfferingId) { NetworkOfferingVO networkOfferingVo = networkOfferingDao.findByIdIncludingRemoved(networkOfferingId); validateIfObjectIsNull(networkOfferingVo, networkOfferingId, "network offering"); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java new file mode 100644 index 00000000000..75049c3486a --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + + +import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class ResourceCounting { + + @PresetVariableDefinition(description = "The number of running user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int runningVms; + @PresetVariableDefinition(description = "The number of stopped user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int stoppedVms; + + public int getRunningVms() { + return runningVms; + } + + public void setRunningVms(int runningVms) { + this.runningVms = runningVms; + } + + public int getStoppedVms() { + return stoppedVms; + } + + public void setStoppedVms(int stoppedVms) { + this.stoppedVms = stoppedVms; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index aff2c040e74..286fe5c60fd 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -84,6 +84,9 @@ public class Value extends GenericPresetVariable { @PresetVariableDefinition(description = "Backup offering of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private BackupOffering backupOffering; + @PresetVariableDefinition(description = "The amount of resources of the usage type.") + private ResourceCounting resourceCounting; + @PresetVariableDefinition(description = "The hypervisor where the resource was deployed. Values can be: XenServer, KVM, VMware, Hyperv, BareMetal, Ovm, Ovm3 and LXC.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.VM_SNAPSHOT, QuotaTypes.SNAPSHOT}) private String hypervisorType; @@ -262,6 +265,14 @@ public class Value extends GenericPresetVariable { this.state = state; } + public ResourceCounting getResourceCounting() { + return resourceCounting; + } + + public void setResourceCounting(ResourceCounting resourceCounting) { + this.resourceCounting = resourceCounting; + } + public GenericPresetVariable getNetworkOffering() { return networkOffering; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java new file mode 100644 index 00000000000..9684ca117b5 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java @@ -0,0 +1,29 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import java.util.List; + +public interface QuotaTariffUsageDao extends GenericDao { + + void persistQuotaTariffUsage(QuotaTariffUsageVO quotaTariffUsage); + + List listQuotaTariffUsages(Long quotaUsageId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java new file mode 100644 index 00000000000..556f552fed6 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Component +public class QuotaTariffUsageDaoImpl extends GenericDaoBase implements QuotaTariffUsageDao { + private SearchBuilder searchQuotaTariffUsages; + + @PostConstruct + public void init() { + searchQuotaTariffUsages = createSearchBuilder(); + searchQuotaTariffUsages.and("quotaUsageId", searchQuotaTariffUsages.entity().getQuotaUsageId(), SearchCriteria.Op.EQ); + searchQuotaTariffUsages.done(); + } + + @Override + public void persistQuotaTariffUsage(final QuotaTariffUsageVO quotaTariffUsage) { + logger.trace("Persisting quota tariff usage [{}].", quotaTariffUsage); + Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(quotaTariffUsage)); + } + + @Override + public List listQuotaTariffUsages(Long quotaUsageId) { + SearchCriteria sc = searchQuotaTariffUsages.create(); + sc.setParameters("quotaUsageId", quotaUsageId); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java new file mode 100644 index 00000000000..126fa11413f --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; + +import java.util.Date; +import java.util.List; + +public interface QuotaUsageJoinDao extends GenericDao { + + List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java new file mode 100644 index 00000000000..b98ea2b3a5d --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.JoinBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.Date; +import java.util.List; + +@Component +public class QuotaUsageJoinDaoImpl extends GenericDaoBase implements QuotaUsageJoinDao { + + private SearchBuilder searchQuotaUsages; + + private SearchBuilder searchQuotaUsagesJoinTariffUsages; + + @Inject + private QuotaTariffUsageDao quotaTariffUsageDao; + + @PostConstruct + public void init() { + searchQuotaUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsages); + searchQuotaUsages.done(); + + SearchBuilder searchQuotaTariffUsages = quotaTariffUsageDao.createSearchBuilder(); + searchQuotaTariffUsages.and("tariffId", searchQuotaTariffUsages.entity().getTariffId(), SearchCriteria.Op.EQ); + searchQuotaUsagesJoinTariffUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsagesJoinTariffUsages); + searchQuotaUsagesJoinTariffUsages.join("searchQuotaTariffUsages", searchQuotaTariffUsages, searchQuotaUsagesJoinTariffUsages.entity().getId(), + searchQuotaTariffUsages.entity().getQuotaUsageId(), JoinBuilder.JoinType.INNER); + searchQuotaUsagesJoinTariffUsages.done(); + } + + private void prepareQuotaUsageSearchBuilder(SearchBuilder searchBuilder) { + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("usageType", searchBuilder.entity().getUsageType(), SearchCriteria.Op.EQ); + searchBuilder.and("resourceId", searchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); + searchBuilder.and("networkId", searchBuilder.entity().getNetworkId(), SearchCriteria.Op.EQ); + searchBuilder.and("offeringId", searchBuilder.entity().getOfferingId(), SearchCriteria.Op.EQ); + searchBuilder.and("startDate", searchBuilder.entity().getStartDate(), SearchCriteria.Op.BETWEEN); + searchBuilder.and("endDate", searchBuilder.entity().getEndDate(), SearchCriteria.Op.BETWEEN); + } + + @Override + public List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId) { + SearchCriteria sc = tariffId == null ? searchQuotaUsages.create() : searchQuotaUsagesJoinTariffUsages.create(); + + sc.setParametersIfNotNull("accountId", accountId); + sc.setParametersIfNotNull("domainId", domainId); + sc.setParametersIfNotNull("usageType", usageType); + sc.setParametersIfNotNull("resourceId", resourceId); + sc.setParametersIfNotNull("networkId", networkId); + sc.setParametersIfNotNull("offeringId", offeringId); + + if (ObjectUtils.allNotNull(startDate, endDate)) { + sc.setParameters("startDate", startDate, endDate); + sc.setParameters("endDate", startDate, endDate); + } + + sc.setJoinParametersIfNotNull("searchQuotaTariffUsages", "tariffId", tariffId); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java new file mode 100644 index 00000000000..4fa9e771713 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java @@ -0,0 +1,86 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +@Entity +@Table(name = "quota_tariff_usage") +public class QuotaTariffUsageVO implements InternalIdentity { + @Id + @Column(name = "id") + private Long id; + + @Column(name = "tariff_id") + private Long tariffId; + + @Column(name = "quota_usage_id") + private Long quotaUsageId; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + public QuotaTariffUsageVO() { + quotaUsed = new BigDecimal(0); + } + + @Override + public long getId() { + return id; + } + + public Long getTariffId() { + return tariffId; + } + + public Long getQuotaUsageId() { + return quotaUsageId; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setId(Long id) { + this.id = id; + } + + public void setTariffId(Long tariffId) { + this.tariffId = tariffId; + } + + public void setQuotaUsageId(Long quotaUsageId) { + this.quotaUsageId = quotaUsageId; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + @Override + public String toString() { + return new ReflectionToStringBuilder(this, ToStringStyle.JSON_STYLE).toString(); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java new file mode 100644 index 00000000000..df9577e23c3 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java @@ -0,0 +1,179 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.math.BigDecimal; +import java.util.Date; + +@Entity +@Table(name = "quota_usage_view") +public class QuotaUsageJoinVO implements InternalIdentity { + + @Id + @Column(name = "id", updatable = false, nullable = false) + private Long id; + + @Column(name = "zone_id") + private Long zoneId = null; + + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "domain_id") + private Long domainId = null; + + @Column(name = "usage_item_id") + private Long usageItemId; + + @Column(name = "usage_type") + private int usageType; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + @Column(name = "start_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date startDate = null; + + @Column(name = "end_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date endDate = null; + + @Column(name = "resource_id") + private Long resourceId = null; + + @Column(name = "network_id") + private Long networkId = null; + + @Column(name = "offering_id") + private Long offeringId = null; + + @Override + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getZoneId() { + return zoneId; + } + + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public Long getDomainId() { + return domainId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public Long getUsageItemId() { + return usageItemId; + } + + public void setUsageItemId(Long usageItemId) { + this.usageItemId = usageItemId; + } + + public int getUsageType() { + return usageType; + } + + public void setUsageType(int usageType) { + this.usageType = usageType; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } + + public Long getNetworkId() { + return networkId; + } + + public void setNetworkId(Long networkId) { + this.networkId = networkId; + } + + public Long getOfferingId() { + return offeringId; + } + + public void setOfferingId(Long offeringId) { + this.offeringId = offeringId; + } + + public QuotaUsageJoinVO () { + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "zoneId", "accountId", "domainId", "usageItemId", "usageType", "quotaUsed", "startDate", + "endDate", "resourceId"); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java new file mode 100644 index 00000000000..92482423100 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java @@ -0,0 +1,62 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.vo; + +import java.util.Date; + +public class QuotaUsageResourceVO { + private String uuid; + private String name; + private Date removed; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public boolean isRemoved() { + return this.removed != null; + } + + public QuotaUsageResourceVO(String uuid, String name, Date removed) { + this.uuid = uuid; + this.name = name; + this.removed = removed; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 304b23b7220..5ca2679c388 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -26,7 +26,9 @@ - + + + diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index fb093a45679..bcdbc3b46ce 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -31,11 +31,13 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -238,6 +240,8 @@ public class PresetVariableHelperTest { value.setVmSnapshotType(VMSnapshot.Type.Disk.toString()); value.setComputingResources(getComputingResourcesForTests()); value.setVolumeType(Volume.Type.DATADISK.toString()); + value.setState(Network.State.Implemented.toString()); + value.setResourceCounting(getResourceCountingForTests()); value.setNetworkOffering(getNetworkOfferingForTests()); value.setVpcOffering(getVpcOfferingForTests()); return value; @@ -276,6 +280,13 @@ public class PresetVariableHelperTest { return configuration; } + private ResourceCounting getResourceCountingForTests() { + ResourceCounting resourceCounting = new ResourceCounting(); + resourceCounting.setRunningVms(1); + resourceCounting.setStoppedVms(1); + return resourceCounting; + } + private List getHostTagsForTests() { return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); } @@ -1343,8 +1354,8 @@ public class PresetVariableHelperTest { Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); Mockito.doReturn(expected.getState()).when(usageVoMock).getState(); + Mockito.doReturn(expected.getResourceCounting()).when(presetVariableHelperSpy).getPresetVariableValueNetworkResourceCounting(Mockito.anyLong()); Mockito.doReturn(expected.getNetworkOffering()).when(presetVariableHelperSpy).getPresetVariableValueNetworkOffering(Mockito.anyLong()); - Mockito.doReturn(UsageTypes.NETWORK).when(usageVoMock).getUsageType(); Value result = new Value(); @@ -1352,7 +1363,27 @@ public class PresetVariableHelperTest { assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getState(), result.getState()); - Assert.assertEquals(expected.getNetworkOffering(), result.getNetworkOffering()); + Assert.assertEquals(expected.getResourceCounting(), result.getResourceCounting()); + } + + @Test + public void getPresetVariableValueNetworkResourceCountingTestSetValueAndReturnObject() { + VMInstanceVO vmInstanceVoMock1 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock1.setState(VirtualMachine.State.Stopped); + + VMInstanceVO vmInstanceVoMock2 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock2.setState(VirtualMachine.State.Running); + + Mockito.doReturn(List.of(vmInstanceVoMock1, vmInstanceVoMock2)).when(vmInstanceDaoMock).listNonRemovedVmsByTypeAndNetwork(Mockito.anyLong(), Mockito.any()); + + mockMethodValidateIfObjectIsNull(); + + ResourceCounting expected = getResourceCountingForTests(); + + ResourceCounting result = presetVariableHelperSpy.getPresetVariableValueNetworkResourceCounting(1L); + + Assert.assertEquals(expected.getRunningVms(), result.getRunningVms()); + Assert.assertEquals(expected.getStoppedVms(), result.getStoppedVms()); } @Test @@ -1362,7 +1393,7 @@ public class PresetVariableHelperTest { presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, null); }); - Mockito.verifyNoInteractions(networkDaoMock); + Mockito.verifyNoInteractions(vpcDaoMock); } @Test diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index d3bd3868ed1..bfe26a9f425 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; @@ -28,24 +27,25 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementItemResponse; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; -import com.cloud.user.Account; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a quota statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - httpMethod = "GET") +@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a Quota statement for the provided Account, Project, or Domain.", + since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaStatementCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Optional, Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, + description = "Name of the Account for which the Quota statement will be generated. Deprecated, please use accountid instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "ID of the Domain for which the Quota statement will be generated. May be used individually or with account.") private Long domainId; @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End of the period of the Quota statement. " + @@ -56,15 +56,25 @@ public class QuotaStatementCmd extends BaseCmd { ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; - @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type") + @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, + description = "Consider only Quota usage records for the specified usage type in the statement.") private Integer usageType; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "ID of the Account for which the Quota statement will be generated. Can not be specified with projectid.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, + description = "ID of the Project for which the Quota statement will be generated. Can not be specified with accountid.", since = "4.23.0") + private Long projectId; + + @Parameter(name = ApiConstants.SHOW_RESOURCES, type = CommandType.BOOLEAN, description = "List the resources of each Quota type in the period.", since = "4.23.0") + private boolean showResources; + @Inject - private QuotaResponseBuilder _responseBuilder; + QuotaResponseBuilder responseBuilder; public Long getAccountId() { return accountId; @@ -99,43 +109,47 @@ public class QuotaStatementCmd extends BaseCmd { } public Date getEndDate() { - return _responseBuilder.startOfNextDay(endDate == null ? new Date() : new Date(endDate.getTime())); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; + } + + public boolean isShowResources() { + return showResources; + } + + public void setShowResources(boolean showResources) { + this.showResources = showResources; + } + + public Long getProjectId() { + return projectId; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; } - Account activeAccountByName = _accountService.getActiveAccountByName(accountName, domainId); - if (activeAccountByName != null) { - return activeAccountByName.getAccountId(); - } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaUsage(this); - - QuotaStatementResponse response = _responseBuilder.createQuotaStatementResponse(quotaUsage); - response.setStartDate(startDate == null ? null : new Date(startDate.getTime())); - response.setEndDate(endDate == null ? null : new Date(endDate.getTime())); - + QuotaStatementResponse response = responseBuilder.createQuotaStatementResponse(this); + response.setStartDate(startDate); + response.setEndDate(endDate); response.setResponseName(getCommandName()); setResponseObject(response); } - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 177fb00d4b5..bde905c487b 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -32,7 +32,6 @@ import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import java.util.Date; import java.util.List; @@ -49,7 +48,7 @@ public interface QuotaResponseBuilder { boolean isUserAllowedToSeeActivationRules(User user); - QuotaStatementResponse createQuotaStatementResponse(List quotaUsage); + QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd); QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); @@ -57,8 +56,6 @@ public interface QuotaResponseBuilder { QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); - List getQuotaUsage(QuotaStatementCmd cmd); - List getQuotaBalance(QuotaBalanceCmd cmd); QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 2d6ec3255f4..c919bb5887c 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -21,13 +21,11 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.math.BigDecimal; -import java.math.RoundingMode; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -36,6 +34,7 @@ import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -43,12 +42,36 @@ import java.util.stream.Collectors; import javax.inject.Inject; import com.cloud.domain.Domain; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.projects.dao.ProjectDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.SnapshotVO; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; @@ -75,6 +98,7 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.ComputingResou import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.ResourceCounting; import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; @@ -94,7 +118,8 @@ import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.cloudstack.quota.vo.QuotaUsageResourceVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections4.CollectionUtils; @@ -106,18 +131,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.domain.DomainVO; -import com.cloud.domain.dao.DomainDao; -import com.cloud.event.ActionEvent; -import com.cloud.event.EventTypes; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.UserDao; -import com.cloud.utils.Pair; - @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { protected Logger logger = LogManager.getLogger(getClass()); @@ -140,8 +153,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject - private ProjectDao projectDao; - @Inject private QuotaAccountDao quotaAccountDao; @Inject private DomainDao domainDao; @@ -159,8 +170,23 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; + @Inject + private IPAddressDao ipAddressDao; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkOfferingDao networkOfferingDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private VMInstanceDao vmInstanceDao; + @Inject + private VMTemplateDao vmTemplateDao; + @Inject + private VolumeDao volumeDao; - private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; + + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class, ResourceCounting.class}; private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); @@ -393,78 +419,212 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } @Override - public QuotaStatementResponse createQuotaStatementResponse(final List quotaUsage) { - if (quotaUsage == null || quotaUsage.isEmpty()) { - throw new InvalidParameterValueException("There is no usage data found for period mentioned."); - } + public QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd) { + Long accountId = getAccountIdForQuotaStatement(cmd); + Long domainId = getDomainIdForQuotaStatement(cmd, accountId); + List quotaUsages = _quotaService.getQuotaUsage(accountId, null, domainId, cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); + + logger.debug("Creating quota statement from [{}] usage records for parameters [{}].", quotaUsages.size(), + ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "accountId", "projectId", "domainId", "startDate", "endDate", "usageType", "showResources")); + createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(quotaUsages, cmd.getUsageType()); + + Map> recordsPerUsageTypes = quotaUsages.stream() + .sorted(Comparator.comparingInt(QuotaUsageJoinVO::getUsageType)) + .collect(Collectors.groupingBy(QuotaUsageJoinVO::getUsageType)); + + List items = new ArrayList<>(); + recordsPerUsageTypes.forEach((key, value) -> items.add(createStatementItem(key, value, cmd.isShowResources()))); QuotaStatementResponse statement = new QuotaStatementResponse(); - - HashMap quotaTariffMap = new HashMap(); - Collection result = QuotaTypes.listQuotaTypes().values(); - - for (QuotaTypes quotaTariff : result) { - quotaTariffMap.put(quotaTariff.getQuotaType(), quotaTariff); - // add dummy record for each usage type - QuotaUsageVO dummy = new QuotaUsageVO(quotaUsage.get(0)); - dummy.setUsageType(quotaTariff.getQuotaType()); - dummy.setQuotaUsed(new BigDecimal(0)); - quotaUsage.add(dummy); - } - - if (logger.isDebugEnabled()) { - logger.debug( - "createQuotaStatementResponse Type=" + quotaUsage.get(0).getUsageType() + " usage=" + quotaUsage.get(0).getQuotaUsed().setScale(2, RoundingMode.HALF_EVEN) - + " rec.id=" + quotaUsage.get(0).getUsageItemId() + " SD=" + quotaUsage.get(0).getStartDate() + " ED=" + quotaUsage.get(0).getEndDate()); - } - - Collections.sort(quotaUsage, new Comparator() { - @Override - public int compare(QuotaUsageVO o1, QuotaUsageVO o2) { - if (o1.getUsageType() == o2.getUsageType()) { - return 0; - } - return o1.getUsageType() < o2.getUsageType() ? -1 : 1; - } - }); - - List items = new ArrayList(); - QuotaStatementItemResponse lineitem; - int type = -1; - BigDecimal usage = new BigDecimal(0); - BigDecimal totalUsage = new BigDecimal(0); - quotaUsage.add(new QuotaUsageVO());// boundary - QuotaUsageVO prev = quotaUsage.get(0); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaStatementResponse record count=" + quotaUsage.size()); - } - for (final QuotaUsageVO quotaRecord : quotaUsage) { - if (type != quotaRecord.getUsageType()) { - if (type != -1) { - lineitem = new QuotaStatementItemResponse(type); - lineitem.setQuotaUsed(usage); - lineitem.setAccountId(prev.getAccountId()); - lineitem.setDomainId(prev.getDomainId()); - lineitem.setUsageUnit(quotaTariffMap.get(type).getQuotaUnit()); - lineitem.setUsageName(quotaTariffMap.get(type).getQuotaName()); - lineitem.setObjectName("quotausage"); - items.add(lineitem); - totalUsage = totalUsage.add(usage); - usage = new BigDecimal(0); - } - type = quotaRecord.getUsageType(); - } - prev = quotaRecord; - usage = usage.add(quotaRecord.getQuotaUsed()); - } - statement.setLineItem(items); - statement.setTotalQuota(totalUsage); + statement.setTotalQuota(items.stream().map(QuotaStatementItemResponse::getQuotaUsed).reduce(BigDecimal.ZERO, BigDecimal::add)); statement.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); statement.setObjectName("statement"); + + if (accountId != null) { + Account account = _accountDao.findByIdIncludingRemoved(accountId); + statement.setAccountId(account.getUuid()); + statement.setAccountName(account.getAccountName()); + domainId = account.getDomainId(); + } + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + statement.setDomainId(domain.getUuid()); + } + return statement; } + protected Long getAccountIdForQuotaStatement(QuotaStatementCmd cmd) { + if (Account.Type.NORMAL.equals(CallContext.current().getCallingAccount().getType())) { + logger.debug("Limiting the Quota statement for the calling Account, as they are a User Account."); + return CallContext.current().getCallingAccountId(); + } + + long accountId = cmd.getEntityOwnerId(); + if (accountId != -1) { + return accountId; + } + + if (cmd.getDomainId() == null) { + logger.debug("Limiting the Quota statement for the calling Account, as 'domainid' was not informed."); + return CallContext.current().getCallingAccountId(); + } + + logger.debug("Allowing admin/domain admin to generate the Quota statement for the provided Domain."); + return null; + } + + protected Long getDomainIdForQuotaStatement(QuotaStatementCmd cmd, Long accountId) { + if (accountId != null) { + logger.debug("Quota statement is already limited to Account [{}].", accountId); + Account account = _accountDao.findByIdIncludingRemoved(accountId); + return account.getDomainId(); + } + + Long domainId = cmd.getDomainId(); + if (domainId != null) { + return domainId; + } + + logger.debug("Limiting the Quota statement for the calling Account's Domain."); + return CallContext.current().getCallingAccount().getDomainId(); + } + + protected void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(List quotaUsages, Integer usageType) { + if (usageType != null) { + logger.debug("As the usage type [{}] was informed as parameter of the API quotaStatement, we will not create dummy records.", usageType); + return; + + } + + for (Integer quotaType : QuotaTypes.listQuotaTypes().keySet()) { + QuotaUsageJoinVO dummy = new QuotaUsageJoinVO(); + dummy.setUsageType(quotaType); + dummy.setQuotaUsed(BigDecimal.ZERO); + quotaUsages.add(dummy); + } + } + + protected QuotaStatementItemResponse createStatementItem(int usageType, List usageRecords, boolean showResources) { + QuotaUsageJoinVO firstRecord = usageRecords.get(0); + int type = firstRecord.getUsageType(); + + QuotaTypes quotaType = QuotaTypes.listQuotaTypes().get(type); + + QuotaStatementItemResponse item = new QuotaStatementItemResponse(type); + item.setQuotaUsed(usageRecords.stream().map(QuotaUsageJoinVO::getQuotaUsed).filter(Objects::nonNull).reduce(BigDecimal.ZERO, BigDecimal::add)); + item.setUsageUnit(quotaType.getQuotaUnit()); + item.setUsageName(quotaType.getQuotaName()); + + setStatementItemResources(item, usageType, usageRecords, showResources); + return item; + } + + protected void setStatementItemResources(QuotaStatementItemResponse statementItem, int usageType, List quotaUsageRecords, boolean showResources) { + if (!showResources) { + return; + } + + List itemDetails = new ArrayList<>(); + + Map quotaUsagesValuesAggregatedById = quotaUsageRecords + .stream() + .filter(quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType) != null) + .collect(Collectors.groupingBy( + quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType), + Collectors.reducing(new BigDecimal(0), QuotaUsageJoinVO::getQuotaUsed, BigDecimal::add) + )); + + for (Map.Entry entry : quotaUsagesValuesAggregatedById.entrySet()) { + QuotaStatementItemResourceResponse detail = new QuotaStatementItemResourceResponse(); + + detail.setQuotaUsed(entry.getValue()); + + QuotaUsageResourceVO resource = getResourceFromIdAndType(entry.getKey(), usageType); + if (resource != null) { + detail.setResourceId(resource.getUuid()); + detail.setDisplayName(resource.getName()); + detail.setRemoved(resource.isRemoved()); + } else { + detail.setDisplayName(""); + + } + itemDetails.add(detail); + } + statementItem.setResources(itemDetails); + } + + protected Long getResourceIdByUsageType(QuotaUsageJoinVO quotaUsageJoinVo, int usageType) { + switch (usageType) { + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + return quotaUsageJoinVo.getNetworkId(); + case QuotaTypes.NETWORK_OFFERING: + return quotaUsageJoinVo.getOfferingId(); + default: + return quotaUsageJoinVo.getResourceId(); + } + } + + protected QuotaUsageResourceVO getResourceFromIdAndType(long resourceId, int usageType) { + switch (usageType) { + case QuotaTypes.ALLOCATED_VM: + case QuotaTypes.RUNNING_VM: + VMInstanceVO vmInstance = vmInstanceDao.findByIdIncludingRemoved(resourceId); + if (vmInstance != null) { + return new QuotaUsageResourceVO(vmInstance.getUuid(), vmInstance.getHostName(), vmInstance.getRemoved()); + } + break; + case QuotaTypes.VOLUME: + case QuotaTypes.VOLUME_SECONDARY: + case QuotaTypes.VM_DISK_BYTES_READ: + case QuotaTypes.VM_DISK_BYTES_WRITE: + case QuotaTypes.VM_DISK_IO_READ: + case QuotaTypes.VM_DISK_IO_WRITE: + VolumeVO volume = volumeDao.findByIdIncludingRemoved(resourceId); + if (volume != null) { + return new QuotaUsageResourceVO(volume.getUuid(), volume.getName(), volume.getRemoved()); + } + break; + case QuotaTypes.VM_SNAPSHOT_ON_PRIMARY: + case QuotaTypes.VM_SNAPSHOT: + case QuotaTypes.SNAPSHOT: + SnapshotVO snapshot = snapshotDao.findByIdIncludingRemoved(resourceId); + if (snapshot != null) { + return new QuotaUsageResourceVO(snapshot.getUuid(), snapshot.getName(), snapshot.getRemoved()); + } + break; + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + NetworkVO network = networkDao.findByIdIncludingRemoved(resourceId); + if (network != null) { + return new QuotaUsageResourceVO(network.getUuid(), network.getName(), network.getRemoved()); + } + break; + case QuotaTypes.TEMPLATE: + case QuotaTypes.ISO: + VMTemplateVO vmTemplate = vmTemplateDao.findByIdIncludingRemoved(resourceId); + if (vmTemplate != null) { + return new QuotaUsageResourceVO(vmTemplate.getUuid(), vmTemplate.getName(), vmTemplate.getRemoved()); + } + break; + case QuotaTypes.NETWORK_OFFERING: + NetworkOfferingVO networkOffering = networkOfferingDao.findByIdIncludingRemoved(resourceId); + if (networkOffering != null) { + return new QuotaUsageResourceVO(networkOffering.getUuid(), networkOffering.getName(), networkOffering.getRemoved()); + } + break; + case QuotaTypes.IP_ADDRESS: + IPAddressVO ipAddress = ipAddressDao.findByIdIncludingRemoved(resourceId); + if (ipAddress != null) { + return new QuotaUsageResourceVO(ipAddress.getUuid(), ipAddress.getName(), ipAddress.getRemoved()); + } + break; + } + return null; + } + @Override public Pair, Integer> listQuotaTariffPlans(final QuotaTariffListCmd cmd) { Date startDate = cmd.getEffectiveDate(); @@ -518,14 +678,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } protected void warnQuotaTariffUpdateDeprecatedFields(QuotaTariffUpdateCmd cmd) { - String warnMessage = "The parameter 's%s' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; + String warnMessage = "The parameter '{}' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; if (cmd.getStartDate() != null) { - logger.warn(String.format(warnMessage,"startdate")); + logger.warn(warnMessage, "startdate"); } if (cmd.getUsageType() != null) { - logger.warn(String.format(warnMessage,"usagetype")); + logger.warn(warnMessage, "usagetype"); } } @@ -712,11 +872,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { return resp; } - @Override - public List getQuotaUsage(QuotaStatementCmd cmd) { - return _quotaService.getQuotaUsage(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); - } - @Override public List getQuotaBalance(QuotaBalanceCmd cmd) { return _quotaService.findQuotaBalanceVO(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getStartDate(), cmd.getEndDate()); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java new file mode 100644 index 00000000000..3e052f73339 --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java @@ -0,0 +1,61 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import java.math.BigDecimal; + +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; + +public class QuotaStatementItemResourceResponse extends BaseResponse { + + @SerializedName(ApiConstants.QUOTA_CONSUMED) + @Param(description = "Quota consumed.") + private BigDecimal quotaUsed; + + @SerializedName(ApiConstants.RESOURCE_ID) + @Param(description = "Resources's ID.") + private String resourceId; + + @SerializedName(ApiConstants.DISPLAY_NAME) + @Param(description = "Resource's display name.") + private String displayName; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "Indicates whether the resource is removed or active.") + private boolean removed; + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setRemoved(boolean removed) { + this.removed = removed; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java index c370d82b3cc..0747c5a9172 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java @@ -17,72 +17,41 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; +import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; public class QuotaStatementItemResponse extends BaseResponse { - @SerializedName("type") - @Param(description = "Usage type") + @SerializedName(ApiConstants.TYPE) + @Param(description = "Usage type.") private int usageType; - @SerializedName("accountid") - @Param(description = "Account id") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain id") - private Long domainId; - - @SerializedName("name") - @Param(description = "Usage type name") + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Usage type.") private String usageName; - @SerializedName("unit") - @Param(description = "Usage unit") + @SerializedName(ApiConstants.UNIT) + @Param(description = "Unit of the Usage type.") private String usageUnit; - @SerializedName("quota") - @Param(description = "Quota consumed") + @SerializedName(ApiConstants.QUOTA) + @Param(description = "Quota consumed.") private BigDecimal quotaUsed; + @SerializedName(ApiConstants.RESOURCES) + @Param(description = "Item's resources.") + private List resources; + public QuotaStatementItemResponse(final int usageType) { this.usageType = usageType; } - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } - - public void setAccountName(String accountName) { - this.accountName = accountName; - } - - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - public String getUsageName() { return usageName; } @@ -112,7 +81,15 @@ public class QuotaStatementItemResponse extends BaseResponse { } public void setQuotaUsed(BigDecimal quotaUsed) { - this.quotaUsed = quotaUsed.setScale(2, RoundingMode.HALF_EVEN); + this.quotaUsed = quotaUsed; + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + this.resources = resources; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java index 0a7ba4dbeb9..81cb1011182 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java @@ -18,56 +18,56 @@ package org.apache.cloudstack.api.response; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import java.util.List; public class QuotaStatementResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; + @SerializedName(ApiConstants.ACCOUNT_ID) + @Param(description = "ID of the Account.") + private String accountId; - @SerializedName("account") - @Param(description = "Account name") + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "Name of the Account.") private String accountName; - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "ID of the Domain.") + private String domainId; - @SerializedName("quotausage") - @Param(description = "List of quota usage under various types", responseObject = QuotaStatementItemResponse.class) + @SerializedName(ApiConstants.QUOTA_USAGE) + @Param(description = "List of Quota usage under various types.", responseObject = QuotaStatementItemResponse.class) private List lineItem; - @SerializedName("totalquota") - @Param(description = "Total quota used during this period") + @SerializedName(ApiConstants.TOTAL_QUOTA) + @Param(description = "Total Quota consumed during this period.") private BigDecimal totalQuota; - @SerializedName("startdate") - @Param(description = "Start date") + @SerializedName(ApiConstants.START_DATE) + @Param(description = "Start date of the Quota statement.") private Date startDate = null; - @SerializedName("enddate") - @Param(description = "End date") + @SerializedName(ApiConstants.END_DATE) + @Param(description = "End date of the Quota statement.") private Date endDate = null; - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Currency of the Quota statement.") private String currency; public QuotaStatementResponse() { super(); } - public Long getAccountId() { + public String getAccountId() { return accountId; } - public void setAccountId(Long accountId) { + public void setAccountId(String accountId) { this.accountId = accountId; } @@ -79,45 +79,36 @@ public class QuotaStatementResponse extends BaseResponse { this.accountName = accountName; } - public Long getDomainId() { + public String getDomainId() { return domainId; } - public void setDomainId(Long domainId) { + public void setDomainId(String domainId) { this.domainId = domainId; } - public List getLineItem() { - return lineItem; - } - public void setLineItem(List lineItem) { this.lineItem = lineItem; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - - public BigDecimal getTotalQuota() { - return totalQuota; + this.endDate = endDate; } public void setTotalQuota(BigDecimal totalQuota) { - this.totalQuota = totalQuota.setScale(2, RoundingMode.HALF_EVEN); + this.totalQuota = totalQuota; } public String getCurrency() { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index a421d0f066a..78acfc11682 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -21,14 +21,14 @@ import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import com.cloud.user.AccountVO; import com.cloud.utils.component.PluggableService; public interface QuotaService extends PluggableService { - List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); + List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index 2d7d623d1d9..a0ba2fbc751 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -53,10 +53,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; -import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.commons.lang3.ObjectUtils; import org.springframework.stereotype.Component; @@ -80,7 +80,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private QuotaAccountDao _quotaAcc; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageJoinDao quotaUsageJoinDao; @Inject private DomainDao _domainDao; @Inject @@ -213,27 +213,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi } @Override - public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { - // if accountId is not specified, use accountName and domainId - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } - } - + public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { if (startDate.after(endDate)) { throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); } @@ -241,7 +221,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi logger.debug("Getting quota records of type [{}] for account [{}] in domain [{}], between [{}] and [{}].", usageType, accountId, domainId, startDate, endDate); - return _quotaUsageDao.findQuotaUsage(accountId, domainId, usageType, startDate, endDate); + return quotaUsageJoinDao.findQuotaUsage(accountId, domainId, usageType, null, null, null, startDate, endDate, null); } @Override diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java index d6f9f747fa8..e67638db632 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java @@ -16,38 +16,29 @@ // under the License. package org.apache.cloudstack.api.command; -import junit.framework.TestCase; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - @RunWith(MockitoJUnitRunner.class) -public class QuotaStatementCmdTest extends TestCase { +public class QuotaStatementCmdTest { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder responseBuilderMock; @Test - public void testQuotaStatementCmd() throws NoSuchFieldException, IllegalAccessException { + public void executeTestVerifyCalls() { QuotaStatementCmd cmd = new QuotaStatementCmd(); cmd.setAccountName("admin"); + cmd.responseBuilder = responseBuilderMock; - Field rbField = QuotaStatementCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + Mockito.doReturn(new QuotaStatementResponse()).when(responseBuilderMock).createQuotaStatementResponse(Mockito.any()); - List quotaUsageVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaUsage(Mockito.eq(cmd))).thenReturn(quotaUsageVOList); - Mockito.when(responseBuilder.createQuotaStatementResponse(Mockito.eq(quotaUsageVOList))).thenReturn(new QuotaStatementResponse()); cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).getQuotaUsage(Mockito.eq(cmd)); + + Mockito.verify(responseBuilderMock).createQuotaStatementResponse(cmd); } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index ea88a106b84..81b4992082d 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; @@ -42,6 +44,7 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; @@ -67,6 +70,7 @@ import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.commons.lang3.time.DateUtils; @@ -914,4 +918,199 @@ public class QuotaResponseBuilderImplTest extends TestCase { Assert.assertTrue(formattedVariables.containsValue("accountname")); Assert.assertTrue(formattedVariables.containsValue("zonename")); } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeDifferentFromNullDoNothing() { + List listUsage = new ArrayList<>(); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, 1); + + Assert.assertTrue(listUsage.isEmpty()); + } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeIsNullAddDummyForAllQuotaTypes() { + List listUsage = new ArrayList<>(); + listUsage.add(new QuotaUsageJoinVO()); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, null); + + Assert.assertEquals(QuotaTypes.listQuotaTypes().size() + 1, listUsage.size()); + + QuotaTypes.listQuotaTypes().entrySet().forEach(entry -> { + Assert.assertTrue(listUsage.stream().anyMatch(usage -> usage.getUsageType() == entry.getKey() && usage.getQuotaUsed().equals(BigDecimal.ZERO))); + }); + } + + private List getQuotaUsagesForTest() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + List quotaUsages = new ArrayList<>(); + + QuotaUsageJoinVO quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(1l); + quotaUsage.setDomainId(2l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(10)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-01")); + quotaUsage.setEndDate(sdf.parse("2022-01-02")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(4l); + quotaUsage.setDomainId(5l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(null); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-03")); + quotaUsage.setEndDate(sdf.parse("2022-01-04")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(6l); + quotaUsage.setDomainId(7l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(5)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-05")); + quotaUsage.setEndDate(sdf.parse("2022-01-06")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + return quotaUsages; + } + + @Test + public void createStatementItemTestReturnItem() { + List quotaUsages = getQuotaUsagesForTest(); + Mockito.doNothing().when(quotaResponseBuilderSpy).setStatementItemResources(Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyBoolean()); + + QuotaStatementItemResponse result = quotaResponseBuilderSpy.createStatementItem(0, quotaUsages, false); + + QuotaUsageJoinVO expected = quotaUsages.get(0); + QuotaTypes quotaTypeExpected = QuotaTypes.listQuotaTypes().get(expected.getUsageType()); + Assert.assertEquals(BigDecimal.valueOf(15), result.getQuotaUsed()); + Assert.assertEquals(quotaTypeExpected.getQuotaUnit(), result.getUsageUnit()); + Assert.assertEquals(quotaTypeExpected.getQuotaName(), result.getUsageName()); + } + + @Test + public void setStatementItemResourcesTestDoNotShowResourcesDoNothing() { + QuotaStatementItemResponse item = new QuotaStatementItemResponse(1); + + quotaResponseBuilderSpy.setStatementItemResources(item, 0, getQuotaUsagesForTest(), false); + + Assert.assertNull(item.getResources()); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountForNormalUser() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsEntityOwnerIdWhenProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(42L).when(cmd).getEntityOwnerId(); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(42L), result); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountWhenCallerIsAdminAndDomainIsNotProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(null).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsAdminAndDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(10L).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertNull(result); + } + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsAccountDomainIdWhenAccountIdIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + AccountVO account = Mockito.mock(AccountVO.class); + + Mockito.doReturn(account).when(accountDaoMock).findByIdIncludingRemoved(55L); + Mockito.doReturn(77L).when(account).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, 55L); + + Assert.assertEquals(Long.valueOf(77L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsProvidedDomainIdWhenAccountIdIsNull() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(99L).when(cmd).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(Long.valueOf(99L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestFallsBackToCallingAccountDomainIdWhenNeitherAccountNorDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + Account account = Mockito.mock(Account.class); + + Mockito.doReturn(null).when(cmd).getDomainId(); + Mockito.doReturn(123L).when(account).getDomainId(); + Mockito.doReturn(account).when(callContextMock).getCallingAccount(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(123L, result.longValue()); + } + } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 259264f3b0e..a0fe63de851 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.joda.time.DateTime; @@ -63,6 +64,8 @@ public class QuotaServiceImplTest extends TestCase { @Mock QuotaBalanceDao quotaBalanceDao; @Mock + QuotaUsageJoinDao quotaUsageJoinDaoMock; + @Mock QuotaResponseBuilder respBldr; @Mock private AccountVO accountVoMock; @@ -85,9 +88,9 @@ public class QuotaServiceImplTest extends TestCase { quotaAccountDaoField.setAccessible(true); quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); - Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); + Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("quotaUsageJoinDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageJoinDaoMock); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); @@ -142,7 +145,8 @@ public class QuotaServiceImplTest extends TestCase { final Date endDate = new Date(); quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); - Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); + Mockito.verify(quotaUsageJoinDaoMock, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); } @Test diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml deleted file mode 100644 index 061caf1573f..00000000000 --- a/plugins/host-allocators/random/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - cloud-plugin-host-allocator-random - Apache CloudStack Plugin - Host Allocator Random - - org.apache.cloudstack - cloudstack-plugins - 4.23.0.0-SNAPSHOT - ../../pom.xml - - diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java deleted file mode 100644 index 42129944a19..00000000000 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; - -@Component -public class RandomAllocator extends AdapterBase implements HostAllocator { - @Inject - private HostDao _hostDao; - @Inject - private ResourceManager _resourceMgr; - @Inject - private ClusterDao clusterDao; - @Inject - private ClusterDetailsDao clusterDetailsDao; - @Inject - private CapacityManager capacityManager; - - protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - List taggedHosts = new ArrayList<>(); - if (offeringHostTag != null) { - taggedHosts.addAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag)); - } - if (templateTag != null) { - List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - if (taggedHosts.isEmpty()) { - taggedHosts = templateTaggedHosts; - } else { - taggedHosts.retainAll(templateTaggedHosts); - } - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", - taggedHosts.size(), - (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::toString).toArray(), ","))), - type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); - } - return taggedHosts; - } - private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - List hostsCopy = null; - List suitableHosts = new ArrayList<>(); - - if (type == Host.Type.Storage) { - return suitableHosts; - } - String offeringHostTag = offering.getHostTag(); - - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - String templateTag = template.getTemplateTag(); - String hostTag = null; - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostTag = ObjectUtils.allNotNull(offeringHostTag, templateTag) ? - String.format("%s, %s", offeringHostTag, templateTag) : - ObjectUtils.firstNonNull(offeringHostTag, templateTag); - logger.debug("Looking for hosts in dc [{}], pod [{}], cluster [{}] and complying with host tag(s): [{}]", dcId, podId, clusterId, hostTag); - } else { - logger.debug("Looking for hosts in dc: {} pod: {} cluster: {}", dcId , podId, clusterId); - } - if (hosts != null) { - // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList<>(hosts); - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); - } else { - hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); - } - } else { - // list all computing hosts, regardless of whether they support routing...it's random after all - if (offeringHostTag != null) { - hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); - } else { - hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); - } - } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); - - if (hostsCopy.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); - return null; - } - - logger.debug("Random Allocator found {} hosts", hostsCopy.size()); - if (hostsCopy.isEmpty()) { - return suitableHosts; - } - - Collections.shuffle(hostsCopy); - for (Host host : hostsCopy) { - if (suitableHosts.size() == returnUpTo) { - break; - } - if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Host %s is in avoid set, skipping this and trying other available hosts", host)); - } - continue; - } - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Not using host %s; host has cpu capability? %s, host has capacity? %s", host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second())); - } - continue; - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found a suitable host, adding to list: %s", host)); - } - suitableHosts.add(host); - } - if (logger.isDebugEnabled()) { - logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - if (CollectionUtils.isEmpty(hosts)) { - if (logger.isDebugEnabled()) { - logger.debug("Random Allocator found 0 hosts as given host list is empty"); - } - return new ArrayList<>(); - } - return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, - Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity); - } - - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } -} diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties deleted file mode 100644 index dcfe8d3537f..00000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -name=host-allocator-random -parent=allocator diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml deleted file mode 100644 index d84eaafaa5a..00000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java deleted file mode 100644 index 538d7157184..00000000000 --- a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.collections.CollectionUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; - -@RunWith(MockitoJUnitRunner.class) -public class RandomAllocatorTest { - - @Mock - HostDao hostDao; - @InjectMocks - RandomAllocator randomAllocator; - - @Test - public void testListHostsByTags() { - Host.Type type = Host.Type.Routing; - Long id = 1L; - String templateTag = "tag1"; - String offeringTag = "tag2"; - HostVO host1 = Mockito.mock(HostVO.class); - HostVO host2 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, offeringTag)).thenReturn(List.of(host1, host2)); - - // No template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - List result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Different template tagged host - HostVO host3 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Matching template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - - // No template tag - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, null); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(2, result.size()); - - // No offering tag - result = randomAllocator.listHostsByTags(type, id, id, id, null, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - } -} diff --git a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java index 5a027425776..140302590ba 100644 --- a/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java +++ b/plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/qemu/QemuImgTest.java @@ -57,7 +57,7 @@ public class QemuImgTest { Connect conn = new Connect("qemu:///system", false); conn.getVersion(); libVirtAvailable = true; - } catch (LibvirtException ignored) {} + } catch (LibvirtException | UnsatisfiedLinkError | ExceptionInInitializerError ignored) {} Assume.assumeTrue("libvirt not available", libVirtAvailable); } diff --git a/plugins/pom.xml b/plugins/pom.xml index b368c26ad67..a5702d62890 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -84,8 +84,6 @@ ha-planners/skip-heurestics - host-allocators/random - hypervisors/baremetal hypervisors/external hypervisors/hyperv diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 3f06bee8ac8..83dacf74e8d 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -153,6 +153,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver // CAN_CREATE_VOLUME_FROM_SNAPSHOT see note from CAN_CREATE_VOLUME_FROM_VOLUME mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString()); mapCapabilities.put(DataStoreCapabilities.CAN_REVERT_VOLUME_TO_SNAPSHOT.toString(), Boolean.TRUE.toString()); + mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_TEMPLATE_FROM_SNAPSHOT.toString(), + Boolean.toString(system_snapshot)); return mapCapabilities; } @@ -720,6 +722,13 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + private static boolean canCopySnapshotToVolumeCond(DataObject srcData, DataObject dstData) { + return srcData.getType() == DataObjectType.SNAPSHOT && dstData.getType() == DataObjectType.VOLUME + && srcData.getDataStore().getRole() == DataStoreRole.Primary + && dstData.getDataStore().getRole() == DataStoreRole.Primary + && srcData.getDataStore().getId() == dstData.getDataStore().getId(); + } + private static boolean canCopySnapshotCond(DataObject srcData, DataObject dstData) { return srcData.getType() == DataObjectType.SNAPSHOT && dstData.getType() == DataObjectType.SNAPSHOT && (dstData.getDataStore().getRole() == DataStoreRole.Image @@ -747,7 +756,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { logger.debug("LinstorPrimaryDataStoreDriverImpl.canCopy: " + srcData.getType() + " -> " + dstData.getType()); - if (canCopySnapshotCond(srcData, dstData)) { + if (canCopySnapshotToVolumeCond(srcData, dstData)) { + StoragePoolVO storagePool = _storagePoolDao.findById(srcData.getDataStore().getId()); + return storagePool.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME); + } else if (canCopySnapshotCond(srcData, dstData)) { SnapshotInfo sinfo = (SnapshotInfo) srcData; VolumeInfo volume = sinfo.getBaseVolume(); StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId()); @@ -766,6 +778,18 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return false; } + private CopyCommandResult copySnapshotToVolume(SnapshotInfo snapshotInfo, VolumeInfo volumeInfo) { + StoragePoolVO storagePoolVO = _storagePoolDao.findById(snapshotInfo.getDataStore().getId()); + String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(); + createResourceFromSnapshot(snapshotInfo.getId(), rscName, storagePoolVO); + + VolumeObjectTO volumeTO = (VolumeObjectTO) volumeInfo.getTO(); + volumeTO.setPath(volumeInfo.getUuid()); + volumeTO.setSize(volumeInfo.getSize()); + + return new CopyCommandResult(null, new CopyCmdAnswer(volumeTO)); + } + @Override public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCallback callback) { @@ -773,7 +797,9 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver + srcData.getType() + " -> " + dstData.getType()); final CopyCommandResult res; - if (canCopySnapshotCond(srcData, dstData)) { + if (canCopySnapshotToVolumeCond(srcData, dstData)) { + res = copySnapshotToVolume((SnapshotInfo) srcData, (VolumeInfo) dstData); + } else if (canCopySnapshotCond(srcData, dstData)) { String errMsg = null; Answer answer = copySnapshot(srcData, dstData); if (answer != null && !answer.getResult()) { diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java index 4653cfa358b..23266e17f4f 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -25,13 +25,23 @@ import com.linbit.linstor.api.model.ResourceGroup; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.storage.DataStoreRole; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.util.LinstorUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import static org.mockito.Mockito.mock; @@ -42,6 +52,9 @@ public class LinstorPrimaryDataStoreDriverImplTest { private DevelopersApi api; + @Mock + private PrimaryDataStoreDao _storagePoolDao; + @InjectMocks private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver; @@ -85,4 +98,100 @@ public class LinstorPrimaryDataStoreDriverImplTest { layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); } + + @Test + public void testGetCapabilitiesIncludesCreateTemplateFromSnapshotMatchesSystemSnapshot() { + Map caps = linstorPrimaryDataStoreDriver.getCapabilities(); + + Assert.assertEquals( + "CAN_CREATE_TEMPLATE_FROM_SNAPSHOT should match STORAGE_SYSTEM_SNAPSHOT", + caps.get(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString()), + caps.get(DataStoreCapabilities.CAN_CREATE_TEMPLATE_FROM_SNAPSHOT.toString())); + } + + @Test + public void testCanCopySnapshotToVolumeOnSamePrimary() { + DataStore primaryStore = mock(DataStore.class); + when(primaryStore.getRole()).thenReturn(DataStoreRole.Primary); + when(primaryStore.getId()).thenReturn(1L); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(primaryStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(primaryStore); + + StoragePoolVO pool = mock(StoragePoolVO.class); + when(pool.getStorageProviderName()).thenReturn(LinstorUtil.PROVIDER_NAME); + when(_storagePoolDao.findById(1L)).thenReturn(pool); + + Assert.assertTrue("canCopy should return true for SNAPSHOT -> VOLUME on same Linstor primary", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } + + @Test + public void testCanCopySnapshotToVolumeRejectsNonLinstor() { + DataStore primaryStore = mock(DataStore.class); + when(primaryStore.getRole()).thenReturn(DataStoreRole.Primary); + when(primaryStore.getId()).thenReturn(1L); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(primaryStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(primaryStore); + + StoragePoolVO pool = mock(StoragePoolVO.class); + when(pool.getStorageProviderName()).thenReturn("SomeOtherProvider"); + when(_storagePoolDao.findById(1L)).thenReturn(pool); + + Assert.assertFalse("canCopy should return false for non-Linstor storage", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } + + @Test + public void testCanCopySnapshotToVolumeRejectsCrossPrimary() { + DataStore srcStore = mock(DataStore.class); + when(srcStore.getRole()).thenReturn(DataStoreRole.Primary); + when(srcStore.getId()).thenReturn(1L); + + DataStore destStore = mock(DataStore.class); + when(destStore.getRole()).thenReturn(DataStoreRole.Primary); + when(destStore.getId()).thenReturn(2L); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(srcStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(destStore); + + Assert.assertFalse("canCopy should return false for SNAPSHOT -> VOLUME across different primary stores", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } + + @Test + public void testCanCopySnapshotToVolumeRejectsImageDest() { + DataStore primaryStore = mock(DataStore.class); + when(primaryStore.getRole()).thenReturn(DataStoreRole.Primary); + + DataStore imageStore = mock(DataStore.class); + when(imageStore.getRole()).thenReturn(DataStoreRole.Image); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(primaryStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(imageStore); + + Assert.assertFalse("canCopy should return false when destination is Image store", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java new file mode 100644 index 00000000000..58fcc62cdc3 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.capacity.CapacityManager; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import org.apache.commons.collections.CollectionUtils; + +import javax.inject.Inject; +import java.util.List; + +public abstract class BaseAllocator extends AdapterBase implements HostAllocator { + + @Inject + protected HostDao hostDao; + + @Inject + protected CapacityManager capacityManager; + + protected void retainHostsMatchingServiceOfferingAndTemplateTags(List availableHosts, Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { + logger.debug("Hosts {} will be checked for template and host tags compatibility.", availableHosts); + + if (offeringHostTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", offeringHostTag); + List hostsWithHostTag = hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag); + logger.debug("Retaining hosts {} because they match the offering host tag {}.", hostsWithHostTag, offeringHostTag); + availableHosts.retainAll(hostsWithHostTag); + } + + if (templateTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Template.", templateTag); + List hostsWithTemplateTag = hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); + logger.debug("Retaining hosts {} because they match the template tag {}.", hostsWithTemplateTag, templateTag); + availableHosts.retainAll(hostsWithTemplateTag); + } + + logger.debug("Remaining hosts after template tag and host tags validations are {}.", availableHosts); + } + + protected void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { + List hostsWithTagRules = hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTagOnOffering); + + if (CollectionUtils.isEmpty(hostsWithTagRules)) { + logger.info("No hosts found with tag rules matching the compute offering tag [{}].", hostTagOnOffering); + return; + } + + logger.info("Found hosts {} with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); + clusterHosts.addAll(hostsWithTagRules); + } + + /** + * Adds hosts with enough CPU capability and enough CPU capacity to the suitable hosts list. + */ + protected boolean hostHasCpuCapabilityAndCapacity(boolean considerReservedCapacity, ServiceOffering offering, Host host) { + logger.debug("Looking for CPU frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), offering::getRamSize); + Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); + Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); + Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); + + if (hasCpuCapability && hasCpuCapacity) { + logger.debug("Host {} is a suitable host as it has enough CPU capability and CPU capacity.", () -> host); + return true; + } + + logger.debug("Cannot use host {}. Does the host have CPU capability? {}. Does the host have CPU capacity? {}..", () -> host, () -> hasCpuCapability, () -> hasCpuCapacity); + return false; + } + +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 1110804959e..8b4ca318bf7 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -16,6 +16,41 @@ // under the License. package com.cloud.agent.manager.allocator.impl; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.firstfitleastconsumed; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.random; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.userdispersing; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.configuration.Config; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.deploy.DeploymentClusterPlanner; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.deploy.FirstFitPlanner; +import com.cloud.gpu.GPU; +import com.cloud.host.DetailVO; +import com.cloud.host.Host; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDetailsDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.vm.VMInstanceDetailVO; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; + import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; @@ -29,59 +64,19 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.gpu.dao.VgpuProfileDao; - +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.Capacity; -import com.cloud.capacity.CapacityManager; -import com.cloud.capacity.CapacityVO; -import com.cloud.capacity.dao.CapacityDao; -import com.cloud.configuration.Config; -import com.cloud.configuration.ConfigurationManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentClusterPlanner; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.deploy.FirstFitPlanner; -import com.cloud.gpu.GPU; -import com.cloud.host.DetailVO; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.host.dao.HostDetailsDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.service.ServiceOfferingDetailsVO; -import com.cloud.service.dao.ServiceOfferingDetailsDao; -import com.cloud.storage.GuestOSCategoryVO; -import com.cloud.storage.GuestOSVO; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.dao.GuestOSCategoryDao; -import com.cloud.storage.dao.GuestOSDao; -import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VMInstanceDetailVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.dao.VMInstanceDetailsDao; -import com.cloud.vm.dao.VMInstanceDao; - +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.jetbrains.annotations.NotNull; +import org.springframework.stereotype.Component; /** * An allocator that tries to find a fit on a computing host. This allocator does not care whether or not the host supports routing. */ @Component -public class FirstFitAllocator extends AdapterBase implements HostAllocator { - @Inject - protected HostDao _hostDao = null; +public class FirstFitAllocator extends BaseAllocator { @Inject HostDetailsDao _hostDetailsDao = null; @Inject @@ -95,289 +90,183 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { @Inject protected ResourceManager _resourceMgr; @Inject - ClusterDao _clusterDao; - @Inject - ClusterDetailsDao _clusterDetailsDao; - @Inject ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @Inject - CapacityManager _capacityMgr; - @Inject CapacityDao _capacityDao; @Inject - VMInstanceDetailsDao _vmInstanceDetailsDao; - @Inject - private VgpuProfileDao vgpuProfileDao; + VMInstanceDetailsDao vmInstanceDetailsDao; boolean _checkHvm = true; + static DecimalFormat decimalFormat = new DecimalFormat("#.##"); @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - Account account = vmProfile.getOwner(); - - boolean isVMDeployedWithUefi = false; - VMInstanceDetailVO vmInstanceDetailVO = _vmInstanceDetailsDao.findDetail(vmProfile.getId(), "UEFI"); - if(vmInstanceDetailVO != null){ - if ("secure".equalsIgnoreCase(vmInstanceDetailVO.getValue()) || "legacy".equalsIgnoreCase(vmInstanceDetailVO.getValue())) { - isVMDeployedWithUefi = true; - } - } - logger.info(" Guest VM is requested with Custom[UEFI] Boot Type "+ isVMDeployedWithUefi); - - - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of routing or not - return new ArrayList<>(); - } - String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - logger.debug("Looking for hosts in {}", paramAsStringToLog); - - String hostTagOnOffering = offering.getHostTag(); - String hostTagOnTemplate = template.getTemplateTag(); - String hostTagUefi = "UEFI"; - - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; - - List clusterHosts = new ArrayList<>(); - List hostsMatchingUefiTag = new ArrayList<>(); - if(isVMDeployedWithUefi){ - hostsMatchingUefiTag = _hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagUefi + "' are:" + hostsMatchingUefiTag); - } - } - - - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - if (haVmTag != null) { - clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag); - } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - clusterHosts = _resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId); - } else { - List hostsMatchingOfferingTag = new ArrayList<>(); - List hostsMatchingTemplateTag = new ArrayList<>(); - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsMatchingOfferingTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsMatchingOfferingTag); - } - } - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } - hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsMatchingTemplateTag); - } - } - - if (hasSvcOfferingTag && hasTemplateTag) { - hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - if (logger.isDebugEnabled()) { - logger.debug("Found " + hostsMatchingOfferingTag.size() + " Hosts satisfying both tags, host ids are:" + hostsMatchingOfferingTag); - } - - clusterHosts = hostsMatchingOfferingTag; - } else { - if (hasSvcOfferingTag) { - clusterHosts = hostsMatchingOfferingTag; - } else { - clusterHosts = hostsMatchingTemplateTag; - } - } - } - } - - if (isVMDeployedWithUefi) { - clusterHosts.retainAll(hostsMatchingUefiTag); - } - - clusterHosts.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); - - - if (clusterHosts.isEmpty()) { - logger.warn("No suitable host found for VM [{}] with tags {} in {}.", vmProfile, hostTagOnOffering, paramAsStringToLog); - return null; - } - // add all hosts that we are not considering to the avoid list - List allhostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); - allhostsInCluster.removeAll(clusterHosts); - - logger.debug(() -> String.format("Adding hosts [%s] to the avoid set because these hosts do not support HA.", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allhostsInCluster, "uuid", "name"))); - - for (HostVO host : allhostsInCluster) { - avoid.addHost(host.getId()); - } - - return allocateTo(vmProfile, plan, offering, template, avoid, clusterHosts, returnUpTo, considerReservedCapacity, account); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { + boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); Account account = vmProfile.getOwner(); - List suitableHosts = new ArrayList<>(); - List hostsCopy = new ArrayList<>(hosts); - - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of - // routing or not. - return suitableHosts; - } String hostTagOnOffering = offering.getHostTag(); String hostTagOnTemplate = template.getTemplateTag(); - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; + String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - if (haVmTag != null) { - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + List suitableHosts = retrieveHosts(vmProfile, type, (List) hosts, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); + + if (suitableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in {}.", vmProfile, paramAsStringToLog); + return null; + } + + if (CollectionUtils.isEmpty(hosts)) { + addHostsToAvoidSet(type, avoid, clusterId, podId, dcId, suitableHosts); + } + + return allocateTo(vmProfile, plan, offering, template, avoid, suitableHosts, returnUpTo, considerReservedCapacity, account); + } + + protected List retrieveHosts(VirtualMachineProfile vmProfile, Type type, List hostsToFilter, Long clusterId, Long podId, long dcId, String hostTagOnOffering, + String hostTagOnTemplate) { + String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + List clusterHosts; + + if (CollectionUtils.isNotEmpty(hostsToFilter)) { + clusterHosts = new ArrayList<>(hostsToFilter); } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); - } else { - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); - - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsCopy); - } - } - - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } - - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); - - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsCopy); - } - } - } + clusterHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); } - hostsCopy.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); - - if (!hostsCopy.isEmpty()) { - suitableHosts = allocateTo(vmProfile, plan, offering, template, avoid, hostsCopy, returnUpTo, considerReservedCapacity, account); + if (haVmTag != null) { + clusterHosts.retainAll(hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { + clusterHosts.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); + } else { + retainHostsMatchingServiceOfferingAndTemplateTags(clusterHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); } + filterHostsWithUefiEnabled(type, vmProfile, clusterId, podId, dcId, clusterHosts); + + addHostsBasedOnTagRules(hostTagOnOffering, clusterHosts); + + return clusterHosts; + + } + + /** + * Add all hosts to the avoid set that were not considered during the allocation + */ + protected void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Long podId, long dcId, List suitableHosts) { + List allHostsInCluster = hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); + + allHostsInCluster.removeAll(suitableHosts); + + logger.debug("Adding hosts [{}] to the avoid set because these hosts were not considered for allocation.", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allHostsInCluster, "uuid", "name")); + + for (HostVO host : allHostsInCluster) { + avoid.addHost(host.getId()); + } + } + + protected void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmProfile, Long clusterId, Long podId, long dcId, List clusterHosts) { + VMInstanceDetailVO vmInstanceDetailVO = vmInstanceDetailsDao.findDetail(vmProfile.getId(), "UEFI"); + + if (vmInstanceDetailVO == null) { + return; + } + + if (!StringUtils.equalsAnyIgnoreCase(vmInstanceDetailVO.getValue(), ApiConstants.BootMode.SECURE.toString(), ApiConstants.BootMode.LEGACY.toString())) { + return; + } + + logger.info("Guest VM is requested with Custom[UEFI] Boot Type enabled."); + + List hostsMatchingUefiTag = hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); + + logger.debug("Hosts with UEFI enabled are {}.", hostsMatchingUefiTag); + clusterHosts.retainAll(hostsMatchingUefiTag); + } + + protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity, Account account) { + String vmAllocationAlgorithm = DeploymentClusterPlanner.VmAllocationAlgorithm.value(); + if (random.toString().equals(vmAllocationAlgorithm)) { + Collections.shuffle(hosts); + } else if (userdispersing.toString().equals(vmAllocationAlgorithm)) { + hosts = reorderHostsByNumberOfVms(plan, hosts, account); + } else if (firstfitleastconsumed.toString().equals(vmAllocationAlgorithm)) { + hosts = reorderHostsByCapacity(plan, hosts); + } + + logger.debug("FirstFitAllocator has {} hosts to check for allocation {}.", hosts.size(), hosts); + hosts = prioritizeHosts(template, offering, hosts); + logger.debug("Found {} hosts for allocation after prioritization: {}.", hosts.size(), hosts); + + List suitableHosts = checkHostsCompatibilities(offering, vmProfile, avoid, hosts, returnUpTo, considerReservedCapacity); + logger.debug("Host Allocator returning {} suitable hosts.", suitableHosts.size()); + return suitableHosts; } - protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity, Account account) { - String vmAllocationAlgorithm = DeploymentClusterPlanner.VmAllocationAlgorithm.value(); - if (vmAllocationAlgorithm.equals("random")) { - // Shuffle this so that we don't check the hosts in the same order. - Collections.shuffle(hosts); - } else if (vmAllocationAlgorithm.equals("userdispersing")) { - hosts = reorderHostsByNumberOfVms(plan, hosts, account); - } else if(vmAllocationAlgorithm.equals("firstfitleastconsumed")){ - hosts = reorderHostsByCapacity(plan, hosts); - } - - if (logger.isDebugEnabled()) { - logger.debug("FirstFitAllocator has " + hosts.size() + " hosts to check for allocation: " + hosts); - } - - // We will try to reorder the host lists such that we give priority to hosts that have - // the minimums to support a VM's requirements - hosts = prioritizeHosts(template, offering, hosts); - - if (logger.isDebugEnabled()) { - logger.debug("Found " + hosts.size() + " hosts for allocation after prioritization: " + hosts); - } - - if (logger.isDebugEnabled()) { - logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize() + " MB"); - } - - long serviceOfferingId = offering.getId(); + protected List checkHostsCompatibilities(ServiceOffering offering, VirtualMachineProfile vmProfile, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { List suitableHosts = new ArrayList<>(); - ServiceOfferingDetailsVO offeringDetails = null; + logger.debug("Checking compatibility for the following hosts {}.", suitableHosts); for (Host host : hosts) { if (suitableHosts.size() == returnUpTo) { break; } + if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug("Host: {} is in avoid set, skipping this and trying other available hosts", host); - } + logger.debug("Host [{}] is in avoid set, skipping this and trying other available hosts", host); continue; } - //find number of guest VMs occupying capacity on this host. - if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { + if (capacityManager.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Adding host [{}] to the avoid set because this host already has the max number of running (user and/or system) VMs.", host); avoid.addHost(host.getId()); continue; } - // Check if GPU device is required by offering and host has the availability - if (_resourceMgr.isGPUDeviceAvailable(offering, host, vmProfile.getId())) { - logger.debug("Host [{}] has required GPU devices available.", host); - } else { - // If GPU is not available, skip this host - logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", host); - avoid.addHost(host.getId()); + if (offeringRequestedVGpuAndHostDoesNotHaveIt(offering, vmProfile, avoid, host)) { continue; } - Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (cpuCapabilityAndCapacity.first() && cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug("Found a suitable host, adding to list: {}", host); - } + if (hostHasCpuCapabilityAndCapacity(considerReservedCapacity, offering, host)) { suitableHosts.add(host); - } else { - if (logger.isDebugEnabled()) { - logger.debug("Not using host {}; host has cpu capability? {}, host has capacity?{}", - host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second()); - } - avoid.addHost(host.getId()); + continue; } + avoid.addHost(host.getId()); } - - if (logger.isDebugEnabled()) { - logger.debug("Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; } - // Reorder hosts in the decreasing order of free capacity. + protected boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offering, VirtualMachineProfile vmProfile, ExcludeList avoid, Host host) { + if (_resourceMgr.isGPUDeviceAvailable(offering, host, vmProfile.getId())) { + logger.debug("Host [{}] has required GPU devices available.", () -> host); + return false; + } + + logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", () -> host); + avoid.addHost(host.getId()); + return true; + } + + /** + * Reorder hosts in the decreasing order of free capacity. + */ private List reorderHostsByCapacity(DeploymentPlan plan, List hosts) { Long zoneId = plan.getDataCenterId(); Long clusterId = plan.getClusterId(); @@ -388,26 +277,10 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) .collect(Collectors.toMap(Map.Entry::getKey, entry -> decimalFormat.format(entry.getValue() * 100) + "%", (e1, e2) -> e1, LinkedHashMap::new)); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts: [{}] in descending order of free capacity (percentage) in the cluster: {}", - hostIdsByFreeCapacity, sortedHostByCapacity); - } + logger.debug("List of hosts: [{}] in descending order of free capacity (percentage) in the cluster: {}.", + hostIdsByFreeCapacity, sortedHostByCapacity); - //now filter the given list of Hosts by this ordered list - Map hostMap = new HashMap<>(); - for (Host host : hosts) { - hostMap.put(host.getId(), host); - } - List matchingHostIds = new ArrayList<>(hostMap.keySet()); - - hostIdsByFreeCapacity.retainAll(matchingHostIds); - - List reorderedHosts = new ArrayList<>(); - for(Long id: hostIdsByFreeCapacity){ - reorderedHosts.add(hostMap.get(id)); - } - - return reorderedHosts; + return filterHosts(hosts, hostIdsByFreeCapacity); } private Pair, Map> getOrderedHostsByCapacity(Long zoneId, Long clusterId) { @@ -450,116 +323,154 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { Long clusterId = plan.getClusterId(); List hostIdsByVmCount = _vmInstanceDao.listHostIdsByVmCount(dcId, podId, clusterId, account.getAccountId()); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts in ascending order of number of VMs: " + hostIdsByVmCount); - } + logger.debug("List of hosts in ascending order of number of VMs: {}.", hostIdsByVmCount); - //now filter the given list of Hosts by this ordered list + return filterHosts(hosts, hostIdsByVmCount); + } + + /** + * Filter the given list of Hosts considering the ordered list + */ + private List filterHosts(List hosts, List orderedHostIdsList) { Map hostMap = new HashMap<>(); + for (Host host : hosts) { hostMap.put(host.getId(), host); } List matchingHostIds = new ArrayList<>(hostMap.keySet()); - - hostIdsByVmCount.retainAll(matchingHostIds); + orderedHostIdsList.retainAll(matchingHostIds); List reorderedHosts = new ArrayList<>(); - for (Long id : hostIdsByVmCount) { + for(Long id: orderedHostIdsList){ reorderedHosts.add(hostMap.get(id)); } return reorderedHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - + /** + * Reorder the host list giving priority to hosts that have the minimum to support the VM's requirements. + */ protected List prioritizeHosts(VMTemplateVO template, ServiceOffering offering, List hosts) { if (template == null) { return hosts; } - // Determine the guest OS category of the template - String templateGuestOSCategory = getTemplateGuestOSCategory(template); + List hostsToCheck = filterHostWithNoHvmIfTemplateRequested(template, hosts); List prioritizedHosts = new ArrayList<>(); - List noHvmHosts = new ArrayList<>(); - - // If a template requires HVM and a host doesn't support HVM, remove it from consideration - List hostsToCheck = new ArrayList<>(); - if (template.isRequiresHvm()) { - for (Host host : hosts) { - if (hostSupportsHVM(host)) { - hostsToCheck.add(host); - } else { - noHvmHosts.add(host); - } - } - } else { - hostsToCheck.addAll(hosts); - } - - if (logger.isDebugEnabled()) { - if (noHvmHosts.size() > 0) { - logger.debug("Not considering hosts: " + noHvmHosts + " to deploy template: " + template + " as they are not HVM enabled"); - } - } - // If a host is tagged with the same guest OS category as the template, move it to a high priority list - // If a host is tagged with a different guest OS category than the template, move it to a low priority list List highPriorityHosts = new ArrayList<>(); List lowPriorityHosts = new ArrayList<>(); - for (Host host : hostsToCheck) { - String hostGuestOSCategory = getHostGuestOSCategory(host); - if (hostGuestOSCategory == null) { - continue; - } else if (templateGuestOSCategory != null && templateGuestOSCategory.equals(hostGuestOSCategory)) { - highPriorityHosts.add(host); - } else { - lowPriorityHosts.add(host); - } - } + prioritizeHostsWithMatchingGuestOs(template, hostsToCheck, highPriorityHosts, lowPriorityHosts); hostsToCheck.removeAll(highPriorityHosts); hostsToCheck.removeAll(lowPriorityHosts); - // Prioritize the remaining hosts by HVM capability - for (Host host : hostsToCheck) { - if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { - // Host and template both do not support hvm, put it as first consideration - prioritizedHosts.add(0, host); - } else { - // Template doesn't require hvm, but the machine supports it, make it last for consideration - prioritizedHosts.add(host); - } - } - - // Merge the lists + prioritizeHostsByHvmCapability(template, hostsToCheck, prioritizedHosts); prioritizedHosts.addAll(0, highPriorityHosts); prioritizedHosts.addAll(lowPriorityHosts); - // if service offering is not GPU enabled then move all the GPU enabled hosts to the end of priority list. - if (_serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null && offering.getVgpuProfileId() == null) { + prioritizeHostsByGpuEnabled(offering, prioritizedHosts); - List gpuEnabledHosts = new ArrayList<>(); - // Check for GPU enabled hosts. - for (Host host : prioritizedHosts) { - if (_resourceMgr.isHostGpuEnabled(host.getId())) { - gpuEnabledHosts.add(host); - } - } - // Move GPU enabled hosts to the end of list - if(!gpuEnabledHosts.isEmpty()) { - prioritizedHosts.removeAll(gpuEnabledHosts); - prioritizedHosts.addAll(gpuEnabledHosts); - } - } return prioritizedHosts; } + + /** + * If a template requires HVM and a host doesn't support HVM, remove it from consideration. + */ + protected List filterHostWithNoHvmIfTemplateRequested(VMTemplateVO template, List hosts) { + List hostsToCheck = new ArrayList<>(); + + if (!template.isRequiresHvm()) { + logger.debug("Template [{}] does not require HVM, therefore, the hosts {} will not be checked for HVM compatibility.", template, hostsToCheck); + hostsToCheck.addAll(hosts); + return hostsToCheck; + } + + List noHvmHosts = new ArrayList<>(); + logger.debug("Template [{}] requires HVM, therefore, the hosts %s will be checked for HVM compatibility.", template, hostsToCheck); + + for (Host host : hosts) { + if (hostSupportsHVM(host)) { + hostsToCheck.add(host); + } else { + noHvmHosts.add(host); + } + } + + if (!noHvmHosts.isEmpty()) { + logger.debug("Not considering hosts {} to deploy VM using template {} as they are not HVM enabled.", noHvmHosts, template); + } + + return hostsToCheck; + } + + + /** + * If service offering did not request for vGPU, then move all host with GPU to the end of the host priority list. + */ + protected void prioritizeHostsByGpuEnabled(ServiceOffering offering, List prioritizedHosts) { + boolean serviceOfferingRequestedVGpu = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) != null || offering.getVgpuProfileId() != null; + + if (serviceOfferingRequestedVGpu) { + return; + } + + List gpuEnabledHosts = new ArrayList<>(); + + for (Host host : prioritizedHosts) { + if (_resourceMgr.isHostGpuEnabled(host.getId())) { + gpuEnabledHosts.add(host); + } + } + + if (!gpuEnabledHosts.isEmpty()) { + prioritizedHosts.removeAll(gpuEnabledHosts); + prioritizedHosts.addAll(gpuEnabledHosts); + } + } + + /** + * Prioritize remaining host by HVM capability. + * + *
    + *
  • If host and template both do not support HVM, put it at the start of the list.
    • + *
  • If the template doesn't require HVM, but the machine supports it, append it to the list.
    • + *
+ */ + protected void prioritizeHostsByHvmCapability(VMTemplateVO template, List hostsToCheck, List prioritizedHosts) { + for (Host host : hostsToCheck) { + if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { + prioritizedHosts.add(0, host); + } else { + prioritizedHosts.add(host); + } + } + } + + + /** + *
    + *
  • If a host is tagged with the same guest OS category as the template, move it to a high priority list.
    • + *
  • If a host is tagged with a different guest OS category than the template, move it to a low priority list.
    • + *
+ */ + protected void prioritizeHostsWithMatchingGuestOs(VMTemplateVO template, List hostsToCheck, List highPriorityHosts, List lowPriorityHosts) { + String templateGuestOSCategory = getTemplateGuestOSCategory(template); + + for (Host host : hostsToCheck) { + String hostGuestOSCategory = getHostGuestOSCategory(host); + + if (StringUtils.equals(templateGuestOSCategory, hostGuestOSCategory)) { + highPriorityHosts.add(host); + } else if (hostGuestOSCategory != null) { + lowPriorityHosts.add(host); + } + } + } + + protected boolean hostSupportsHVM(Host host) { if (!_checkHvm) { return true; @@ -621,19 +532,8 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (_configDao != null) { Map configs = _configDao.getConfiguration(params); String value = configs.get("xenserver.check.hvm"); - _checkHvm = value == null ? true : Boolean.parseBoolean(value); + _checkHvm = value == null || Boolean.parseBoolean(value); } return true; } - - @Override - public boolean start() { - return true; - } - - @Override - public boolean stop() { - return true; - } - } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java new file mode 100644 index 00000000000..96b4255b876 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.host.Host; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.VMTemplateVO; +import com.cloud.vm.VirtualMachineProfile; + +@Component +public class RandomAllocator extends BaseAllocator { + @Inject + private ResourceManager _resourceMgr; + + protected List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + + long dcId = plan.getDataCenterId(); + Long podId = plan.getPodId(); + Long clusterId = plan.getClusterId(); + ServiceOffering offering = vmProfile.getServiceOffering(); + + String offeringHostTag = offering.getHostTag(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); + logger.debug("Looking for hosts in zone [{}], pod [{}], cluster [{}].", dcId, podId, clusterId); + + List availableHosts = retrieveHosts(type, (List) hosts, template, offeringHostTag, clusterId, podId, dcId); + + if (availableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in zone [{}], pod [{}], cluster [{}].", vmProfile, dcId, podId, clusterId); + return null; + } + + return filterAvailableHosts(avoid, returnUpTo, considerReservedCapacity, availableHosts, offering); + } + + protected List filterAvailableHosts(ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity, List availableHosts, ServiceOffering offering) { + logger.debug("Random Allocator found [{}] available hosts. They will be checked if they are in the avoid set and for CPU capability and capacity.", availableHosts::size); + List suitableHosts = new ArrayList<>(); + + Collections.shuffle(availableHosts); + for (Host host : availableHosts) { + if (suitableHosts.size() == returnUpTo) { + break; + } + + if (avoid.shouldAvoid(host)) { + logger.debug("Host [{}] is in the avoid set, skipping it and trying other available hosts.", () -> host); + continue; + } + + if (!hostHasCpuCapabilityAndCapacity(considerReservedCapacity, offering, host)) { + continue; + } + + logger.debug("Found the suitable host [{}], adding to list.", () -> host); + suitableHosts.add(host); + } + + logger.debug("Random Host Allocator returning {} suitable hosts.", suitableHosts::size); + return suitableHosts; + } + + /** + * @return all computing hosts, regardless of whether they support routing. + */ + protected List retrieveHosts(Type type, List hosts, VMTemplateVO template, String offeringHostTag, Long clusterId, Long podId, long dcId) { + List availableHosts; + String templateTag = template.getTemplateTag(); + + if (CollectionUtils.isNotEmpty(hosts)) { + availableHosts = new ArrayList<>(hosts); + } else { + availableHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + } + + if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { + retainHostsMatchingServiceOfferingAndTemplateTags(availableHosts, type, dcId, podId, clusterId, offeringHostTag, templateTag); + } else { + List hostsWithNoRuleTag = hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); + logger.debug("Retaining hosts {} because they do not have rule tags.", hostsWithNoRuleTag); + availableHosts.retainAll(hostsWithNoRuleTag); + } + + addHostsBasedOnTagRules(offeringHostTag, availableHosts); + + return availableHosts; + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity) { + return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); + } +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java index fd8e65e2476..04a0b182b7b 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java @@ -28,32 +28,24 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public class TestingAllocator extends AdapterBase implements HostAllocator { @Inject HostDao _hostDao; - Long _computingHost; Long _storageHost; Long _routingHost; @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - List availableHosts = new ArrayList(); + boolean considerReservedCapacity) { + List availableHosts = new ArrayList<>(); Host host = null; if (type == Host.Type.Routing && _routingHost != null) { host = _hostDao.findById(_routingHost); @@ -66,13 +58,6 @@ public class TestingAllocator extends AdapterBase implements HostAllocator { return availableHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - @Override public boolean configure(String name, Map params) { String value = (String)params.get(Host.Type.Routing.toString()); diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index f163a3d52a5..7fb4a97e149 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -20,7 +20,6 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -1714,20 +1713,19 @@ StateListener, Configurable { @Override public void reorderHostsByPriority(Map priorities, List hosts) { - logger.debug("Re-ordering hosts {} by priorities {}", hosts, priorities); + if (CollectionUtils.isEmpty(hosts)){ + logger.debug("Hosts list is empty; therefore, there is nothing to reorder."); + return; + } + logger.info("Re-ordering hosts [{}] by priorities [{}].", hosts, priorities); hosts.removeIf(host -> DataCenterDeployment.PROHIBITED_HOST_PRIORITY.equals(getHostPriority(priorities, host.getId()))); + hosts.sort((host1, host2) -> { + int res = getHostPriority(priorities, host1.getId()).compareTo(getHostPriority(priorities, host2.getId())); + return -res; + }); - Collections.sort(hosts, new Comparator<>() { - @Override - public int compare(Host host1, Host host2) { - int res = getHostPriority(priorities, host1.getId()).compareTo(getHostPriority(priorities, host2.getId())); - return -res; - } - } - ); - - logger.debug("Hosts after re-ordering are: {}", hosts); + logger.info("Hosts after re-ordering are: [{}].", hosts); } private Integer getHostPriority(Map priorities, Long hostId) { diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 729da7caa33..27facb304eb 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -1042,7 +1042,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(ownerId, ownerType, resourceType, tag); - ActionEventUtils.onActionEvent(caller.getId(), caller.getAccountId(), + Long callingUserId = CallContext.current().getCallingUserId(); + ActionEventUtils.onActionEvent(callingUserId, caller.getAccountId(), caller.getDomainId(), EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, "Resource limit updated. Resource Type: " + resourceType + ", New Value: " + max, ownerResourceId, ownerResourceType.toString()); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index d4266496e98..337bdbbbc63 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1696,11 +1696,7 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen List suitableHosts = new ArrayList<>(); for (final HostAllocator allocator : hostAllocators) { - if (CollectionUtils.isNotEmpty(compatibleHosts)) { - suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, compatibleHosts, HostAllocator.RETURN_UPTO_ALL, false); - } else { - suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, HostAllocator.RETURN_UPTO_ALL, false); - } + suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, compatibleHosts, HostAllocator.RETURN_UPTO_ALL, false); if (CollectionUtils.isNotEmpty(suitableHosts)) { break; @@ -1709,10 +1705,10 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen _dpMgr.reorderHostsByPriority(plan.getHostPriorities(), suitableHosts); - if (suitableHosts.isEmpty()) { + if (CollectionUtils.isEmpty(suitableHosts)) { logger.warn("No suitable hosts found."); } else { - logger.debug("Hosts having capacity and suitable for migration: {}", suitableHosts); + logger.debug("Hosts having capacity and are suitable for migration: {}", suitableHosts); } // Only list hosts of the same architecture as the source Host in a multi-arch zone diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 0e013421fdd..7863d66e357 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4359,9 +4359,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return resourceLimitService.getResourceLimitStorageTags(diskOfferingVO); } - private List reserveStorageResourcesForVm(Account owner, Long diskOfferingId, Long diskSize, List dataDiskInfoList, Long rootDiskOfferingId, ServiceOfferingVO offering, Long rootDiskSize) throws ResourceAllocationException { - List checkedReservations = new ArrayList<>(); - + private void reserveStorageResourcesForVm(List checkedReservations, Account owner, Long diskOfferingId, Long diskSize, List dataDiskInfoList, Long rootDiskOfferingId, ServiceOfferingVO offering, Long rootDiskSize) throws ResourceAllocationException { List rootResourceLimitStorageTags = getResourceLimitStorageTags(rootDiskOfferingId != null ? rootDiskOfferingId : offering.getDiskOfferingId()); CheckedReservation rootVolumeReservation = new CheckedReservation(owner, ResourceType.volume, rootResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); checkedReservations.add(rootVolumeReservation); @@ -4369,12 +4367,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir checkedReservations.add(rootPrimaryStorageReservation); if (diskOfferingId != null) { - List additionalResourceLimitStorageTags = diskOfferingId != null ? getResourceLimitStorageTags(diskOfferingId) : null; + List additionalResourceLimitStorageTags = getResourceLimitStorageTags(diskOfferingId); DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); Long size = verifyAndGetDiskSize(diskOffering, diskSize); - CheckedReservation additionalVolumeReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService) : null; + CheckedReservation additionalVolumeReservation = new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); checkedReservations.add(additionalVolumeReservation); - CheckedReservation additionalPrimaryStorageReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, size, reservationDao, resourceLimitService) : null; + CheckedReservation additionalPrimaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, size, reservationDao, resourceLimitService); checkedReservations.add(additionalPrimaryStorageReservation); } @@ -4390,7 +4388,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir checkedReservations.add(additionalPrimaryStorageReservation); } } - return checkedReservations; } private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, @@ -4404,8 +4401,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Long rootDiskOfferingId, Long rootDiskKmsKeyId, long volumesSize, Volume volume, Snapshot snapshot) throws ResourceAllocationException { List checkedReservations = new ArrayList<>(); + try { - checkedReservations = reserveStorageResourcesForVm(owner, diskOfferingId, diskSize, dataDiskInfoList, rootDiskOfferingId, offering, volumesSize); + reserveStorageResourcesForVm(checkedReservations, owner, diskOfferingId, diskSize, dataDiskInfoList, rootDiskOfferingId, offering, volumesSize); // verify security group ids if (securityGroupIdList != null) { @@ -4696,14 +4694,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir logger.error("error during resource reservation and allocation", e); throw new CloudRuntimeException(e); } finally { - for (CheckedReservation checkedReservation : checkedReservations) { - try { - checkedReservation.close(); - } catch (Exception e) { - logger.error("error during resource reservation and allocation", e); - throw new CloudRuntimeException(e); - } - } + ReservationHelper.closeAll(checkedReservations); } } diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index d3781649845..28b96b8d194 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -34,6 +34,9 @@ + + diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java new file mode 100644 index 00000000000..ddd1e3959d5 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.capacity.CapacityManager; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachineProfile; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class BaseAllocatorTest { + @Mock + HostDao hostDaoMock; + + @Mock + CapacityManager capacityManagerMock; + + @InjectMocks + @Spy + BaseAllocator baseAllocator = new MockBaseAllocator(); + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 2L; + + private final Long dcId = 3L; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final String hostTag = "hostTag"; + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagShouldRetainHostsWithServiceOfferingTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = null; + + Mockito.doReturn(hostsWithMathingTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagAndHasHostTagOnTemplateShouldRetainHostsWithServiceOfferingTagAndTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + List hostsWithMathingTemplateTags = new ArrayList<>(Arrays.asList(host1, host2)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingTemplateTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasHostTagOnTemplateShouldRetainHostsWithTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestNoServiceTagAndNoTemplateTagShouldHaveAllSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = null; + + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsEmptyShouldNotAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List emptyList = new ArrayList<>(); + + Mockito.doReturn(emptyList).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsNotEmptyShouldAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsMatchingRuleTag = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsMatchingRuleTag).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityAndCpuCapacityShouldReturnTrue() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertTrue(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityButNoCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityButHasCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityAndCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + class MockBaseAllocator extends BaseAllocator { + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, int returnUpTo) { + return null; + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { + return null; + } + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java index 0d6a6fa8ff9..00cac5bbd52 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java @@ -18,214 +18,596 @@ package com.cloud.agent.manager.allocator.impl; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.CapacityVO; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner; import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingDetailsVO; +import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; import com.cloud.utils.Pair; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDetailsDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - +@RunWith(MockitoJUnitRunner.class) public class FirstFitAllocatorTest { - private static final double TOLERANCE = 0.0001; - private FirstFitAllocator allocator; - private CapacityManager capacityMgr; - private ServiceOfferingDetailsDao offeringDetailsDao; - private ResourceManager resourceMgr; + private static final double TOLERANCE = 0.0001; - private DeploymentPlan plan; - private ServiceOffering offering; - private DeploymentPlanner.ExcludeList avoid; - private Account account; + @Mock + HostDao hostDaoMock; - private Host host1; - private Host host2; - private VirtualMachineProfile vmProfile; - ConfigurationDao configDao; + @Mock + ResourceManager resourceManagerMock; - @Before - public void setUp() { - allocator = new FirstFitAllocator(); - capacityMgr = mock(CapacityManager.class); - offeringDetailsDao = mock(ServiceOfferingDetailsDao.class); - resourceMgr = mock(ResourceManager.class); - configDao = mock(ConfigurationDao.class); + @Mock + VMInstanceDetailsDao userVmDetailsDaoMock; - allocator._capacityMgr = capacityMgr; - allocator._serviceOfferingDetailsDao = offeringDetailsDao; - allocator._resourceMgr = resourceMgr; - allocator._configDao = configDao; + @Mock + ServiceOfferingDetailsDao serviceOfferingDetailsDao; - plan = mock(DeploymentPlan.class); - offering = mock(ServiceOffering.class); - avoid = mock(DeploymentPlanner.ExcludeList.class); - account = mock(Account.class); + @Mock + CapacityManager capacityMgr; - host1 = mock(Host.class); - host2 = mock(Host.class); + @Mock + ConfigurationDao configDao; - vmProfile = mock(VirtualMachineProfile.class); - when(vmProfile.getId()).thenReturn(1L); + @Spy + @InjectMocks + FirstFitAllocator firstFitAllocatorSpy; - when(plan.getDataCenterId()).thenReturn(1L); - when(offering.getCpu()).thenReturn(2); - when(offering.getSpeed()).thenReturn(1000); - when(offering.getRamSize()).thenReturn(2048); - when(offering.getId()).thenReturn(123L); - when(offering.getHostTag()).thenReturn(null); - when(offering.getVgpuProfileId()).thenReturn(null); - } + private final Host.Type type = Host.Type.Routing; - @Test - public void testConfigure() throws Exception { - when(configDao.getConfiguration(anyMap())).thenReturn(new HashMap<>()); - assertTrue(allocator._checkHvm); - assertTrue(allocator.configure("test", new HashMap<>())); - } + private final Long clusterId = 1L; - @Test - public void testAllocateTo_SuccessfulMatch() { - List inputHosts = Arrays.asList(host1, host2); + private final Long podId = 2L; - // All hosts are allowed - when(avoid.shouldAvoid(host1)).thenReturn(false); - when(avoid.shouldAvoid(host2)).thenReturn(false); + private final Long dcId = 3L; - // No GPU requirement - when(offeringDetailsDao.findDetail(eq(123L), anyString())).thenReturn(null); + private final List emptyList = new ArrayList<>(); - // CPU capability and capacity is met - when(capacityMgr.checkIfHostReachMaxGuestLimit(any())).thenReturn(false); - when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(eq(host1), eq(offering), eq(true))) - .thenReturn(new Pair<>(true, true)); - when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(eq(host2), eq(offering), eq(true))) - .thenReturn(new Pair<>(true, false)); + private final String hostTag = "hostTag"; + private final String templateTag = "templateTag"; - when(resourceMgr.isGPUDeviceAvailable(offering, host1, vmProfile.getId())).thenReturn(true); - when(resourceMgr.isGPUDeviceAvailable(offering, host2, vmProfile.getId())).thenReturn(true); + private final HostVO host1 = mock(HostVO.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 2, true, account); + private final HostVO host2 = mock(HostVO.class); - // Only host1 should be returned - assertEquals(1, result.size()); - assertTrue(result.contains(host1)); - assertFalse(result.contains(host2)); - } + private final HostVO host3 = mock(HostVO.class); - @Test - public void testAllocateTo_AvoidSetAndGuestLimit() { - List inputHosts = Arrays.asList(host1, host2); + private final ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); - when(avoid.shouldAvoid(host1)).thenReturn(true); // Avoided - when(avoid.shouldAvoid(host2)).thenReturn(false); + private final DeploymentPlanner.ExcludeList excludeList = mock(DeploymentPlanner.ExcludeList.class); - when(capacityMgr.checkIfHostReachMaxGuestLimit(host2)).thenReturn(true); // Reached limit + private final VirtualMachineProfile virtualMachineProfile = mock(VirtualMachineProfile.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 2, true, account); + private final VMTemplateVO vmTemplateVO = mock(VMTemplateVO.class); - assertTrue(result.isEmpty()); - } + private final Account account = mock(Account.class); - @Test - public void testAllocateTo_GPUNotAvailable() { - List inputHosts = Arrays.asList(host1); - when(avoid.shouldAvoid(host1)).thenReturn(false); + private final DeploymentPlan deploymentPlan = mock(DeploymentPlan.class); - // GPU required but not available - var vgpuDetail = mock(com.cloud.service.ServiceOfferingDetailsVO.class); - var pciDetail = mock(com.cloud.service.ServiceOfferingDetailsVO.class); - when(offeringDetailsDao.findDetail(eq(123L), eq("vgpuType"))).thenReturn(vgpuDetail); - when(offeringDetailsDao.findDetail(eq(123L), eq("pciDevice"))).thenReturn(pciDetail); - when(pciDetail.getValue()).thenReturn("NVIDIA"); - when(vgpuDetail.getValue()).thenReturn("GRID"); + private final DeploymentPlanner.ExcludeList avoid = mock(DeploymentPlanner.ExcludeList.class); - when(resourceMgr.isGPUDeviceAvailable(offering, host1, vmProfile.getId())).thenReturn(false); + private final ServiceOffering offering = mock(ServiceOffering.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 1, true, account); + private final boolean considerReservedCapacity = true; - assertTrue(result.isEmpty()); - } + @Test + public void testConfigure() throws Exception { + when(configDao.getConfiguration(Mockito.anyMap())).thenReturn(new HashMap<>()); + Assert.assertTrue(firstFitAllocatorSpy._checkHvm); + Assert.assertTrue(firstFitAllocatorSpy.configure("test", new HashMap<>())); + } - @Test - public void testHostByCombinedCapacityOrder() { - // Test scenario 1: Default capacity usage (0.5 weight) - List mockCapacity = getHostCapacities(); - Map hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.5); + @Test + public void testAllocateTo_SuccessfulMatch() { + List inputHosts = Arrays.asList(host1, host2); - // Verify host ordering and capacity values - Long firstHostId = hostByCombinedCapacity.keySet().iterator().next(); - Assert.assertEquals("Host with ID 1 should be first in ordering", Long.valueOf(1L), firstHostId); - Assert.assertEquals("Host 1 combined capacity should match expected value", - 0.9609375, hostByCombinedCapacity.get(1L), TOLERANCE); - Assert.assertEquals("Host 2 combined capacity should match expected value", - 0.9296875, hostByCombinedCapacity.get(2L), TOLERANCE); + // All hosts are allowed + when(avoid.shouldAvoid(host1)).thenReturn(false); + when(avoid.shouldAvoid(host2)).thenReturn(false); - // Test scenario 2: Modified capacity usage (0.7 weight) - when(mockCapacity.get(0).getUsedCapacity()).thenReturn(1500L); - hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.7); + // CPU capability and capacity is met + when(capacityMgr.checkIfHostReachMaxGuestLimit(Mockito.any(Host.class))).thenReturn(false); + when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host1, offering, true)) + .thenReturn(new Pair<>(true, true)); + when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host2, offering,true)) + .thenReturn(new Pair<>(true, false)); - // Verify new ordering after capacity change - firstHostId = hostByCombinedCapacity.keySet().iterator().next(); - Assert.assertEquals("Host with ID 2 should be first after capacity change", Long.valueOf(2L), firstHostId); - Assert.assertEquals("Host 2 combined capacity should match expected value after change", - 0.9515625, hostByCombinedCapacity.get(2L), TOLERANCE); - Assert.assertEquals("Host 1 combined capacity should match expected value after change", - 0.9484375, hostByCombinedCapacity.get(1L), TOLERANCE); - } + when(resourceManagerMock.isGPUDeviceAvailable(offering, host1, virtualMachineProfile.getId())).thenReturn(true); + when(resourceManagerMock.isGPUDeviceAvailable(offering, host2, virtualMachineProfile.getId())).thenReturn(true); - List getHostCapacities() { - CapacityVO cpuCapacity1 = mock(CapacityVO.class); - when(cpuCapacity1.getHostOrPoolId()).thenReturn(1L); - when(cpuCapacity1.getTotalCapacity()).thenReturn(32000L); - when(cpuCapacity1.getReservedCapacity()).thenReturn(0L); - when(cpuCapacity1.getUsedCapacity()).thenReturn(500L); - when(cpuCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 2, true, account); - CapacityVO cpuCapacity2 = mock(CapacityVO.class); - when(cpuCapacity2.getHostOrPoolId()).thenReturn(2L); - when(cpuCapacity2.getTotalCapacity()).thenReturn(32000L); - when(cpuCapacity2.getReservedCapacity()).thenReturn(0L); - when(cpuCapacity2.getUsedCapacity()).thenReturn(500L); - when(cpuCapacity2.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + // Only host1 should be returned + Assert.assertEquals(1, result.size()); + Assert.assertTrue(result.contains(host1)); + Assert.assertFalse(result.contains(host2)); + } - CapacityVO memCapacity1 = mock(CapacityVO.class); - when(memCapacity1.getHostOrPoolId()).thenReturn(1L); - when(memCapacity1.getTotalCapacity()).thenReturn(8589934592L); - when(memCapacity1.getReservedCapacity()).thenReturn(0L); - when(memCapacity1.getUsedCapacity()).thenReturn(536870912L); - when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + @Test + public void testAllocateTo_AvoidSetAndGuestLimit() { + List inputHosts = Arrays.asList(host1, host2); - CapacityVO memCapacity2 = mock(CapacityVO.class); - when(memCapacity2.getHostOrPoolId()).thenReturn(2L); - when(memCapacity2.getTotalCapacity()).thenReturn(8589934592L); - when(memCapacity2.getReservedCapacity()).thenReturn(0L); - when(memCapacity2.getUsedCapacity()).thenReturn(1073741824L); - when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); - return Arrays.asList(cpuCapacity1, memCapacity1, cpuCapacity2, memCapacity2); - } + when(avoid.shouldAvoid(host1)).thenReturn(true); // Avoided + when(avoid.shouldAvoid(host2)).thenReturn(false); + + when(capacityMgr.checkIfHostReachMaxGuestLimit(host2)).thenReturn(true); // Reached limit + + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 2, true, account); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testAllocateTo_GPUNotAvailable() { + List inputHosts = Arrays.asList(host1); + when(avoid.shouldAvoid(host1)).thenReturn(false); + when(resourceManagerMock.isGPUDeviceAvailable(offering, host1, virtualMachineProfile.getId())).thenReturn(false); + + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 1, true, account); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void allocateToTestHostTypeStorageShouldReturnNull() { + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsEmptyShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(emptyList).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsNotEmptyShouldCallAllocateToMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(VirtualMachineProfile.class), Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).allocateTo(virtualMachineProfile, deploymentPlan, serviceOffering, vmTemplateVO, excludeList, hosts, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity, account); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void allocateToTestProvidedHostsNotNullShouldCallAddHostsToAvoidSetMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(VirtualMachineProfile.class), Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndHaTagNotNullShouldReturnOnlyHostsWithHaTag() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNotNullAndHaTagNotNullShouldReturnOnlyHostsToFilterWithHaTag() { + List hostsToFilter = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, hostsToFilter, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagAndNoHostTagShouldReturnOnlyAllUpAndEnabledNonHaHosts() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List upAndEnabledHostsWithNoHa = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(upAndEnabledHostsWithNoHa).when(resourceManagerMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.nullable(String.class), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, null, null); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagWithHostTagShouldCallRetainHostsMatchingServiceOfferingAndTemplateTags() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void addHostsToAvoidSetTestAllHostsWereConsideredForAllocationShouldNotAddAnyHostToTheAvoidSet() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(suitableHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, suitableHosts); + + Assert.assertTrue(excludeList.getHostsToAvoid().isEmpty()); + } + + @Test + public void addHostsToAvoidSetTestNotAllHostsWereConsideredForAllocationShouldAddHostToTheAvoidSet() { + List allUpAndEnabledNonHAHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List consideredHosts = new ArrayList<>(Arrays.asList(host2, host3)); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(allUpAndEnabledNonHAHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, consideredHosts); + + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void filterHostsWithUefiEnabledTestNoDetailWithUefiShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + VMInstanceDetailVO userVmDetailVO = null; + + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithInvalidModeShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = "Invalid mode"; + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithLegacyModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = ApiConstants.BootMode.LEGACY.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithSecureModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = ApiConstants.BootMode.SECURE.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedButHostDoesNotHaveItShouldReturnTrue() { + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(false).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.any(ServiceOffering.class), Mockito.any(Host.class), Mockito.any(Long.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, virtualMachineProfile, excludeList, host1); + + Assert.assertTrue(result); + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedAndHostDoesHaveItShouldReturnFalse() { + Mockito.doReturn(true).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.any(ServiceOffering.class), Mockito.any(Host.class), Mockito.any(Long.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, virtualMachineProfile, excludeList, host1); + + Assert.assertFalse(result); + Mockito.verify(excludeList, Mockito.never()).addHost(Mockito.anyLong()); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateDoesNotRequireHvm() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateRequiresHvmShouldReturnOnlyHvmHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingRequestedVGpuViaDetailShouldDoNothing() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + ServiceOfferingDetailsVO requestedVGpuType = mock(ServiceOfferingDetailsVO.class); + + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, hosts); + + Assert.assertEquals(3, hosts.size()); + Assert.assertEquals(host1, hosts.get(0)); + Assert.assertEquals(host2, hosts.get(1)); + Assert.assertEquals(host3, hosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingRequestedVGpuViaProfileIdShouldDoNothing() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(1L).when(serviceOffering).getVgpuProfileId(); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, hosts); + + Assert.assertEquals(3, hosts.size()); + Assert.assertEquals(host1, hosts.get(0)); + Assert.assertEquals(host2, hosts.get(1)); + Assert.assertEquals(host3, hosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldReorderList() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(null).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(null).when(serviceOffering).getVgpuProfileId(); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(true).when(resourceManagerMock).isHostGpuEnabled(1L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(2L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(3L); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host2, allHosts.get(0)); + Assert.assertEquals(host3, allHosts.get(1)); + Assert.assertEquals(host1, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldNotReorderListIfThereIsNoHostWithVGpu() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(null).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(null).when(serviceOffering).getVgpuProfileId(); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(Mockito.anyLong()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host1, allHosts.get(0)); + Assert.assertEquals(host2, allHosts.get(1)); + Assert.assertEquals(host3, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateDidNotRequestedHvmShouldPutHostThatDoesNotSupportHvmInStartOfThePriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host2, prioritizedHosts.get(0)); + Assert.assertEquals(host1, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateRequiresHvmShouldNotReorderList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host1, prioritizedHosts.get(0)); + Assert.assertEquals(host2, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsWithMatchingGuestOsTestShouldPutMatchingHostInHighPriorityAndHostsThatDoesNotMatchInLowPriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List highPriorityHosts = new ArrayList<>(); + List lowPriorityHosts = new ArrayList<>(); + String guestOsCategory1 = "guestOsCategory1"; + String guestOsCategory2 = "guestOsCategory2"; + + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getTemplateGuestOSCategory(vmTemplateVO); + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getHostGuestOSCategory(host1); + Mockito.doReturn(guestOsCategory2).when(firstFitAllocatorSpy).getHostGuestOSCategory(host2); + Mockito.doReturn(null).when(firstFitAllocatorSpy).getHostGuestOSCategory(host3); + firstFitAllocatorSpy.prioritizeHostsWithMatchingGuestOs(vmTemplateVO,hostsToCheck, highPriorityHosts, lowPriorityHosts); + + Assert.assertEquals(1, highPriorityHosts.size()); + Assert.assertEquals(host1, highPriorityHosts.get(0)); + Assert.assertEquals(1, lowPriorityHosts.size()); + Assert.assertEquals(host2, lowPriorityHosts.get(0)); + } + + @Test + public void testHostByCombinedCapacityOrder() { + // Test scenario 1: Default capacity usage (0.5 weight) + List mockCapacity = getHostCapacities(); + Map hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.5); + + // Verify host ordering and capacity values + Long firstHostId = hostByCombinedCapacity.keySet().iterator().next(); + Assert.assertEquals("Host with ID 1 should be first in ordering", Long.valueOf(1L), firstHostId); + Assert.assertEquals("Host 1 combined capacity should match expected value", + 0.9609375, hostByCombinedCapacity.get(1L), TOLERANCE); + Assert.assertEquals("Host 2 combined capacity should match expected value", + 0.9296875, hostByCombinedCapacity.get(2L), TOLERANCE); + + // Test scenario 2: Modified capacity usage (0.7 weight) + when(mockCapacity.get(0).getUsedCapacity()).thenReturn(1500L); + hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.7); + + // Verify new ordering after capacity change + firstHostId = hostByCombinedCapacity.keySet().iterator().next(); + Assert.assertEquals("Host with ID 2 should be first after capacity change", Long.valueOf(2L), firstHostId); + Assert.assertEquals("Host 2 combined capacity should match expected value after change", + 0.9515625, hostByCombinedCapacity.get(2L), TOLERANCE); + Assert.assertEquals("Host 1 combined capacity should match expected value after change", + 0.9484375, hostByCombinedCapacity.get(1L), TOLERANCE); + } + + List getHostCapacities() { + CapacityVO cpuCapacity1 = mock(CapacityVO.class); + when(cpuCapacity1.getHostOrPoolId()).thenReturn(1L); + when(cpuCapacity1.getTotalCapacity()).thenReturn(32000L); + when(cpuCapacity1.getReservedCapacity()).thenReturn(0L); + when(cpuCapacity1.getUsedCapacity()).thenReturn(500L); + when(cpuCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + + CapacityVO cpuCapacity2 = mock(CapacityVO.class); + when(cpuCapacity2.getHostOrPoolId()).thenReturn(2L); + when(cpuCapacity2.getTotalCapacity()).thenReturn(32000L); + when(cpuCapacity2.getReservedCapacity()).thenReturn(0L); + when(cpuCapacity2.getUsedCapacity()).thenReturn(500L); + when(cpuCapacity2.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + + CapacityVO memCapacity1 = mock(CapacityVO.class); + when(memCapacity1.getHostOrPoolId()).thenReturn(1L); + when(memCapacity1.getTotalCapacity()).thenReturn(8589934592L); + when(memCapacity1.getReservedCapacity()).thenReturn(0L); + when(memCapacity1.getUsedCapacity()).thenReturn(536870912L); + when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + + CapacityVO memCapacity2 = mock(CapacityVO.class); + when(memCapacity2.getHostOrPoolId()).thenReturn(2L); + when(memCapacity2.getTotalCapacity()).thenReturn(8589934592L); + when(memCapacity2.getReservedCapacity()).thenReturn(0L); + when(memCapacity2.getUsedCapacity()).thenReturn(1073741824L); + when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + return Arrays.asList(cpuCapacity1, memCapacity1, cpuCapacity2, memCapacity2); + } } diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java new file mode 100644 index 00000000000..ab9b322eb84 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.vm.VirtualMachineProfile; +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; + +@RunWith(MockitoJUnitRunner.class) +public class RandomAllocatorTest { + + @Mock + HostDao hostDao; + + @Spy + @InjectMocks + RandomAllocator randomAllocator; + + @Mock + ResourceManager resourceManagerMock; + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 2L; + + private final Long zoneId = 3L; + + private final List emptyList = new ArrayList<>(); + + private final String hostTag = "hostTag"; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final VMTemplateVO vmTemplateVO = Mockito.mock(VMTemplateVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final DeploymentPlanner.ExcludeList excludeList = Mockito.mock(DeploymentPlanner.ExcludeList.class); + + private final VirtualMachineProfile virtualMachineProfile = Mockito.mock(VirtualMachineProfile.class); + + private final DeploymentPlan deploymentPlan = Mockito.mock(DeploymentPlan.class); + + private final boolean considerReservedCapacity = true; + + + @Test + public void testListHostsByTags() { + Host.Type type = Host.Type.Routing; + Long id = 1L; + String templateTag = "tag1"; + String offeringTag = "tag2"; + HostVO host1 = Mockito.mock(HostVO.class); + HostVO host2 = Mockito.mock(HostVO.class); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, offeringTag)).thenReturn(List.of(host1, host2)); + + // No template tagged host + ArrayList noTemplateTaggedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(new ArrayList<>()); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTaggedHosts, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(noTemplateTaggedHosts)); + + // Different template tagged host + ArrayList differentTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); + HostVO host3 = Mockito.mock(HostVO.class); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(List.of(host3)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(differentTemplateTaggedHost, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(differentTemplateTaggedHost)); + + // Matching template tagged host + ArrayList matchingTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(List.of(host1)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(matchingTemplateTaggedHost, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(matchingTemplateTaggedHost)); + Assert.assertEquals(1, matchingTemplateTaggedHost.size()); + + // No template tag + ArrayList noTemplateTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTag, type, zoneId, podId, clusterId, offeringTag, null); + Assert.assertFalse(CollectionUtils.isEmpty(noTemplateTag)); + Assert.assertEquals(2, noTemplateTag.size()); + + // No offering tag + ArrayList noOfferingTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noOfferingTag, type, zoneId, podId, clusterId, null, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(noOfferingTag)); + Assert.assertEquals(1, noOfferingTag.size()); + } + + @Test + public void findSuitableHostsTestHostTypeStorageShouldReturnNull() { + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestNoAvailableHostsShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(emptyList).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestAvailableHostsShouldCallFilterAvailableHostsOnce() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(hosts).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hosts).when(randomAllocator).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(randomAllocator, Mockito.times(1)).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestAvailableHostsReachedReturnUpToLimitShouldReturnOnlyHostsWithinLimit() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = 1; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestReturnUpToAllShouldReturnAllAvailableHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestHost1InAvoidShouldOnlyReturnHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(true).when(excludeList).shouldAvoid(host1); + Mockito.doReturn(false).when(excludeList).shouldAvoid(host2); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void filterAvailableHostsTestOnlyHost2HasCpuCapacityAndCapabilityShouldReturnOnlyHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(false).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host1); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host2); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullNotProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); + } +} diff --git a/test/integration/smoke/test_vm_strict_host_tags.py b/test/integration/smoke/test_vm_strict_host_tags.py index 2377e9a7618..aac3e1ea65f 100644 --- a/test/integration/smoke/test_vm_strict_host_tags.py +++ b/test/integration/smoke/test_vm_strict_host_tags.py @@ -423,7 +423,7 @@ class TestRestoreVMStrictTags(cloudstackTestCase): vm.restore(self.apiclient, templateid=self.template_t2.id, expunge=True) self.fail("VM should not be restored") except Exception as e: - self.assertTrue("Unable to start VM with specified id" in str(e)) + self.assertTrue("Unable to create a deployment for " in str(e)) class TestMigrateVMStrictTags(cloudstackTestCase): diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index aec999e1d00..f262b950f76 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -4191,6 +4191,7 @@ "message.warn.select.template": "Please select a Template for Registration.", "message.warn.zone.mtu.update": "Please note that this limit won't affect pre-existing Network's MTU settings", "message.webhook.deliveries.time.filter": "Webhook deliveries list can be filtered based on date-time. Select 'Custom' for specifying start and end date range.", +"message.webhook.filter.add": "Webhook deliveries can be controlled using filters (currently by Event type). Please select the parameters to add to the applied filters list.", "message.zone.creation.complete": "Zone creation complete.", "message.zone.detail.description": "Populate Zone details.", "message.zone.detail.hint": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 50b80d43cc6..94765968031 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -953,7 +953,7 @@ style="margin-left: 5px" :actions="actions" :resource="record" - :enabled="quickViewEnabled() && actions.length > 0" + :enabled="quickViewEnabled(actions, columns, column.key)" @exec-action="$parent.execAction" /> diff --git a/ui/src/components/view/WebhookFiltersTab.vue b/ui/src/components/view/WebhookFiltersTab.vue index 2d276e2a3cf..513720a8625 100644 --- a/ui/src/components/view/WebhookFiltersTab.vue +++ b/ui/src/components/view/WebhookFiltersTab.vue @@ -18,6 +18,7 @@