From 0b169920f34c815fc169d5a105f330949c8e5259 Mon Sep 17 00:00:00 2001 From: dahn Date: Mon, 27 Apr 2026 09:13:58 +0100 Subject: [PATCH 1/9] make dh group 31 default, support 22-24+31 (#12764) --- .../views/network/CreateVpnCustomerGateway.vue | 16 ++++++++++------ .../main/java/com/cloud/utils/net/NetUtils.java | 2 +- .../java/com/cloud/utils/net/NetUtilsTest.java | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ui/src/views/network/CreateVpnCustomerGateway.vue b/ui/src/views/network/CreateVpnCustomerGateway.vue index f71fc4709e8..bacc04cae31 100644 --- a/ui/src/views/network/CreateVpnCustomerGateway.vue +++ b/ui/src/views/network/CreateVpnCustomerGateway.vue @@ -258,9 +258,13 @@ export default { 'Group 15': 'modp3072', 'Group 16': 'modp4096', 'Group 17': 'modp6144', - 'Group 18': 'modp8192' + 'Group 18': 'modp8192', + 'Group 22': 'modp1024s160', + 'Group 23': 'modp2048s224', + 'Group 24': 'modp2048s256', + 'Group 31': 'curve25519' }, - ikeDhGroupInitialValue: 'Group 5(modp1536)', + ikeDhGroupInitialValue: 'Group 31(curve25519)', isSubmitted: false, ikeversion: 'ike' } @@ -275,12 +279,12 @@ export default { initForm () { this.formRef = ref() this.form = reactive({ - ikeEncryption: 'aes128', + ikeEncryption: 'aes256', ikeHash: 'sha1', ikeversion: 'ike', - ikeDh: 'Group 5(modp1536)', - espEncryption: 'aes128', - espHash: 'sha1', + ikeDh: 'Group 31(curve 25519)', + espEncryption: 'aes256', + espHash: 'sha256', perfectForwardSecrecy: 'None', ikelifetime: '86400', esplifetime: '3600', diff --git a/utils/src/main/java/com/cloud/utils/net/NetUtils.java b/utils/src/main/java/com/cloud/utils/net/NetUtils.java index 65878e055e7..d89d9fa2d93 100644 --- a/utils/src/main/java/com/cloud/utils/net/NetUtils.java +++ b/utils/src/main/java/com/cloud/utils/net/NetUtils.java @@ -1265,7 +1265,7 @@ public class NetUtils { if (group == null && policyType.toLowerCase().matches("ike")) { return false; // StrongSwan requires a DH group for the IKE policy } - if (group != null && !group.matches("modp1024|modp1536|modp2048|modp3072|modp4096|modp6144|modp8192")) { + if (group != null && !group.matches("modp1024|modp1536|modp2048|modp3072|modp4096|modp6144|modp8192|modp1024s160|modp2048s224|modp2048s256|curve25519")) { return false; } } diff --git a/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java b/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java index 4495a123b07..5c9d41f90a2 100644 --- a/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java @@ -131,6 +131,10 @@ public class NetUtilsTest { assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "3des-md5;modp1024")); assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha1;modp3072,aes128-sha1;modp1536")); assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha256;modp3072,aes128-sha512;modp1536")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;modp1024s160")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;modp2048s224")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;modp2048s256")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;curve25519")); assertFalse(NetUtils.isValidS2SVpnPolicy("ike", "aes128-sha1")); assertFalse(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha1")); assertFalse(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha1,aes256-sha1")); From ffebe8eaa64607e4294ee2d072e53b0347de5e9e Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 27 Apr 2026 15:38:52 +0530 Subject: [PATCH 2/9] Fix bulk power state query missing VM lifecycle state field (#13027) * Fix bulk power state query missing VM lifecycle state field The IdsPowerStateSelectSearch partial select did not include the VM lifecycle state, causing isPowerStateInSyncWithInstanceState to always return true when state was null. This prevented retry of failed StopCommands on subsequent ping cycles. * Add defensive check for instance host ID to prevent NPE Co-authored-by: Sachin R Doddaguni Co-authored-by: nvazquez --- .../main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index a38b6af3aa0..d8c9b9253c8 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -358,7 +358,8 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem IdsPowerStateSelectSearch.entity().getPowerHostId(), IdsPowerStateSelectSearch.entity().getPowerState(), IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(), - IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime()); + IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime(), + IdsPowerStateSelectSearch.entity().getState()); IdsPowerStateSelectSearch.done(); CountByOfferingId = createSearchBuilder(Integer.class); @@ -1105,10 +1106,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) { State instanceState = instance.getState(); + if (instanceState == null) { + logger.warn("VM {} has null instance state during power state sync check, treating as out of sync", instance); + return false; + } if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running) || (powerState == VirtualMachine.PowerState.PowerOn && instanceState == State.Stopped)) { HostVO instanceHost = hostDao.findById(instance.getHostId()); - HostVO powerHost = powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); + HostVO powerHost = instance.getHostId() != null && powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); logger.debug("VM: {} on host: {} and power host : {} is in {} state, but power state is {}", instance, instanceHost, powerHost, instanceState, powerState); return false; From 92d82989e327470c48d3ad2123a93011c0fb5ae3 Mon Sep 17 00:00:00 2001 From: Gean Jair Silva <89494158+GeanJS@users.noreply.github.com> Date: Thu, 30 Apr 2026 05:46:26 -0300 Subject: [PATCH 3/9] Correction of the user responsible for the event (#13066) Co-authored-by: gean.silva --- .../java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 9e7c644cf65..fad2da89cf2 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -1037,7 +1037,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(ownerId, ownerType, resourceType, tag); - ActionEventUtils.onActionEvent(caller.getId(), caller.getAccountId(), + Long callingUserId = CallContext.current().getCallingUserId(); + ActionEventUtils.onActionEvent(callingUserId, caller.getAccountId(), caller.getDomainId(), EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, "Resource limit updated. Resource Type: " + resourceType + ", New Value: " + max, ownerResourceId, ownerResourceType.toString()); From c45596cca3ec5d3c98fd3bdc1fb3bca53a3352a1 Mon Sep 17 00:00:00 2001 From: Bryan Lima <42067040+BryanMLima@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:30:02 +0200 Subject: [PATCH 4/9] Refactor of Allocator classes (#9074) * Refactoring Allocator classes * Break into smaller methods random and firfit allocators. * Added unit tests for random and firstfit allocators * Move random allocator from cloud-plugins to cloud-server * Add BaseAllocator abstract class for duplicate code * Add missing license * Add missing license to unit test file * Remove host allocator random dependency * Change exception message on smoke tests * Remove conditional as it was never actually reached in the original flow * Fix tests * Fix flipped parameters * Fix NPE while listing hosts for migration when suitableHosts is null * Remove unnecessary stubbings * Fix checkstyle * Remove unnecessary file * Rename exception error messages * Apply suggestions from code review Co-authored-by: Fabricio Duarte * Rename UserVmDetailVO references to VMInstanceDetailVO * Remove unused imports * Add new line at EOF * Remove unnecessary random allocator pom * Fix GPU allocation mistake * Fix failing tests --------- Co-authored-by: Fabricio Duarte Co-authored-by: Fabricio Duarte --- .../manager/allocator/HostAllocator.java | 33 - .../com/cloud/deploy/DeploymentPlanner.java | 2 +- .../admin/host/FindHostsForMigrationCmd.java | 2 +- .../api/command/admin/host/ListHostsCmd.java | 2 +- client/pom.xml | 5 - .../com/cloud/vm/VirtualMachineManager.java | 9 - .../cloud/vm/VirtualMachineManagerImpl.java | 18 - .../main/java/com/cloud/host/dao/HostDao.java | 2 +- .../java/com/cloud/host/dao/HostDaoImpl.java | 4 +- plugins/host-allocators/random/pom.xml | 30 - .../allocator/impl/RandomAllocator.java | 196 ----- .../host-allocator-random/module.properties | 18 - .../spring-host-allocator-random-context.xml | 34 - .../allocator/impl/RandomAllocatorTest.java | 80 -- plugins/pom.xml | 2 - .../manager/allocator/impl/BaseAllocator.java | 90 +++ .../allocator/impl/FirstFitAllocator.java | 660 +++++++---------- .../allocator/impl/RandomAllocator.java | 132 ++++ .../allocator/impl/TestingAllocator.java | 21 +- .../deploy/DeploymentPlanningManagerImpl.java | 22 +- .../cloud/server/ManagementServerImpl.java | 10 +- .../spring-server-allocator-context.xml | 3 + .../allocator/impl/BaseAllocatorTest.java | 219 ++++++ .../allocator/impl/FirstFitAllocatorTest.java | 690 ++++++++++++++---- .../allocator/impl/RandomAllocatorTest.java | 332 +++++++++ .../smoke/test_vm_strict_host_tags.py | 2 +- 26 files changed, 1615 insertions(+), 1003 deletions(-) delete mode 100644 plugins/host-allocators/random/pom.xml delete mode 100644 plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java delete mode 100644 plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties delete mode 100644 plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml delete mode 100644 plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java diff --git a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java index 604720aaa29..5d028d31d5b 100644 --- a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java @@ -22,19 +22,11 @@ import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.Adapter; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { - /** - * @param UserVm vm - * @param ServiceOffering offering - **/ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - /** * Determines which physical hosts are suitable to * allocate the guest virtual machines on @@ -49,31 +41,6 @@ public interface HostAllocator extends Adapter { public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); - /** - * Determines which physical hosts are suitable to allocate the guest - * virtual machines on - * - * Allocators must set any other hosts not considered for allocation in the - * ExcludeList avoid. Thus the avoid set and the list of hosts suitable, - * together must cover the entire host set in the cluster. - * - * @param VirtualMachineProfile - * vmProfile - * @param DeploymentPlan - * plan - * @param GuestType - * type - * @param ExcludeList - * avoid - * @param int returnUpTo (use -1 to return all possible hosts) - * @param boolean considerReservedCapacity (default should be true, set to - * false if host capacity calculation should not look at reserved - * capacity) - * @return List List of hosts that are suitable for VM allocation - **/ - - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity); - /** * Determines which physical hosts are suitable to allocate the guest * virtual machines on diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 8f7e773070f..22d796d4a77 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing; + random, firstfit, userdispersing, firstfitleastconsumed; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java index abca619f82a..4d6ef741961 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java @@ -78,7 +78,7 @@ public class FindHostsForMigrationCmd extends BaseListCmd { for (Host host : result.first()) { HostForMigrationResponse hostResponse = _responseGenerator.createHostForMigrationResponse(host); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index e202dfad77b..8f5e6c784d6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -252,7 +252,7 @@ public class ListHostsCmd extends BaseListCmd { for (Host host : result.first()) { HostResponse hostResponse = _responseGenerator.createHostResponse(host, getDetails()); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/client/pom.xml b/client/pom.xml index 7118f455ab5..55123de0f98 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -377,11 +377,6 @@ cloud-plugin-explicit-dedication ${project.version} - - org.apache.cloudstack - cloud-plugin-host-allocator-random - ${project.version} - org.apache.cloudstack cloud-plugin-outofbandmanagement-driver-ipmitool diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 57dc1b7bf72..e871bd8672f 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -181,15 +181,6 @@ public interface VirtualMachineManager extends Manager { void advanceReboot(String vmUuid, Map params) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; - /** - * Check to see if a virtual machine can be upgraded to the given service offering - * - * @param vm - * @param offering - * @return true if the host can handle the upgrade, false otherwise - */ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - VirtualMachine findById(long vmId); void storageMigration(String vmUuid, Map volumeToPool); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 71ecc73f325..17ddf870670 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -4017,19 +4017,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - @Override - public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { - boolean isMachineUpgradable = true; - for (final HostAllocator allocator : hostAllocators) { - isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (!isMachineUpgradable) { - break; - } - } - - return isMachineUpgradable; - } - @Override public void reboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ResourceUnavailableException { try { @@ -4469,11 +4456,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { - throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + - newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); - } - final List currentTags = StringUtils.csvTagsToList(currentDiskOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newDiskOffering.getTags()); if (VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(vmInstance.getDataCenterId())) { diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 090b019334f..5f5b2affee0 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -218,7 +218,7 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); - List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags); List findClustersThatMatchHostTagRule(String computeOfferingTags); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 2d8fcca6cdb..99c9a979c3b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -1520,7 +1520,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao } } - public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + public List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags) { List hostTagVOList = _hostTagsDao.findHostRuleTags(); List result = new ArrayList<>(); for (HostTagVO rule: hostTagVOList) { @@ -1534,7 +1534,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao public List findClustersThatMatchHostTagRule(String computeOfferingTags) { Set result = new HashSet<>(); - List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + List hosts = findHostsWithTagRuleThatMatchComputeOfferingTags(computeOfferingTags); for (HostVO host: hosts) { result.add(host.getClusterId()); } diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml deleted file mode 100644 index 061caf1573f..00000000000 --- a/plugins/host-allocators/random/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - cloud-plugin-host-allocator-random - Apache CloudStack Plugin - Host Allocator Random - - org.apache.cloudstack - cloudstack-plugins - 4.23.0.0-SNAPSHOT - ../../pom.xml - - diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java deleted file mode 100644 index 42129944a19..00000000000 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; - -@Component -public class RandomAllocator extends AdapterBase implements HostAllocator { - @Inject - private HostDao _hostDao; - @Inject - private ResourceManager _resourceMgr; - @Inject - private ClusterDao clusterDao; - @Inject - private ClusterDetailsDao clusterDetailsDao; - @Inject - private CapacityManager capacityManager; - - protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - List taggedHosts = new ArrayList<>(); - if (offeringHostTag != null) { - taggedHosts.addAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag)); - } - if (templateTag != null) { - List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - if (taggedHosts.isEmpty()) { - taggedHosts = templateTaggedHosts; - } else { - taggedHosts.retainAll(templateTaggedHosts); - } - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", - taggedHosts.size(), - (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::toString).toArray(), ","))), - type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); - } - return taggedHosts; - } - private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - List hostsCopy = null; - List suitableHosts = new ArrayList<>(); - - if (type == Host.Type.Storage) { - return suitableHosts; - } - String offeringHostTag = offering.getHostTag(); - - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - String templateTag = template.getTemplateTag(); - String hostTag = null; - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostTag = ObjectUtils.allNotNull(offeringHostTag, templateTag) ? - String.format("%s, %s", offeringHostTag, templateTag) : - ObjectUtils.firstNonNull(offeringHostTag, templateTag); - logger.debug("Looking for hosts in dc [{}], pod [{}], cluster [{}] and complying with host tag(s): [{}]", dcId, podId, clusterId, hostTag); - } else { - logger.debug("Looking for hosts in dc: {} pod: {} cluster: {}", dcId , podId, clusterId); - } - if (hosts != null) { - // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList<>(hosts); - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); - } else { - hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); - } - } else { - // list all computing hosts, regardless of whether they support routing...it's random after all - if (offeringHostTag != null) { - hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); - } else { - hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); - } - } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); - - if (hostsCopy.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); - return null; - } - - logger.debug("Random Allocator found {} hosts", hostsCopy.size()); - if (hostsCopy.isEmpty()) { - return suitableHosts; - } - - Collections.shuffle(hostsCopy); - for (Host host : hostsCopy) { - if (suitableHosts.size() == returnUpTo) { - break; - } - if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Host %s is in avoid set, skipping this and trying other available hosts", host)); - } - continue; - } - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Not using host %s; host has cpu capability? %s, host has capacity? %s", host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second())); - } - continue; - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found a suitable host, adding to list: %s", host)); - } - suitableHosts.add(host); - } - if (logger.isDebugEnabled()) { - logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - if (CollectionUtils.isEmpty(hosts)) { - if (logger.isDebugEnabled()) { - logger.debug("Random Allocator found 0 hosts as given host list is empty"); - } - return new ArrayList<>(); - } - return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, - Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity); - } - - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } -} diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties deleted file mode 100644 index dcfe8d3537f..00000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -name=host-allocator-random -parent=allocator diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml deleted file mode 100644 index d84eaafaa5a..00000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java deleted file mode 100644 index 538d7157184..00000000000 --- a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.collections.CollectionUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; - -@RunWith(MockitoJUnitRunner.class) -public class RandomAllocatorTest { - - @Mock - HostDao hostDao; - @InjectMocks - RandomAllocator randomAllocator; - - @Test - public void testListHostsByTags() { - Host.Type type = Host.Type.Routing; - Long id = 1L; - String templateTag = "tag1"; - String offeringTag = "tag2"; - HostVO host1 = Mockito.mock(HostVO.class); - HostVO host2 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, offeringTag)).thenReturn(List.of(host1, host2)); - - // No template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - List result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Different template tagged host - HostVO host3 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Matching template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - - // No template tag - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, null); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(2, result.size()); - - // No offering tag - result = randomAllocator.listHostsByTags(type, id, id, id, null, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - } -} diff --git a/plugins/pom.xml b/plugins/pom.xml index e4904ccdf40..3262404032f 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -84,8 +84,6 @@ ha-planners/skip-heurestics - host-allocators/random - hypervisors/baremetal hypervisors/external hypervisors/hyperv diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java new file mode 100644 index 00000000000..58fcc62cdc3 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.capacity.CapacityManager; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import org.apache.commons.collections.CollectionUtils; + +import javax.inject.Inject; +import java.util.List; + +public abstract class BaseAllocator extends AdapterBase implements HostAllocator { + + @Inject + protected HostDao hostDao; + + @Inject + protected CapacityManager capacityManager; + + protected void retainHostsMatchingServiceOfferingAndTemplateTags(List availableHosts, Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { + logger.debug("Hosts {} will be checked for template and host tags compatibility.", availableHosts); + + if (offeringHostTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", offeringHostTag); + List hostsWithHostTag = hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag); + logger.debug("Retaining hosts {} because they match the offering host tag {}.", hostsWithHostTag, offeringHostTag); + availableHosts.retainAll(hostsWithHostTag); + } + + if (templateTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Template.", templateTag); + List hostsWithTemplateTag = hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); + logger.debug("Retaining hosts {} because they match the template tag {}.", hostsWithTemplateTag, templateTag); + availableHosts.retainAll(hostsWithTemplateTag); + } + + logger.debug("Remaining hosts after template tag and host tags validations are {}.", availableHosts); + } + + protected void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { + List hostsWithTagRules = hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTagOnOffering); + + if (CollectionUtils.isEmpty(hostsWithTagRules)) { + logger.info("No hosts found with tag rules matching the compute offering tag [{}].", hostTagOnOffering); + return; + } + + logger.info("Found hosts {} with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); + clusterHosts.addAll(hostsWithTagRules); + } + + /** + * Adds hosts with enough CPU capability and enough CPU capacity to the suitable hosts list. + */ + protected boolean hostHasCpuCapabilityAndCapacity(boolean considerReservedCapacity, ServiceOffering offering, Host host) { + logger.debug("Looking for CPU frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), offering::getRamSize); + Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); + Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); + Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); + + if (hasCpuCapability && hasCpuCapacity) { + logger.debug("Host {} is a suitable host as it has enough CPU capability and CPU capacity.", () -> host); + return true; + } + + logger.debug("Cannot use host {}. Does the host have CPU capability? {}. Does the host have CPU capacity? {}..", () -> host, () -> hasCpuCapability, () -> hasCpuCapacity); + return false; + } + +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 1110804959e..8b4ca318bf7 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -16,6 +16,41 @@ // under the License. package com.cloud.agent.manager.allocator.impl; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.firstfitleastconsumed; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.random; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.userdispersing; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.configuration.Config; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.deploy.DeploymentClusterPlanner; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.deploy.FirstFitPlanner; +import com.cloud.gpu.GPU; +import com.cloud.host.DetailVO; +import com.cloud.host.Host; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDetailsDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.vm.VMInstanceDetailVO; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; + import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; @@ -29,59 +64,19 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.gpu.dao.VgpuProfileDao; - +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.Capacity; -import com.cloud.capacity.CapacityManager; -import com.cloud.capacity.CapacityVO; -import com.cloud.capacity.dao.CapacityDao; -import com.cloud.configuration.Config; -import com.cloud.configuration.ConfigurationManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentClusterPlanner; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.deploy.FirstFitPlanner; -import com.cloud.gpu.GPU; -import com.cloud.host.DetailVO; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.host.dao.HostDetailsDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.service.ServiceOfferingDetailsVO; -import com.cloud.service.dao.ServiceOfferingDetailsDao; -import com.cloud.storage.GuestOSCategoryVO; -import com.cloud.storage.GuestOSVO; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.dao.GuestOSCategoryDao; -import com.cloud.storage.dao.GuestOSDao; -import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VMInstanceDetailVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.dao.VMInstanceDetailsDao; -import com.cloud.vm.dao.VMInstanceDao; - +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.jetbrains.annotations.NotNull; +import org.springframework.stereotype.Component; /** * An allocator that tries to find a fit on a computing host. This allocator does not care whether or not the host supports routing. */ @Component -public class FirstFitAllocator extends AdapterBase implements HostAllocator { - @Inject - protected HostDao _hostDao = null; +public class FirstFitAllocator extends BaseAllocator { @Inject HostDetailsDao _hostDetailsDao = null; @Inject @@ -95,289 +90,183 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { @Inject protected ResourceManager _resourceMgr; @Inject - ClusterDao _clusterDao; - @Inject - ClusterDetailsDao _clusterDetailsDao; - @Inject ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @Inject - CapacityManager _capacityMgr; - @Inject CapacityDao _capacityDao; @Inject - VMInstanceDetailsDao _vmInstanceDetailsDao; - @Inject - private VgpuProfileDao vgpuProfileDao; + VMInstanceDetailsDao vmInstanceDetailsDao; boolean _checkHvm = true; + static DecimalFormat decimalFormat = new DecimalFormat("#.##"); @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - Account account = vmProfile.getOwner(); - - boolean isVMDeployedWithUefi = false; - VMInstanceDetailVO vmInstanceDetailVO = _vmInstanceDetailsDao.findDetail(vmProfile.getId(), "UEFI"); - if(vmInstanceDetailVO != null){ - if ("secure".equalsIgnoreCase(vmInstanceDetailVO.getValue()) || "legacy".equalsIgnoreCase(vmInstanceDetailVO.getValue())) { - isVMDeployedWithUefi = true; - } - } - logger.info(" Guest VM is requested with Custom[UEFI] Boot Type "+ isVMDeployedWithUefi); - - - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of routing or not - return new ArrayList<>(); - } - String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - logger.debug("Looking for hosts in {}", paramAsStringToLog); - - String hostTagOnOffering = offering.getHostTag(); - String hostTagOnTemplate = template.getTemplateTag(); - String hostTagUefi = "UEFI"; - - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; - - List clusterHosts = new ArrayList<>(); - List hostsMatchingUefiTag = new ArrayList<>(); - if(isVMDeployedWithUefi){ - hostsMatchingUefiTag = _hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagUefi + "' are:" + hostsMatchingUefiTag); - } - } - - - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - if (haVmTag != null) { - clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag); - } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - clusterHosts = _resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId); - } else { - List hostsMatchingOfferingTag = new ArrayList<>(); - List hostsMatchingTemplateTag = new ArrayList<>(); - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsMatchingOfferingTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsMatchingOfferingTag); - } - } - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } - hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsMatchingTemplateTag); - } - } - - if (hasSvcOfferingTag && hasTemplateTag) { - hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - if (logger.isDebugEnabled()) { - logger.debug("Found " + hostsMatchingOfferingTag.size() + " Hosts satisfying both tags, host ids are:" + hostsMatchingOfferingTag); - } - - clusterHosts = hostsMatchingOfferingTag; - } else { - if (hasSvcOfferingTag) { - clusterHosts = hostsMatchingOfferingTag; - } else { - clusterHosts = hostsMatchingTemplateTag; - } - } - } - } - - if (isVMDeployedWithUefi) { - clusterHosts.retainAll(hostsMatchingUefiTag); - } - - clusterHosts.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); - - - if (clusterHosts.isEmpty()) { - logger.warn("No suitable host found for VM [{}] with tags {} in {}.", vmProfile, hostTagOnOffering, paramAsStringToLog); - return null; - } - // add all hosts that we are not considering to the avoid list - List allhostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); - allhostsInCluster.removeAll(clusterHosts); - - logger.debug(() -> String.format("Adding hosts [%s] to the avoid set because these hosts do not support HA.", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allhostsInCluster, "uuid", "name"))); - - for (HostVO host : allhostsInCluster) { - avoid.addHost(host.getId()); - } - - return allocateTo(vmProfile, plan, offering, template, avoid, clusterHosts, returnUpTo, considerReservedCapacity, account); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { + boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); Account account = vmProfile.getOwner(); - List suitableHosts = new ArrayList<>(); - List hostsCopy = new ArrayList<>(hosts); - - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of - // routing or not. - return suitableHosts; - } String hostTagOnOffering = offering.getHostTag(); String hostTagOnTemplate = template.getTemplateTag(); - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; + String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - if (haVmTag != null) { - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + List suitableHosts = retrieveHosts(vmProfile, type, (List) hosts, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); + + if (suitableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in {}.", vmProfile, paramAsStringToLog); + return null; + } + + if (CollectionUtils.isEmpty(hosts)) { + addHostsToAvoidSet(type, avoid, clusterId, podId, dcId, suitableHosts); + } + + return allocateTo(vmProfile, plan, offering, template, avoid, suitableHosts, returnUpTo, considerReservedCapacity, account); + } + + protected List retrieveHosts(VirtualMachineProfile vmProfile, Type type, List hostsToFilter, Long clusterId, Long podId, long dcId, String hostTagOnOffering, + String hostTagOnTemplate) { + String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + List clusterHosts; + + if (CollectionUtils.isNotEmpty(hostsToFilter)) { + clusterHosts = new ArrayList<>(hostsToFilter); } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); - } else { - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); - - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsCopy); - } - } - - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } - - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); - - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsCopy); - } - } - } + clusterHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); } - hostsCopy.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); - - if (!hostsCopy.isEmpty()) { - suitableHosts = allocateTo(vmProfile, plan, offering, template, avoid, hostsCopy, returnUpTo, considerReservedCapacity, account); + if (haVmTag != null) { + clusterHosts.retainAll(hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { + clusterHosts.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); + } else { + retainHostsMatchingServiceOfferingAndTemplateTags(clusterHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); } + filterHostsWithUefiEnabled(type, vmProfile, clusterId, podId, dcId, clusterHosts); + + addHostsBasedOnTagRules(hostTagOnOffering, clusterHosts); + + return clusterHosts; + + } + + /** + * Add all hosts to the avoid set that were not considered during the allocation + */ + protected void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Long podId, long dcId, List suitableHosts) { + List allHostsInCluster = hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); + + allHostsInCluster.removeAll(suitableHosts); + + logger.debug("Adding hosts [{}] to the avoid set because these hosts were not considered for allocation.", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allHostsInCluster, "uuid", "name")); + + for (HostVO host : allHostsInCluster) { + avoid.addHost(host.getId()); + } + } + + protected void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmProfile, Long clusterId, Long podId, long dcId, List clusterHosts) { + VMInstanceDetailVO vmInstanceDetailVO = vmInstanceDetailsDao.findDetail(vmProfile.getId(), "UEFI"); + + if (vmInstanceDetailVO == null) { + return; + } + + if (!StringUtils.equalsAnyIgnoreCase(vmInstanceDetailVO.getValue(), ApiConstants.BootMode.SECURE.toString(), ApiConstants.BootMode.LEGACY.toString())) { + return; + } + + logger.info("Guest VM is requested with Custom[UEFI] Boot Type enabled."); + + List hostsMatchingUefiTag = hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); + + logger.debug("Hosts with UEFI enabled are {}.", hostsMatchingUefiTag); + clusterHosts.retainAll(hostsMatchingUefiTag); + } + + protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity, Account account) { + String vmAllocationAlgorithm = DeploymentClusterPlanner.VmAllocationAlgorithm.value(); + if (random.toString().equals(vmAllocationAlgorithm)) { + Collections.shuffle(hosts); + } else if (userdispersing.toString().equals(vmAllocationAlgorithm)) { + hosts = reorderHostsByNumberOfVms(plan, hosts, account); + } else if (firstfitleastconsumed.toString().equals(vmAllocationAlgorithm)) { + hosts = reorderHostsByCapacity(plan, hosts); + } + + logger.debug("FirstFitAllocator has {} hosts to check for allocation {}.", hosts.size(), hosts); + hosts = prioritizeHosts(template, offering, hosts); + logger.debug("Found {} hosts for allocation after prioritization: {}.", hosts.size(), hosts); + + List suitableHosts = checkHostsCompatibilities(offering, vmProfile, avoid, hosts, returnUpTo, considerReservedCapacity); + logger.debug("Host Allocator returning {} suitable hosts.", suitableHosts.size()); + return suitableHosts; } - protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity, Account account) { - String vmAllocationAlgorithm = DeploymentClusterPlanner.VmAllocationAlgorithm.value(); - if (vmAllocationAlgorithm.equals("random")) { - // Shuffle this so that we don't check the hosts in the same order. - Collections.shuffle(hosts); - } else if (vmAllocationAlgorithm.equals("userdispersing")) { - hosts = reorderHostsByNumberOfVms(plan, hosts, account); - } else if(vmAllocationAlgorithm.equals("firstfitleastconsumed")){ - hosts = reorderHostsByCapacity(plan, hosts); - } - - if (logger.isDebugEnabled()) { - logger.debug("FirstFitAllocator has " + hosts.size() + " hosts to check for allocation: " + hosts); - } - - // We will try to reorder the host lists such that we give priority to hosts that have - // the minimums to support a VM's requirements - hosts = prioritizeHosts(template, offering, hosts); - - if (logger.isDebugEnabled()) { - logger.debug("Found " + hosts.size() + " hosts for allocation after prioritization: " + hosts); - } - - if (logger.isDebugEnabled()) { - logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize() + " MB"); - } - - long serviceOfferingId = offering.getId(); + protected List checkHostsCompatibilities(ServiceOffering offering, VirtualMachineProfile vmProfile, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { List suitableHosts = new ArrayList<>(); - ServiceOfferingDetailsVO offeringDetails = null; + logger.debug("Checking compatibility for the following hosts {}.", suitableHosts); for (Host host : hosts) { if (suitableHosts.size() == returnUpTo) { break; } + if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug("Host: {} is in avoid set, skipping this and trying other available hosts", host); - } + logger.debug("Host [{}] is in avoid set, skipping this and trying other available hosts", host); continue; } - //find number of guest VMs occupying capacity on this host. - if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { + if (capacityManager.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Adding host [{}] to the avoid set because this host already has the max number of running (user and/or system) VMs.", host); avoid.addHost(host.getId()); continue; } - // Check if GPU device is required by offering and host has the availability - if (_resourceMgr.isGPUDeviceAvailable(offering, host, vmProfile.getId())) { - logger.debug("Host [{}] has required GPU devices available.", host); - } else { - // If GPU is not available, skip this host - logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", host); - avoid.addHost(host.getId()); + if (offeringRequestedVGpuAndHostDoesNotHaveIt(offering, vmProfile, avoid, host)) { continue; } - Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (cpuCapabilityAndCapacity.first() && cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug("Found a suitable host, adding to list: {}", host); - } + if (hostHasCpuCapabilityAndCapacity(considerReservedCapacity, offering, host)) { suitableHosts.add(host); - } else { - if (logger.isDebugEnabled()) { - logger.debug("Not using host {}; host has cpu capability? {}, host has capacity?{}", - host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second()); - } - avoid.addHost(host.getId()); + continue; } + avoid.addHost(host.getId()); } - - if (logger.isDebugEnabled()) { - logger.debug("Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; } - // Reorder hosts in the decreasing order of free capacity. + protected boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offering, VirtualMachineProfile vmProfile, ExcludeList avoid, Host host) { + if (_resourceMgr.isGPUDeviceAvailable(offering, host, vmProfile.getId())) { + logger.debug("Host [{}] has required GPU devices available.", () -> host); + return false; + } + + logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", () -> host); + avoid.addHost(host.getId()); + return true; + } + + /** + * Reorder hosts in the decreasing order of free capacity. + */ private List reorderHostsByCapacity(DeploymentPlan plan, List hosts) { Long zoneId = plan.getDataCenterId(); Long clusterId = plan.getClusterId(); @@ -388,26 +277,10 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) .collect(Collectors.toMap(Map.Entry::getKey, entry -> decimalFormat.format(entry.getValue() * 100) + "%", (e1, e2) -> e1, LinkedHashMap::new)); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts: [{}] in descending order of free capacity (percentage) in the cluster: {}", - hostIdsByFreeCapacity, sortedHostByCapacity); - } + logger.debug("List of hosts: [{}] in descending order of free capacity (percentage) in the cluster: {}.", + hostIdsByFreeCapacity, sortedHostByCapacity); - //now filter the given list of Hosts by this ordered list - Map hostMap = new HashMap<>(); - for (Host host : hosts) { - hostMap.put(host.getId(), host); - } - List matchingHostIds = new ArrayList<>(hostMap.keySet()); - - hostIdsByFreeCapacity.retainAll(matchingHostIds); - - List reorderedHosts = new ArrayList<>(); - for(Long id: hostIdsByFreeCapacity){ - reorderedHosts.add(hostMap.get(id)); - } - - return reorderedHosts; + return filterHosts(hosts, hostIdsByFreeCapacity); } private Pair, Map> getOrderedHostsByCapacity(Long zoneId, Long clusterId) { @@ -450,116 +323,154 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { Long clusterId = plan.getClusterId(); List hostIdsByVmCount = _vmInstanceDao.listHostIdsByVmCount(dcId, podId, clusterId, account.getAccountId()); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts in ascending order of number of VMs: " + hostIdsByVmCount); - } + logger.debug("List of hosts in ascending order of number of VMs: {}.", hostIdsByVmCount); - //now filter the given list of Hosts by this ordered list + return filterHosts(hosts, hostIdsByVmCount); + } + + /** + * Filter the given list of Hosts considering the ordered list + */ + private List filterHosts(List hosts, List orderedHostIdsList) { Map hostMap = new HashMap<>(); + for (Host host : hosts) { hostMap.put(host.getId(), host); } List matchingHostIds = new ArrayList<>(hostMap.keySet()); - - hostIdsByVmCount.retainAll(matchingHostIds); + orderedHostIdsList.retainAll(matchingHostIds); List reorderedHosts = new ArrayList<>(); - for (Long id : hostIdsByVmCount) { + for(Long id: orderedHostIdsList){ reorderedHosts.add(hostMap.get(id)); } return reorderedHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - + /** + * Reorder the host list giving priority to hosts that have the minimum to support the VM's requirements. + */ protected List prioritizeHosts(VMTemplateVO template, ServiceOffering offering, List hosts) { if (template == null) { return hosts; } - // Determine the guest OS category of the template - String templateGuestOSCategory = getTemplateGuestOSCategory(template); + List hostsToCheck = filterHostWithNoHvmIfTemplateRequested(template, hosts); List prioritizedHosts = new ArrayList<>(); - List noHvmHosts = new ArrayList<>(); - - // If a template requires HVM and a host doesn't support HVM, remove it from consideration - List hostsToCheck = new ArrayList<>(); - if (template.isRequiresHvm()) { - for (Host host : hosts) { - if (hostSupportsHVM(host)) { - hostsToCheck.add(host); - } else { - noHvmHosts.add(host); - } - } - } else { - hostsToCheck.addAll(hosts); - } - - if (logger.isDebugEnabled()) { - if (noHvmHosts.size() > 0) { - logger.debug("Not considering hosts: " + noHvmHosts + " to deploy template: " + template + " as they are not HVM enabled"); - } - } - // If a host is tagged with the same guest OS category as the template, move it to a high priority list - // If a host is tagged with a different guest OS category than the template, move it to a low priority list List highPriorityHosts = new ArrayList<>(); List lowPriorityHosts = new ArrayList<>(); - for (Host host : hostsToCheck) { - String hostGuestOSCategory = getHostGuestOSCategory(host); - if (hostGuestOSCategory == null) { - continue; - } else if (templateGuestOSCategory != null && templateGuestOSCategory.equals(hostGuestOSCategory)) { - highPriorityHosts.add(host); - } else { - lowPriorityHosts.add(host); - } - } + prioritizeHostsWithMatchingGuestOs(template, hostsToCheck, highPriorityHosts, lowPriorityHosts); hostsToCheck.removeAll(highPriorityHosts); hostsToCheck.removeAll(lowPriorityHosts); - // Prioritize the remaining hosts by HVM capability - for (Host host : hostsToCheck) { - if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { - // Host and template both do not support hvm, put it as first consideration - prioritizedHosts.add(0, host); - } else { - // Template doesn't require hvm, but the machine supports it, make it last for consideration - prioritizedHosts.add(host); - } - } - - // Merge the lists + prioritizeHostsByHvmCapability(template, hostsToCheck, prioritizedHosts); prioritizedHosts.addAll(0, highPriorityHosts); prioritizedHosts.addAll(lowPriorityHosts); - // if service offering is not GPU enabled then move all the GPU enabled hosts to the end of priority list. - if (_serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null && offering.getVgpuProfileId() == null) { + prioritizeHostsByGpuEnabled(offering, prioritizedHosts); - List gpuEnabledHosts = new ArrayList<>(); - // Check for GPU enabled hosts. - for (Host host : prioritizedHosts) { - if (_resourceMgr.isHostGpuEnabled(host.getId())) { - gpuEnabledHosts.add(host); - } - } - // Move GPU enabled hosts to the end of list - if(!gpuEnabledHosts.isEmpty()) { - prioritizedHosts.removeAll(gpuEnabledHosts); - prioritizedHosts.addAll(gpuEnabledHosts); - } - } return prioritizedHosts; } + + /** + * If a template requires HVM and a host doesn't support HVM, remove it from consideration. + */ + protected List filterHostWithNoHvmIfTemplateRequested(VMTemplateVO template, List hosts) { + List hostsToCheck = new ArrayList<>(); + + if (!template.isRequiresHvm()) { + logger.debug("Template [{}] does not require HVM, therefore, the hosts {} will not be checked for HVM compatibility.", template, hostsToCheck); + hostsToCheck.addAll(hosts); + return hostsToCheck; + } + + List noHvmHosts = new ArrayList<>(); + logger.debug("Template [{}] requires HVM, therefore, the hosts %s will be checked for HVM compatibility.", template, hostsToCheck); + + for (Host host : hosts) { + if (hostSupportsHVM(host)) { + hostsToCheck.add(host); + } else { + noHvmHosts.add(host); + } + } + + if (!noHvmHosts.isEmpty()) { + logger.debug("Not considering hosts {} to deploy VM using template {} as they are not HVM enabled.", noHvmHosts, template); + } + + return hostsToCheck; + } + + + /** + * If service offering did not request for vGPU, then move all host with GPU to the end of the host priority list. + */ + protected void prioritizeHostsByGpuEnabled(ServiceOffering offering, List prioritizedHosts) { + boolean serviceOfferingRequestedVGpu = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) != null || offering.getVgpuProfileId() != null; + + if (serviceOfferingRequestedVGpu) { + return; + } + + List gpuEnabledHosts = new ArrayList<>(); + + for (Host host : prioritizedHosts) { + if (_resourceMgr.isHostGpuEnabled(host.getId())) { + gpuEnabledHosts.add(host); + } + } + + if (!gpuEnabledHosts.isEmpty()) { + prioritizedHosts.removeAll(gpuEnabledHosts); + prioritizedHosts.addAll(gpuEnabledHosts); + } + } + + /** + * Prioritize remaining host by HVM capability. + * + *
    + *
  • If host and template both do not support HVM, put it at the start of the list.
    • + *
  • If the template doesn't require HVM, but the machine supports it, append it to the list.
    • + *
+ */ + protected void prioritizeHostsByHvmCapability(VMTemplateVO template, List hostsToCheck, List prioritizedHosts) { + for (Host host : hostsToCheck) { + if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { + prioritizedHosts.add(0, host); + } else { + prioritizedHosts.add(host); + } + } + } + + + /** + *
    + *
  • If a host is tagged with the same guest OS category as the template, move it to a high priority list.
    • + *
  • If a host is tagged with a different guest OS category than the template, move it to a low priority list.
    • + *
+ */ + protected void prioritizeHostsWithMatchingGuestOs(VMTemplateVO template, List hostsToCheck, List highPriorityHosts, List lowPriorityHosts) { + String templateGuestOSCategory = getTemplateGuestOSCategory(template); + + for (Host host : hostsToCheck) { + String hostGuestOSCategory = getHostGuestOSCategory(host); + + if (StringUtils.equals(templateGuestOSCategory, hostGuestOSCategory)) { + highPriorityHosts.add(host); + } else if (hostGuestOSCategory != null) { + lowPriorityHosts.add(host); + } + } + } + + protected boolean hostSupportsHVM(Host host) { if (!_checkHvm) { return true; @@ -621,19 +532,8 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (_configDao != null) { Map configs = _configDao.getConfiguration(params); String value = configs.get("xenserver.check.hvm"); - _checkHvm = value == null ? true : Boolean.parseBoolean(value); + _checkHvm = value == null || Boolean.parseBoolean(value); } return true; } - - @Override - public boolean start() { - return true; - } - - @Override - public boolean stop() { - return true; - } - } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java new file mode 100644 index 00000000000..96b4255b876 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.host.Host; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.VMTemplateVO; +import com.cloud.vm.VirtualMachineProfile; + +@Component +public class RandomAllocator extends BaseAllocator { + @Inject + private ResourceManager _resourceMgr; + + protected List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + + long dcId = plan.getDataCenterId(); + Long podId = plan.getPodId(); + Long clusterId = plan.getClusterId(); + ServiceOffering offering = vmProfile.getServiceOffering(); + + String offeringHostTag = offering.getHostTag(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); + logger.debug("Looking for hosts in zone [{}], pod [{}], cluster [{}].", dcId, podId, clusterId); + + List availableHosts = retrieveHosts(type, (List) hosts, template, offeringHostTag, clusterId, podId, dcId); + + if (availableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in zone [{}], pod [{}], cluster [{}].", vmProfile, dcId, podId, clusterId); + return null; + } + + return filterAvailableHosts(avoid, returnUpTo, considerReservedCapacity, availableHosts, offering); + } + + protected List filterAvailableHosts(ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity, List availableHosts, ServiceOffering offering) { + logger.debug("Random Allocator found [{}] available hosts. They will be checked if they are in the avoid set and for CPU capability and capacity.", availableHosts::size); + List suitableHosts = new ArrayList<>(); + + Collections.shuffle(availableHosts); + for (Host host : availableHosts) { + if (suitableHosts.size() == returnUpTo) { + break; + } + + if (avoid.shouldAvoid(host)) { + logger.debug("Host [{}] is in the avoid set, skipping it and trying other available hosts.", () -> host); + continue; + } + + if (!hostHasCpuCapabilityAndCapacity(considerReservedCapacity, offering, host)) { + continue; + } + + logger.debug("Found the suitable host [{}], adding to list.", () -> host); + suitableHosts.add(host); + } + + logger.debug("Random Host Allocator returning {} suitable hosts.", suitableHosts::size); + return suitableHosts; + } + + /** + * @return all computing hosts, regardless of whether they support routing. + */ + protected List retrieveHosts(Type type, List hosts, VMTemplateVO template, String offeringHostTag, Long clusterId, Long podId, long dcId) { + List availableHosts; + String templateTag = template.getTemplateTag(); + + if (CollectionUtils.isNotEmpty(hosts)) { + availableHosts = new ArrayList<>(hosts); + } else { + availableHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + } + + if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { + retainHostsMatchingServiceOfferingAndTemplateTags(availableHosts, type, dcId, podId, clusterId, offeringHostTag, templateTag); + } else { + List hostsWithNoRuleTag = hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); + logger.debug("Retaining hosts {} because they do not have rule tags.", hostsWithNoRuleTag); + availableHosts.retainAll(hostsWithNoRuleTag); + } + + addHostsBasedOnTagRules(offeringHostTag, availableHosts); + + return availableHosts; + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity) { + return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); + } +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java index fd8e65e2476..04a0b182b7b 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java @@ -28,32 +28,24 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public class TestingAllocator extends AdapterBase implements HostAllocator { @Inject HostDao _hostDao; - Long _computingHost; Long _storageHost; Long _routingHost; @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - List availableHosts = new ArrayList(); + boolean considerReservedCapacity) { + List availableHosts = new ArrayList<>(); Host host = null; if (type == Host.Type.Routing && _routingHost != null) { host = _hostDao.findById(_routingHost); @@ -66,13 +58,6 @@ public class TestingAllocator extends AdapterBase implements HostAllocator { return availableHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - @Override public boolean configure(String name, Map params) { String value = (String)params.get(Host.Type.Routing.toString()); diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index f163a3d52a5..7fb4a97e149 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -20,7 +20,6 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -1714,20 +1713,19 @@ StateListener, Configurable { @Override public void reorderHostsByPriority(Map priorities, List hosts) { - logger.debug("Re-ordering hosts {} by priorities {}", hosts, priorities); + if (CollectionUtils.isEmpty(hosts)){ + logger.debug("Hosts list is empty; therefore, there is nothing to reorder."); + return; + } + logger.info("Re-ordering hosts [{}] by priorities [{}].", hosts, priorities); hosts.removeIf(host -> DataCenterDeployment.PROHIBITED_HOST_PRIORITY.equals(getHostPriority(priorities, host.getId()))); + hosts.sort((host1, host2) -> { + int res = getHostPriority(priorities, host1.getId()).compareTo(getHostPriority(priorities, host2.getId())); + return -res; + }); - Collections.sort(hosts, new Comparator<>() { - @Override - public int compare(Host host1, Host host2) { - int res = getHostPriority(priorities, host1.getId()).compareTo(getHostPriority(priorities, host2.getId())); - return -res; - } - } - ); - - logger.debug("Hosts after re-ordering are: {}", hosts); + logger.info("Hosts after re-ordering are: [{}].", hosts); } private Integer getHostPriority(Map priorities, Long hostId) { diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index d4266496e98..337bdbbbc63 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1696,11 +1696,7 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen List suitableHosts = new ArrayList<>(); for (final HostAllocator allocator : hostAllocators) { - if (CollectionUtils.isNotEmpty(compatibleHosts)) { - suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, compatibleHosts, HostAllocator.RETURN_UPTO_ALL, false); - } else { - suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, HostAllocator.RETURN_UPTO_ALL, false); - } + suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, compatibleHosts, HostAllocator.RETURN_UPTO_ALL, false); if (CollectionUtils.isNotEmpty(suitableHosts)) { break; @@ -1709,10 +1705,10 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen _dpMgr.reorderHostsByPriority(plan.getHostPriorities(), suitableHosts); - if (suitableHosts.isEmpty()) { + if (CollectionUtils.isEmpty(suitableHosts)) { logger.warn("No suitable hosts found."); } else { - logger.debug("Hosts having capacity and suitable for migration: {}", suitableHosts); + logger.debug("Hosts having capacity and are suitable for migration: {}", suitableHosts); } // Only list hosts of the same architecture as the source Host in a multi-arch zone diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index d3781649845..28b96b8d194 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -34,6 +34,9 @@ + + diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java new file mode 100644 index 00000000000..ddd1e3959d5 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.capacity.CapacityManager; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachineProfile; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class BaseAllocatorTest { + @Mock + HostDao hostDaoMock; + + @Mock + CapacityManager capacityManagerMock; + + @InjectMocks + @Spy + BaseAllocator baseAllocator = new MockBaseAllocator(); + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 2L; + + private final Long dcId = 3L; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final String hostTag = "hostTag"; + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagShouldRetainHostsWithServiceOfferingTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = null; + + Mockito.doReturn(hostsWithMathingTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagAndHasHostTagOnTemplateShouldRetainHostsWithServiceOfferingTagAndTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + List hostsWithMathingTemplateTags = new ArrayList<>(Arrays.asList(host1, host2)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingTemplateTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasHostTagOnTemplateShouldRetainHostsWithTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestNoServiceTagAndNoTemplateTagShouldHaveAllSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = null; + + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsEmptyShouldNotAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List emptyList = new ArrayList<>(); + + Mockito.doReturn(emptyList).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsNotEmptyShouldAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsMatchingRuleTag = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsMatchingRuleTag).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityAndCpuCapacityShouldReturnTrue() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertTrue(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityButNoCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityButHasCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityAndCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + class MockBaseAllocator extends BaseAllocator { + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, int returnUpTo) { + return null; + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { + return null; + } + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java index 0d6a6fa8ff9..00cac5bbd52 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java @@ -18,214 +18,596 @@ package com.cloud.agent.manager.allocator.impl; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.CapacityVO; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner; import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingDetailsVO; +import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; import com.cloud.utils.Pair; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDetailsDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - +@RunWith(MockitoJUnitRunner.class) public class FirstFitAllocatorTest { - private static final double TOLERANCE = 0.0001; - private FirstFitAllocator allocator; - private CapacityManager capacityMgr; - private ServiceOfferingDetailsDao offeringDetailsDao; - private ResourceManager resourceMgr; + private static final double TOLERANCE = 0.0001; - private DeploymentPlan plan; - private ServiceOffering offering; - private DeploymentPlanner.ExcludeList avoid; - private Account account; + @Mock + HostDao hostDaoMock; - private Host host1; - private Host host2; - private VirtualMachineProfile vmProfile; - ConfigurationDao configDao; + @Mock + ResourceManager resourceManagerMock; - @Before - public void setUp() { - allocator = new FirstFitAllocator(); - capacityMgr = mock(CapacityManager.class); - offeringDetailsDao = mock(ServiceOfferingDetailsDao.class); - resourceMgr = mock(ResourceManager.class); - configDao = mock(ConfigurationDao.class); + @Mock + VMInstanceDetailsDao userVmDetailsDaoMock; - allocator._capacityMgr = capacityMgr; - allocator._serviceOfferingDetailsDao = offeringDetailsDao; - allocator._resourceMgr = resourceMgr; - allocator._configDao = configDao; + @Mock + ServiceOfferingDetailsDao serviceOfferingDetailsDao; - plan = mock(DeploymentPlan.class); - offering = mock(ServiceOffering.class); - avoid = mock(DeploymentPlanner.ExcludeList.class); - account = mock(Account.class); + @Mock + CapacityManager capacityMgr; - host1 = mock(Host.class); - host2 = mock(Host.class); + @Mock + ConfigurationDao configDao; - vmProfile = mock(VirtualMachineProfile.class); - when(vmProfile.getId()).thenReturn(1L); + @Spy + @InjectMocks + FirstFitAllocator firstFitAllocatorSpy; - when(plan.getDataCenterId()).thenReturn(1L); - when(offering.getCpu()).thenReturn(2); - when(offering.getSpeed()).thenReturn(1000); - when(offering.getRamSize()).thenReturn(2048); - when(offering.getId()).thenReturn(123L); - when(offering.getHostTag()).thenReturn(null); - when(offering.getVgpuProfileId()).thenReturn(null); - } + private final Host.Type type = Host.Type.Routing; - @Test - public void testConfigure() throws Exception { - when(configDao.getConfiguration(anyMap())).thenReturn(new HashMap<>()); - assertTrue(allocator._checkHvm); - assertTrue(allocator.configure("test", new HashMap<>())); - } + private final Long clusterId = 1L; - @Test - public void testAllocateTo_SuccessfulMatch() { - List inputHosts = Arrays.asList(host1, host2); + private final Long podId = 2L; - // All hosts are allowed - when(avoid.shouldAvoid(host1)).thenReturn(false); - when(avoid.shouldAvoid(host2)).thenReturn(false); + private final Long dcId = 3L; - // No GPU requirement - when(offeringDetailsDao.findDetail(eq(123L), anyString())).thenReturn(null); + private final List emptyList = new ArrayList<>(); - // CPU capability and capacity is met - when(capacityMgr.checkIfHostReachMaxGuestLimit(any())).thenReturn(false); - when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(eq(host1), eq(offering), eq(true))) - .thenReturn(new Pair<>(true, true)); - when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(eq(host2), eq(offering), eq(true))) - .thenReturn(new Pair<>(true, false)); + private final String hostTag = "hostTag"; + private final String templateTag = "templateTag"; - when(resourceMgr.isGPUDeviceAvailable(offering, host1, vmProfile.getId())).thenReturn(true); - when(resourceMgr.isGPUDeviceAvailable(offering, host2, vmProfile.getId())).thenReturn(true); + private final HostVO host1 = mock(HostVO.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 2, true, account); + private final HostVO host2 = mock(HostVO.class); - // Only host1 should be returned - assertEquals(1, result.size()); - assertTrue(result.contains(host1)); - assertFalse(result.contains(host2)); - } + private final HostVO host3 = mock(HostVO.class); - @Test - public void testAllocateTo_AvoidSetAndGuestLimit() { - List inputHosts = Arrays.asList(host1, host2); + private final ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); - when(avoid.shouldAvoid(host1)).thenReturn(true); // Avoided - when(avoid.shouldAvoid(host2)).thenReturn(false); + private final DeploymentPlanner.ExcludeList excludeList = mock(DeploymentPlanner.ExcludeList.class); - when(capacityMgr.checkIfHostReachMaxGuestLimit(host2)).thenReturn(true); // Reached limit + private final VirtualMachineProfile virtualMachineProfile = mock(VirtualMachineProfile.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 2, true, account); + private final VMTemplateVO vmTemplateVO = mock(VMTemplateVO.class); - assertTrue(result.isEmpty()); - } + private final Account account = mock(Account.class); - @Test - public void testAllocateTo_GPUNotAvailable() { - List inputHosts = Arrays.asList(host1); - when(avoid.shouldAvoid(host1)).thenReturn(false); + private final DeploymentPlan deploymentPlan = mock(DeploymentPlan.class); - // GPU required but not available - var vgpuDetail = mock(com.cloud.service.ServiceOfferingDetailsVO.class); - var pciDetail = mock(com.cloud.service.ServiceOfferingDetailsVO.class); - when(offeringDetailsDao.findDetail(eq(123L), eq("vgpuType"))).thenReturn(vgpuDetail); - when(offeringDetailsDao.findDetail(eq(123L), eq("pciDevice"))).thenReturn(pciDetail); - when(pciDetail.getValue()).thenReturn("NVIDIA"); - when(vgpuDetail.getValue()).thenReturn("GRID"); + private final DeploymentPlanner.ExcludeList avoid = mock(DeploymentPlanner.ExcludeList.class); - when(resourceMgr.isGPUDeviceAvailable(offering, host1, vmProfile.getId())).thenReturn(false); + private final ServiceOffering offering = mock(ServiceOffering.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 1, true, account); + private final boolean considerReservedCapacity = true; - assertTrue(result.isEmpty()); - } + @Test + public void testConfigure() throws Exception { + when(configDao.getConfiguration(Mockito.anyMap())).thenReturn(new HashMap<>()); + Assert.assertTrue(firstFitAllocatorSpy._checkHvm); + Assert.assertTrue(firstFitAllocatorSpy.configure("test", new HashMap<>())); + } - @Test - public void testHostByCombinedCapacityOrder() { - // Test scenario 1: Default capacity usage (0.5 weight) - List mockCapacity = getHostCapacities(); - Map hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.5); + @Test + public void testAllocateTo_SuccessfulMatch() { + List inputHosts = Arrays.asList(host1, host2); - // Verify host ordering and capacity values - Long firstHostId = hostByCombinedCapacity.keySet().iterator().next(); - Assert.assertEquals("Host with ID 1 should be first in ordering", Long.valueOf(1L), firstHostId); - Assert.assertEquals("Host 1 combined capacity should match expected value", - 0.9609375, hostByCombinedCapacity.get(1L), TOLERANCE); - Assert.assertEquals("Host 2 combined capacity should match expected value", - 0.9296875, hostByCombinedCapacity.get(2L), TOLERANCE); + // All hosts are allowed + when(avoid.shouldAvoid(host1)).thenReturn(false); + when(avoid.shouldAvoid(host2)).thenReturn(false); - // Test scenario 2: Modified capacity usage (0.7 weight) - when(mockCapacity.get(0).getUsedCapacity()).thenReturn(1500L); - hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.7); + // CPU capability and capacity is met + when(capacityMgr.checkIfHostReachMaxGuestLimit(Mockito.any(Host.class))).thenReturn(false); + when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host1, offering, true)) + .thenReturn(new Pair<>(true, true)); + when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host2, offering,true)) + .thenReturn(new Pair<>(true, false)); - // Verify new ordering after capacity change - firstHostId = hostByCombinedCapacity.keySet().iterator().next(); - Assert.assertEquals("Host with ID 2 should be first after capacity change", Long.valueOf(2L), firstHostId); - Assert.assertEquals("Host 2 combined capacity should match expected value after change", - 0.9515625, hostByCombinedCapacity.get(2L), TOLERANCE); - Assert.assertEquals("Host 1 combined capacity should match expected value after change", - 0.9484375, hostByCombinedCapacity.get(1L), TOLERANCE); - } + when(resourceManagerMock.isGPUDeviceAvailable(offering, host1, virtualMachineProfile.getId())).thenReturn(true); + when(resourceManagerMock.isGPUDeviceAvailable(offering, host2, virtualMachineProfile.getId())).thenReturn(true); - List getHostCapacities() { - CapacityVO cpuCapacity1 = mock(CapacityVO.class); - when(cpuCapacity1.getHostOrPoolId()).thenReturn(1L); - when(cpuCapacity1.getTotalCapacity()).thenReturn(32000L); - when(cpuCapacity1.getReservedCapacity()).thenReturn(0L); - when(cpuCapacity1.getUsedCapacity()).thenReturn(500L); - when(cpuCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 2, true, account); - CapacityVO cpuCapacity2 = mock(CapacityVO.class); - when(cpuCapacity2.getHostOrPoolId()).thenReturn(2L); - when(cpuCapacity2.getTotalCapacity()).thenReturn(32000L); - when(cpuCapacity2.getReservedCapacity()).thenReturn(0L); - when(cpuCapacity2.getUsedCapacity()).thenReturn(500L); - when(cpuCapacity2.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + // Only host1 should be returned + Assert.assertEquals(1, result.size()); + Assert.assertTrue(result.contains(host1)); + Assert.assertFalse(result.contains(host2)); + } - CapacityVO memCapacity1 = mock(CapacityVO.class); - when(memCapacity1.getHostOrPoolId()).thenReturn(1L); - when(memCapacity1.getTotalCapacity()).thenReturn(8589934592L); - when(memCapacity1.getReservedCapacity()).thenReturn(0L); - when(memCapacity1.getUsedCapacity()).thenReturn(536870912L); - when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + @Test + public void testAllocateTo_AvoidSetAndGuestLimit() { + List inputHosts = Arrays.asList(host1, host2); - CapacityVO memCapacity2 = mock(CapacityVO.class); - when(memCapacity2.getHostOrPoolId()).thenReturn(2L); - when(memCapacity2.getTotalCapacity()).thenReturn(8589934592L); - when(memCapacity2.getReservedCapacity()).thenReturn(0L); - when(memCapacity2.getUsedCapacity()).thenReturn(1073741824L); - when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); - return Arrays.asList(cpuCapacity1, memCapacity1, cpuCapacity2, memCapacity2); - } + when(avoid.shouldAvoid(host1)).thenReturn(true); // Avoided + when(avoid.shouldAvoid(host2)).thenReturn(false); + + when(capacityMgr.checkIfHostReachMaxGuestLimit(host2)).thenReturn(true); // Reached limit + + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 2, true, account); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testAllocateTo_GPUNotAvailable() { + List inputHosts = Arrays.asList(host1); + when(avoid.shouldAvoid(host1)).thenReturn(false); + when(resourceManagerMock.isGPUDeviceAvailable(offering, host1, virtualMachineProfile.getId())).thenReturn(false); + + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 1, true, account); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void allocateToTestHostTypeStorageShouldReturnNull() { + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsEmptyShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(emptyList).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsNotEmptyShouldCallAllocateToMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(VirtualMachineProfile.class), Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).allocateTo(virtualMachineProfile, deploymentPlan, serviceOffering, vmTemplateVO, excludeList, hosts, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity, account); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void allocateToTestProvidedHostsNotNullShouldCallAddHostsToAvoidSetMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(VirtualMachineProfile.class), Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndHaTagNotNullShouldReturnOnlyHostsWithHaTag() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNotNullAndHaTagNotNullShouldReturnOnlyHostsToFilterWithHaTag() { + List hostsToFilter = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, hostsToFilter, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagAndNoHostTagShouldReturnOnlyAllUpAndEnabledNonHaHosts() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List upAndEnabledHostsWithNoHa = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(upAndEnabledHostsWithNoHa).when(resourceManagerMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.nullable(String.class), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, null, null); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagWithHostTagShouldCallRetainHostsMatchingServiceOfferingAndTemplateTags() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void addHostsToAvoidSetTestAllHostsWereConsideredForAllocationShouldNotAddAnyHostToTheAvoidSet() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(suitableHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, suitableHosts); + + Assert.assertTrue(excludeList.getHostsToAvoid().isEmpty()); + } + + @Test + public void addHostsToAvoidSetTestNotAllHostsWereConsideredForAllocationShouldAddHostToTheAvoidSet() { + List allUpAndEnabledNonHAHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List consideredHosts = new ArrayList<>(Arrays.asList(host2, host3)); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(allUpAndEnabledNonHAHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, consideredHosts); + + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void filterHostsWithUefiEnabledTestNoDetailWithUefiShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + VMInstanceDetailVO userVmDetailVO = null; + + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithInvalidModeShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = "Invalid mode"; + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithLegacyModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = ApiConstants.BootMode.LEGACY.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithSecureModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = ApiConstants.BootMode.SECURE.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedButHostDoesNotHaveItShouldReturnTrue() { + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(false).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.any(ServiceOffering.class), Mockito.any(Host.class), Mockito.any(Long.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, virtualMachineProfile, excludeList, host1); + + Assert.assertTrue(result); + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedAndHostDoesHaveItShouldReturnFalse() { + Mockito.doReturn(true).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.any(ServiceOffering.class), Mockito.any(Host.class), Mockito.any(Long.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, virtualMachineProfile, excludeList, host1); + + Assert.assertFalse(result); + Mockito.verify(excludeList, Mockito.never()).addHost(Mockito.anyLong()); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateDoesNotRequireHvm() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateRequiresHvmShouldReturnOnlyHvmHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingRequestedVGpuViaDetailShouldDoNothing() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + ServiceOfferingDetailsVO requestedVGpuType = mock(ServiceOfferingDetailsVO.class); + + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, hosts); + + Assert.assertEquals(3, hosts.size()); + Assert.assertEquals(host1, hosts.get(0)); + Assert.assertEquals(host2, hosts.get(1)); + Assert.assertEquals(host3, hosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingRequestedVGpuViaProfileIdShouldDoNothing() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(1L).when(serviceOffering).getVgpuProfileId(); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, hosts); + + Assert.assertEquals(3, hosts.size()); + Assert.assertEquals(host1, hosts.get(0)); + Assert.assertEquals(host2, hosts.get(1)); + Assert.assertEquals(host3, hosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldReorderList() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(null).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(null).when(serviceOffering).getVgpuProfileId(); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(true).when(resourceManagerMock).isHostGpuEnabled(1L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(2L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(3L); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host2, allHosts.get(0)); + Assert.assertEquals(host3, allHosts.get(1)); + Assert.assertEquals(host1, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldNotReorderListIfThereIsNoHostWithVGpu() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(null).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(null).when(serviceOffering).getVgpuProfileId(); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(Mockito.anyLong()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host1, allHosts.get(0)); + Assert.assertEquals(host2, allHosts.get(1)); + Assert.assertEquals(host3, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateDidNotRequestedHvmShouldPutHostThatDoesNotSupportHvmInStartOfThePriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host2, prioritizedHosts.get(0)); + Assert.assertEquals(host1, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateRequiresHvmShouldNotReorderList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host1, prioritizedHosts.get(0)); + Assert.assertEquals(host2, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsWithMatchingGuestOsTestShouldPutMatchingHostInHighPriorityAndHostsThatDoesNotMatchInLowPriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List highPriorityHosts = new ArrayList<>(); + List lowPriorityHosts = new ArrayList<>(); + String guestOsCategory1 = "guestOsCategory1"; + String guestOsCategory2 = "guestOsCategory2"; + + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getTemplateGuestOSCategory(vmTemplateVO); + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getHostGuestOSCategory(host1); + Mockito.doReturn(guestOsCategory2).when(firstFitAllocatorSpy).getHostGuestOSCategory(host2); + Mockito.doReturn(null).when(firstFitAllocatorSpy).getHostGuestOSCategory(host3); + firstFitAllocatorSpy.prioritizeHostsWithMatchingGuestOs(vmTemplateVO,hostsToCheck, highPriorityHosts, lowPriorityHosts); + + Assert.assertEquals(1, highPriorityHosts.size()); + Assert.assertEquals(host1, highPriorityHosts.get(0)); + Assert.assertEquals(1, lowPriorityHosts.size()); + Assert.assertEquals(host2, lowPriorityHosts.get(0)); + } + + @Test + public void testHostByCombinedCapacityOrder() { + // Test scenario 1: Default capacity usage (0.5 weight) + List mockCapacity = getHostCapacities(); + Map hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.5); + + // Verify host ordering and capacity values + Long firstHostId = hostByCombinedCapacity.keySet().iterator().next(); + Assert.assertEquals("Host with ID 1 should be first in ordering", Long.valueOf(1L), firstHostId); + Assert.assertEquals("Host 1 combined capacity should match expected value", + 0.9609375, hostByCombinedCapacity.get(1L), TOLERANCE); + Assert.assertEquals("Host 2 combined capacity should match expected value", + 0.9296875, hostByCombinedCapacity.get(2L), TOLERANCE); + + // Test scenario 2: Modified capacity usage (0.7 weight) + when(mockCapacity.get(0).getUsedCapacity()).thenReturn(1500L); + hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.7); + + // Verify new ordering after capacity change + firstHostId = hostByCombinedCapacity.keySet().iterator().next(); + Assert.assertEquals("Host with ID 2 should be first after capacity change", Long.valueOf(2L), firstHostId); + Assert.assertEquals("Host 2 combined capacity should match expected value after change", + 0.9515625, hostByCombinedCapacity.get(2L), TOLERANCE); + Assert.assertEquals("Host 1 combined capacity should match expected value after change", + 0.9484375, hostByCombinedCapacity.get(1L), TOLERANCE); + } + + List getHostCapacities() { + CapacityVO cpuCapacity1 = mock(CapacityVO.class); + when(cpuCapacity1.getHostOrPoolId()).thenReturn(1L); + when(cpuCapacity1.getTotalCapacity()).thenReturn(32000L); + when(cpuCapacity1.getReservedCapacity()).thenReturn(0L); + when(cpuCapacity1.getUsedCapacity()).thenReturn(500L); + when(cpuCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + + CapacityVO cpuCapacity2 = mock(CapacityVO.class); + when(cpuCapacity2.getHostOrPoolId()).thenReturn(2L); + when(cpuCapacity2.getTotalCapacity()).thenReturn(32000L); + when(cpuCapacity2.getReservedCapacity()).thenReturn(0L); + when(cpuCapacity2.getUsedCapacity()).thenReturn(500L); + when(cpuCapacity2.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + + CapacityVO memCapacity1 = mock(CapacityVO.class); + when(memCapacity1.getHostOrPoolId()).thenReturn(1L); + when(memCapacity1.getTotalCapacity()).thenReturn(8589934592L); + when(memCapacity1.getReservedCapacity()).thenReturn(0L); + when(memCapacity1.getUsedCapacity()).thenReturn(536870912L); + when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + + CapacityVO memCapacity2 = mock(CapacityVO.class); + when(memCapacity2.getHostOrPoolId()).thenReturn(2L); + when(memCapacity2.getTotalCapacity()).thenReturn(8589934592L); + when(memCapacity2.getReservedCapacity()).thenReturn(0L); + when(memCapacity2.getUsedCapacity()).thenReturn(1073741824L); + when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + return Arrays.asList(cpuCapacity1, memCapacity1, cpuCapacity2, memCapacity2); + } } diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java new file mode 100644 index 00000000000..ab9b322eb84 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.vm.VirtualMachineProfile; +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; + +@RunWith(MockitoJUnitRunner.class) +public class RandomAllocatorTest { + + @Mock + HostDao hostDao; + + @Spy + @InjectMocks + RandomAllocator randomAllocator; + + @Mock + ResourceManager resourceManagerMock; + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 2L; + + private final Long zoneId = 3L; + + private final List emptyList = new ArrayList<>(); + + private final String hostTag = "hostTag"; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final VMTemplateVO vmTemplateVO = Mockito.mock(VMTemplateVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final DeploymentPlanner.ExcludeList excludeList = Mockito.mock(DeploymentPlanner.ExcludeList.class); + + private final VirtualMachineProfile virtualMachineProfile = Mockito.mock(VirtualMachineProfile.class); + + private final DeploymentPlan deploymentPlan = Mockito.mock(DeploymentPlan.class); + + private final boolean considerReservedCapacity = true; + + + @Test + public void testListHostsByTags() { + Host.Type type = Host.Type.Routing; + Long id = 1L; + String templateTag = "tag1"; + String offeringTag = "tag2"; + HostVO host1 = Mockito.mock(HostVO.class); + HostVO host2 = Mockito.mock(HostVO.class); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, offeringTag)).thenReturn(List.of(host1, host2)); + + // No template tagged host + ArrayList noTemplateTaggedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(new ArrayList<>()); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTaggedHosts, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(noTemplateTaggedHosts)); + + // Different template tagged host + ArrayList differentTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); + HostVO host3 = Mockito.mock(HostVO.class); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(List.of(host3)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(differentTemplateTaggedHost, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(differentTemplateTaggedHost)); + + // Matching template tagged host + ArrayList matchingTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(List.of(host1)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(matchingTemplateTaggedHost, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(matchingTemplateTaggedHost)); + Assert.assertEquals(1, matchingTemplateTaggedHost.size()); + + // No template tag + ArrayList noTemplateTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTag, type, zoneId, podId, clusterId, offeringTag, null); + Assert.assertFalse(CollectionUtils.isEmpty(noTemplateTag)); + Assert.assertEquals(2, noTemplateTag.size()); + + // No offering tag + ArrayList noOfferingTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noOfferingTag, type, zoneId, podId, clusterId, null, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(noOfferingTag)); + Assert.assertEquals(1, noOfferingTag.size()); + } + + @Test + public void findSuitableHostsTestHostTypeStorageShouldReturnNull() { + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestNoAvailableHostsShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(emptyList).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestAvailableHostsShouldCallFilterAvailableHostsOnce() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(hosts).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hosts).when(randomAllocator).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(randomAllocator, Mockito.times(1)).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestAvailableHostsReachedReturnUpToLimitShouldReturnOnlyHostsWithinLimit() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = 1; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestReturnUpToAllShouldReturnAllAvailableHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestHost1InAvoidShouldOnlyReturnHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(true).when(excludeList).shouldAvoid(host1); + Mockito.doReturn(false).when(excludeList).shouldAvoid(host2); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void filterAvailableHostsTestOnlyHost2HasCpuCapacityAndCapabilityShouldReturnOnlyHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(false).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host1); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host2); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullNotProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); + } +} diff --git a/test/integration/smoke/test_vm_strict_host_tags.py b/test/integration/smoke/test_vm_strict_host_tags.py index 2377e9a7618..aac3e1ea65f 100644 --- a/test/integration/smoke/test_vm_strict_host_tags.py +++ b/test/integration/smoke/test_vm_strict_host_tags.py @@ -423,7 +423,7 @@ class TestRestoreVMStrictTags(cloudstackTestCase): vm.restore(self.apiclient, templateid=self.template_t2.id, expunge=True) self.fail("VM should not be restored") except Exception as e: - self.assertTrue("Unable to start VM with specified id" in str(e)) + self.assertTrue("Unable to create a deployment for " in str(e)) class TestMigrateVMStrictTags(cloudstackTestCase): From 1f5dba9bd2d33e95c63cc04b57df51b5c7921630 Mon Sep 17 00:00:00 2001 From: Fabricio Duarte Date: Thu, 30 Apr 2026 12:22:35 -0300 Subject: [PATCH 5/9] Release reserved storage resources on VM deployment failure (#13048) --- .../db/schema-42200to42210-cleanup.sql | 6 +++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 24 ++++++------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 54baf226ac4..2f104568c14 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,3 +18,9 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; + +-- Entries remaining on `cloud`.`resource_reservation` during the upgrade process are stale, so delete them. +-- This script was added to normalize volume/primary storage reservations that got stuck due to a bug on VM deployment, +-- but it is more interesting to introduce a smarter logic to clean these stale reservations in the future without the need +-- for upgrades (for instance, by having a heartbeat_time column for the reservations and automatically cleaning old entries). +DELETE FROM `cloud`.`resource_reservation`; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index fe18263fd3b..b23291586ef 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4301,9 +4301,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return resourceLimitService.getResourceLimitStorageTags(diskOfferingVO); } - private List reserveStorageResourcesForVm(Account owner, Long diskOfferingId, Long diskSize, List dataDiskInfoList, Long rootDiskOfferingId, ServiceOfferingVO offering, Long rootDiskSize) throws ResourceAllocationException { - List checkedReservations = new ArrayList<>(); - + private void reserveStorageResourcesForVm(List checkedReservations, Account owner, Long diskOfferingId, Long diskSize, List dataDiskInfoList, Long rootDiskOfferingId, ServiceOfferingVO offering, Long rootDiskSize) throws ResourceAllocationException { List rootResourceLimitStorageTags = getResourceLimitStorageTags(rootDiskOfferingId != null ? rootDiskOfferingId : offering.getDiskOfferingId()); CheckedReservation rootVolumeReservation = new CheckedReservation(owner, ResourceType.volume, rootResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); checkedReservations.add(rootVolumeReservation); @@ -4311,12 +4309,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir checkedReservations.add(rootPrimaryStorageReservation); if (diskOfferingId != null) { - List additionalResourceLimitStorageTags = diskOfferingId != null ? getResourceLimitStorageTags(diskOfferingId) : null; + List additionalResourceLimitStorageTags = getResourceLimitStorageTags(diskOfferingId); DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); Long size = verifyAndGetDiskSize(diskOffering, diskSize); - CheckedReservation additionalVolumeReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService) : null; + CheckedReservation additionalVolumeReservation = new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); checkedReservations.add(additionalVolumeReservation); - CheckedReservation additionalPrimaryStorageReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, size, reservationDao, resourceLimitService) : null; + CheckedReservation additionalPrimaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, size, reservationDao, resourceLimitService); checkedReservations.add(additionalPrimaryStorageReservation); } @@ -4332,7 +4330,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir checkedReservations.add(additionalPrimaryStorageReservation); } } - return checkedReservations; } private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, @@ -4344,10 +4341,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, Long rootDiskOfferingId, long volumesSize, Volume volume, Snapshot snapshot) throws ResourceAllocationException { - List checkedReservations = new ArrayList<>(); + List checkedReservations = new ArrayList<>(); try { - checkedReservations = reserveStorageResourcesForVm(owner, diskOfferingId, diskSize, dataDiskInfoList, rootDiskOfferingId, offering, volumesSize); + reserveStorageResourcesForVm(checkedReservations, owner, diskOfferingId, diskSize, dataDiskInfoList, rootDiskOfferingId, offering, volumesSize); // verify security group ids if (securityGroupIdList != null) { @@ -4638,14 +4635,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir logger.error("error during resource reservation and allocation", e); throw new CloudRuntimeException(e); } finally { - for (CheckedReservation checkedReservation : checkedReservations) { - try { - checkedReservation.close(); - } catch (Exception e) { - logger.error("error during resource reservation and allocation", e); - throw new CloudRuntimeException(e); - } - } + ReservationHelper.closeAll(checkedReservations); } } From c07f1fd5d29ab2c6ba3a419dbf6d21b0a4f3900b Mon Sep 17 00:00:00 2001 From: Henrique Sato Date: Fri, 1 May 2026 06:54:40 -0300 Subject: [PATCH 6/9] Number of running and stopped VMs as preset variables for `Network` type Quota tariffs (#11689) Co-authored-by: Fabricio Duarte --- .../presetvariables/PresetVariableHelper.java | 14 ++++- .../presetvariables/ResourceCounting.java | 52 +++++++++++++++++++ .../activationrule/presetvariables/Value.java | 11 ++++ .../PresetVariableHelperTest.java | 37 +++++++++++-- .../response/QuotaResponseBuilderImpl.java | 3 +- 5 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index cd87052e878..23020292027 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -35,6 +35,7 @@ import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -783,10 +784,21 @@ public class PresetVariableHelper { value.setId(network.getUuid()); value.setName(network.getName()); value.setState(usageRecord.getState()); - + value.setResourceCounting(getPresetVariableValueNetworkResourceCounting(networkId)); value.setNetworkOffering(getPresetVariableValueNetworkOffering(network.getNetworkOfferingId())); } + protected ResourceCounting getPresetVariableValueNetworkResourceCounting(Long networkId) { + ResourceCounting resourceCounting = new ResourceCounting(); + List vmInstancesVO = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(networkId, VirtualMachine.Type.User); + int runningVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Running)).count(); + int stoppedVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Stopped)).count(); + + resourceCounting.setRunningVms(runningVms); + resourceCounting.setStoppedVms(stoppedVms); + return resourceCounting; + } + protected GenericPresetVariable getPresetVariableValueNetworkOffering(Long networkOfferingId) { NetworkOfferingVO networkOfferingVo = networkOfferingDao.findByIdIncludingRemoved(networkOfferingId); validateIfObjectIsNull(networkOfferingVo, networkOfferingId, "network offering"); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java new file mode 100644 index 00000000000..75049c3486a --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + + +import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class ResourceCounting { + + @PresetVariableDefinition(description = "The number of running user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int runningVms; + @PresetVariableDefinition(description = "The number of stopped user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int stoppedVms; + + public int getRunningVms() { + return runningVms; + } + + public void setRunningVms(int runningVms) { + this.runningVms = runningVms; + } + + public int getStoppedVms() { + return stoppedVms; + } + + public void setStoppedVms(int stoppedVms) { + this.stoppedVms = stoppedVms; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index aff2c040e74..286fe5c60fd 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -84,6 +84,9 @@ public class Value extends GenericPresetVariable { @PresetVariableDefinition(description = "Backup offering of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private BackupOffering backupOffering; + @PresetVariableDefinition(description = "The amount of resources of the usage type.") + private ResourceCounting resourceCounting; + @PresetVariableDefinition(description = "The hypervisor where the resource was deployed. Values can be: XenServer, KVM, VMware, Hyperv, BareMetal, Ovm, Ovm3 and LXC.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.VM_SNAPSHOT, QuotaTypes.SNAPSHOT}) private String hypervisorType; @@ -262,6 +265,14 @@ public class Value extends GenericPresetVariable { this.state = state; } + public ResourceCounting getResourceCounting() { + return resourceCounting; + } + + public void setResourceCounting(ResourceCounting resourceCounting) { + this.resourceCounting = resourceCounting; + } + public GenericPresetVariable getNetworkOffering() { return networkOffering; } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index fb093a45679..bcdbc3b46ce 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -31,11 +31,13 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -238,6 +240,8 @@ public class PresetVariableHelperTest { value.setVmSnapshotType(VMSnapshot.Type.Disk.toString()); value.setComputingResources(getComputingResourcesForTests()); value.setVolumeType(Volume.Type.DATADISK.toString()); + value.setState(Network.State.Implemented.toString()); + value.setResourceCounting(getResourceCountingForTests()); value.setNetworkOffering(getNetworkOfferingForTests()); value.setVpcOffering(getVpcOfferingForTests()); return value; @@ -276,6 +280,13 @@ public class PresetVariableHelperTest { return configuration; } + private ResourceCounting getResourceCountingForTests() { + ResourceCounting resourceCounting = new ResourceCounting(); + resourceCounting.setRunningVms(1); + resourceCounting.setStoppedVms(1); + return resourceCounting; + } + private List getHostTagsForTests() { return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); } @@ -1343,8 +1354,8 @@ public class PresetVariableHelperTest { Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); Mockito.doReturn(expected.getState()).when(usageVoMock).getState(); + Mockito.doReturn(expected.getResourceCounting()).when(presetVariableHelperSpy).getPresetVariableValueNetworkResourceCounting(Mockito.anyLong()); Mockito.doReturn(expected.getNetworkOffering()).when(presetVariableHelperSpy).getPresetVariableValueNetworkOffering(Mockito.anyLong()); - Mockito.doReturn(UsageTypes.NETWORK).when(usageVoMock).getUsageType(); Value result = new Value(); @@ -1352,7 +1363,27 @@ public class PresetVariableHelperTest { assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getState(), result.getState()); - Assert.assertEquals(expected.getNetworkOffering(), result.getNetworkOffering()); + Assert.assertEquals(expected.getResourceCounting(), result.getResourceCounting()); + } + + @Test + public void getPresetVariableValueNetworkResourceCountingTestSetValueAndReturnObject() { + VMInstanceVO vmInstanceVoMock1 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock1.setState(VirtualMachine.State.Stopped); + + VMInstanceVO vmInstanceVoMock2 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock2.setState(VirtualMachine.State.Running); + + Mockito.doReturn(List.of(vmInstanceVoMock1, vmInstanceVoMock2)).when(vmInstanceDaoMock).listNonRemovedVmsByTypeAndNetwork(Mockito.anyLong(), Mockito.any()); + + mockMethodValidateIfObjectIsNull(); + + ResourceCounting expected = getResourceCountingForTests(); + + ResourceCounting result = presetVariableHelperSpy.getPresetVariableValueNetworkResourceCounting(1L); + + Assert.assertEquals(expected.getRunningVms(), result.getRunningVms()); + Assert.assertEquals(expected.getStoppedVms(), result.getStoppedVms()); } @Test @@ -1362,7 +1393,7 @@ public class PresetVariableHelperTest { presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, null); }); - Mockito.verifyNoInteractions(networkDaoMock); + Mockito.verifyNoInteractions(vpcDaoMock); } @Test diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 173c0723731..c919bb5887c 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -98,6 +98,7 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.ComputingResou import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.ResourceCounting; import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; @@ -185,7 +186,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private VolumeDao volumeDao; - private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class, ResourceCounting.class}; private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); From a17bff9ba8770349c3b8344cb45778e89d4379f4 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 3 May 2026 22:09:41 +0530 Subject: [PATCH 7/9] ui: fix webhook filters listing (#13068) --- ui/public/locales/en.json | 1 + ui/src/components/view/ListView.vue | 2 +- ui/src/components/view/WebhookFiltersTab.vue | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 4a23b454252..e2045eba8df 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -4167,6 +4167,7 @@ "message.warn.select.template": "Please select a Template for Registration.", "message.warn.zone.mtu.update": "Please note that this limit won't affect pre-existing Network's MTU settings", "message.webhook.deliveries.time.filter": "Webhook deliveries list can be filtered based on date-time. Select 'Custom' for specifying start and end date range.", +"message.webhook.filter.add": "Webhook deliveries can be controlled using filters (currently by Event type). Please select the parameters to add to the applied filters list.", "message.zone.creation.complete": "Zone creation complete.", "message.zone.detail.description": "Populate Zone details.", "message.zone.detail.hint": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 66dd6b3db9e..b03293efaca 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -949,7 +949,7 @@ style="margin-left: 5px" :actions="actions" :resource="record" - :enabled="quickViewEnabled() && actions.length > 0" + :enabled="quickViewEnabled(actions, columns, column.key)" @exec-action="$parent.execAction" /> diff --git a/ui/src/components/view/WebhookFiltersTab.vue b/ui/src/components/view/WebhookFiltersTab.vue index 2d276e2a3cf..513720a8625 100644 --- a/ui/src/components/view/WebhookFiltersTab.vue +++ b/ui/src/components/view/WebhookFiltersTab.vue @@ -18,6 +18,7 @@