From 0b169920f34c815fc169d5a105f330949c8e5259 Mon Sep 17 00:00:00 2001 From: dahn Date: Mon, 27 Apr 2026 09:13:58 +0100 Subject: [PATCH 01/10] make dh group 31 default, support 22-24+31 (#12764) --- .../views/network/CreateVpnCustomerGateway.vue | 16 ++++++++++------ .../main/java/com/cloud/utils/net/NetUtils.java | 2 +- .../java/com/cloud/utils/net/NetUtilsTest.java | 4 ++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ui/src/views/network/CreateVpnCustomerGateway.vue b/ui/src/views/network/CreateVpnCustomerGateway.vue index f71fc4709e8..bacc04cae31 100644 --- a/ui/src/views/network/CreateVpnCustomerGateway.vue +++ b/ui/src/views/network/CreateVpnCustomerGateway.vue @@ -258,9 +258,13 @@ export default { 'Group 15': 'modp3072', 'Group 16': 'modp4096', 'Group 17': 'modp6144', - 'Group 18': 'modp8192' + 'Group 18': 'modp8192', + 'Group 22': 'modp1024s160', + 'Group 23': 'modp2048s224', + 'Group 24': 'modp2048s256', + 'Group 31': 'curve25519' }, - ikeDhGroupInitialValue: 'Group 5(modp1536)', + ikeDhGroupInitialValue: 'Group 31(curve25519)', isSubmitted: false, ikeversion: 'ike' } @@ -275,12 +279,12 @@ export default { initForm () { this.formRef = ref() this.form = reactive({ - ikeEncryption: 'aes128', + ikeEncryption: 'aes256', ikeHash: 'sha1', ikeversion: 'ike', - ikeDh: 'Group 5(modp1536)', - espEncryption: 'aes128', - espHash: 'sha1', + ikeDh: 'Group 31(curve 25519)', + espEncryption: 'aes256', + espHash: 'sha256', perfectForwardSecrecy: 'None', ikelifetime: '86400', esplifetime: '3600', diff --git a/utils/src/main/java/com/cloud/utils/net/NetUtils.java b/utils/src/main/java/com/cloud/utils/net/NetUtils.java index 65878e055e7..d89d9fa2d93 100644 --- a/utils/src/main/java/com/cloud/utils/net/NetUtils.java +++ b/utils/src/main/java/com/cloud/utils/net/NetUtils.java @@ -1265,7 +1265,7 @@ public class NetUtils { if (group == null && policyType.toLowerCase().matches("ike")) { return false; // StrongSwan requires a DH group for the IKE policy } - if (group != null && !group.matches("modp1024|modp1536|modp2048|modp3072|modp4096|modp6144|modp8192")) { + if (group != null && !group.matches("modp1024|modp1536|modp2048|modp3072|modp4096|modp6144|modp8192|modp1024s160|modp2048s224|modp2048s256|curve25519")) { return false; } } diff --git a/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java b/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java index 4495a123b07..5c9d41f90a2 100644 --- a/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java @@ -131,6 +131,10 @@ public class NetUtilsTest { assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "3des-md5;modp1024")); assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha1;modp3072,aes128-sha1;modp1536")); assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha256;modp3072,aes128-sha512;modp1536")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;modp1024s160")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;modp2048s224")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;modp2048s256")); + assertTrue(NetUtils.isValidS2SVpnPolicy("ike", "aes256-sha256;curve25519")); assertFalse(NetUtils.isValidS2SVpnPolicy("ike", "aes128-sha1")); assertFalse(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha1")); assertFalse(NetUtils.isValidS2SVpnPolicy("ike", "3des-sha1,aes256-sha1")); From ffebe8eaa64607e4294ee2d072e53b0347de5e9e Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 27 Apr 2026 15:38:52 +0530 Subject: [PATCH 02/10] Fix bulk power state query missing VM lifecycle state field (#13027) * Fix bulk power state query missing VM lifecycle state field The IdsPowerStateSelectSearch partial select did not include the VM lifecycle state, causing isPowerStateInSyncWithInstanceState to always return true when state was null. This prevented retry of failed StopCommands on subsequent ping cycles. * Add defensive check for instance host ID to prevent NPE Co-authored-by: Sachin R Doddaguni Co-authored-by: nvazquez --- .../main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index a38b6af3aa0..d8c9b9253c8 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -358,7 +358,8 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem IdsPowerStateSelectSearch.entity().getPowerHostId(), IdsPowerStateSelectSearch.entity().getPowerState(), IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(), - IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime()); + IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime(), + IdsPowerStateSelectSearch.entity().getState()); IdsPowerStateSelectSearch.done(); CountByOfferingId = createSearchBuilder(Integer.class); @@ -1105,10 +1106,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) { State instanceState = instance.getState(); + if (instanceState == null) { + logger.warn("VM {} has null instance state during power state sync check, treating as out of sync", instance); + return false; + } if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running) || (powerState == VirtualMachine.PowerState.PowerOn && instanceState == State.Stopped)) { HostVO instanceHost = hostDao.findById(instance.getHostId()); - HostVO powerHost = powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); + HostVO powerHost = instance.getHostId() != null && powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); logger.debug("VM: {} on host: {} and power host : {} is in {} state, but power state is {}", instance, instanceHost, powerHost, instanceState, powerState); return false; From 089eb36e471c263ff1a90376cadaba7a595fc9e8 Mon Sep 17 00:00:00 2001 From: Sergiy Kukunin Date: Tue, 28 Apr 2026 22:53:08 -0700 Subject: [PATCH 03/10] Linstor: fix create volume from snapshot on primary storage (#13043) * Linstor: fix create volume from snapshot on primary storage When creating a volume from a snapshot on Linstor primary storage (with lin.backup.snapshots=false), the operation fails with: "Only the following image types are currently supported: VHD, OVA, QCOW2, RAW (for PowerFlex and FiberChannel)" Root cause: the Linstor driver does not handle SNAPSHOT -> VOLUME in its canCopy()/copyAsync() methods. This causes DataMotionServiceImpl to fall through to StorageSystemDataMotionStrategy (selected because Linstor advertises STORAGE_SYSTEM_SNAPSHOT=true). That strategy's verifyFormatWithPoolType() rejects RAW format for Linstor pools, since RAW is only allowed for PowerFlex and FiberChannel. Additionally, VolumeOrchestrator.createVolumeFromSnapshot() attempts to back up the snapshot to secondary storage when the storage plugin does not advertise CAN_CREATE_TEMPLATE_FROM_SNAPSHOT. This backup fails because the snapshot only exists on Linstor primary storage. Fix: - Add CAN_CREATE_TEMPLATE_FROM_SNAPSHOT capability so the orchestrator skips the backup-to-secondary path - Add canCopySnapshotToVolumeCond() to match SNAPSHOT -> VOLUME when both are on the same Linstor primary store - Wire it into canCopy() to intercept at DataMotionServiceImpl before strategy selection, bypassing StorageSystemDataMotionStrategy - Implement copySnapshotToVolume() which delegates to the existing createResourceFromSnapshot() for native Linstor snapshot restore This follows the same pattern used by the StorPool plugin, which handles SNAPSHOT -> VOLUME directly in its driver rather than going through StorageSystemDataMotionStrategy. Tested on CloudStack 4.22 with Linstor LVM_THIN storage, creating a volume from a 1TB CNPG Postgres database snapshot. Volume creates successfully with correct path and deletes cleanly. * Let CloudRuntimeException propagate from copySnapshotToVolume Remove try/catch in copySnapshotToVolume so that CloudRuntimeException from createResourceFromSnapshot propagates to the caller, ensuring CloudStack properly notices and reports the failure. * Fix CAN_CREATE_TEMPLATE_FROM_SNAPSHOT breaking template creation Setting CAN_CREATE_TEMPLATE_FROM_SNAPSHOT unconditionally to true caused createTemplate from snapshot to take the StorPool-specific code path in TemplateManagerImpl, which sends a CopyCommand to a system VM that Linstor cannot handle. Fix: make CAN_CREATE_TEMPLATE_FROM_SNAPSHOT conditional on the same flag as STORAGE_SYSTEM_SNAPSHOT (!BackupSnapshots). When snapshots are backed up to secondary (the default), the old template creation flow works. When snapshots stay on primary, the direct path is used. Also fix checkstyle: remove unused DataObject import in test. Co-authored-by: Claude Opus 4.6 (1M context) --- .../LinstorPrimaryDataStoreDriverImpl.java | 30 ++++- ...LinstorPrimaryDataStoreDriverImplTest.java | 109 ++++++++++++++++++ 2 files changed, 137 insertions(+), 2 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 3f06bee8ac8..83dacf74e8d 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -153,6 +153,8 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver // CAN_CREATE_VOLUME_FROM_SNAPSHOT see note from CAN_CREATE_VOLUME_FROM_VOLUME mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_SNAPSHOT.toString(), Boolean.TRUE.toString()); mapCapabilities.put(DataStoreCapabilities.CAN_REVERT_VOLUME_TO_SNAPSHOT.toString(), Boolean.TRUE.toString()); + mapCapabilities.put(DataStoreCapabilities.CAN_CREATE_TEMPLATE_FROM_SNAPSHOT.toString(), + Boolean.toString(system_snapshot)); return mapCapabilities; } @@ -720,6 +722,13 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + private static boolean canCopySnapshotToVolumeCond(DataObject srcData, DataObject dstData) { + return srcData.getType() == DataObjectType.SNAPSHOT && dstData.getType() == DataObjectType.VOLUME + && srcData.getDataStore().getRole() == DataStoreRole.Primary + && dstData.getDataStore().getRole() == DataStoreRole.Primary + && srcData.getDataStore().getId() == dstData.getDataStore().getId(); + } + private static boolean canCopySnapshotCond(DataObject srcData, DataObject dstData) { return srcData.getType() == DataObjectType.SNAPSHOT && dstData.getType() == DataObjectType.SNAPSHOT && (dstData.getDataStore().getRole() == DataStoreRole.Image @@ -747,7 +756,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { logger.debug("LinstorPrimaryDataStoreDriverImpl.canCopy: " + srcData.getType() + " -> " + dstData.getType()); - if (canCopySnapshotCond(srcData, dstData)) { + if (canCopySnapshotToVolumeCond(srcData, dstData)) { + StoragePoolVO storagePool = _storagePoolDao.findById(srcData.getDataStore().getId()); + return storagePool.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME); + } else if (canCopySnapshotCond(srcData, dstData)) { SnapshotInfo sinfo = (SnapshotInfo) srcData; VolumeInfo volume = sinfo.getBaseVolume(); StoragePoolVO storagePool = _storagePoolDao.findById(volume.getPoolId()); @@ -766,6 +778,18 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return false; } + private CopyCommandResult copySnapshotToVolume(SnapshotInfo snapshotInfo, VolumeInfo volumeInfo) { + StoragePoolVO storagePoolVO = _storagePoolDao.findById(snapshotInfo.getDataStore().getId()); + String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(); + createResourceFromSnapshot(snapshotInfo.getId(), rscName, storagePoolVO); + + VolumeObjectTO volumeTO = (VolumeObjectTO) volumeInfo.getTO(); + volumeTO.setPath(volumeInfo.getUuid()); + volumeTO.setSize(volumeInfo.getSize()); + + return new CopyCommandResult(null, new CopyCmdAnswer(volumeTO)); + } + @Override public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCallback callback) { @@ -773,7 +797,9 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver + srcData.getType() + " -> " + dstData.getType()); final CopyCommandResult res; - if (canCopySnapshotCond(srcData, dstData)) { + if (canCopySnapshotToVolumeCond(srcData, dstData)) { + res = copySnapshotToVolume((SnapshotInfo) srcData, (VolumeInfo) dstData); + } else if (canCopySnapshotCond(srcData, dstData)) { String errMsg = null; Answer answer = copySnapshot(srcData, dstData); if (answer != null && !answer.getResult()) { diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java index 4653cfa358b..23266e17f4f 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -25,13 +25,23 @@ import com.linbit.linstor.api.model.ResourceGroup; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.storage.DataStoreRole; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.util.LinstorUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import static org.mockito.Mockito.mock; @@ -42,6 +52,9 @@ public class LinstorPrimaryDataStoreDriverImplTest { private DevelopersApi api; + @Mock + private PrimaryDataStoreDao _storagePoolDao; + @InjectMocks private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver; @@ -85,4 +98,100 @@ public class LinstorPrimaryDataStoreDriverImplTest { layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); } + + @Test + public void testGetCapabilitiesIncludesCreateTemplateFromSnapshotMatchesSystemSnapshot() { + Map caps = linstorPrimaryDataStoreDriver.getCapabilities(); + + Assert.assertEquals( + "CAN_CREATE_TEMPLATE_FROM_SNAPSHOT should match STORAGE_SYSTEM_SNAPSHOT", + caps.get(DataStoreCapabilities.STORAGE_SYSTEM_SNAPSHOT.toString()), + caps.get(DataStoreCapabilities.CAN_CREATE_TEMPLATE_FROM_SNAPSHOT.toString())); + } + + @Test + public void testCanCopySnapshotToVolumeOnSamePrimary() { + DataStore primaryStore = mock(DataStore.class); + when(primaryStore.getRole()).thenReturn(DataStoreRole.Primary); + when(primaryStore.getId()).thenReturn(1L); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(primaryStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(primaryStore); + + StoragePoolVO pool = mock(StoragePoolVO.class); + when(pool.getStorageProviderName()).thenReturn(LinstorUtil.PROVIDER_NAME); + when(_storagePoolDao.findById(1L)).thenReturn(pool); + + Assert.assertTrue("canCopy should return true for SNAPSHOT -> VOLUME on same Linstor primary", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } + + @Test + public void testCanCopySnapshotToVolumeRejectsNonLinstor() { + DataStore primaryStore = mock(DataStore.class); + when(primaryStore.getRole()).thenReturn(DataStoreRole.Primary); + when(primaryStore.getId()).thenReturn(1L); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(primaryStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(primaryStore); + + StoragePoolVO pool = mock(StoragePoolVO.class); + when(pool.getStorageProviderName()).thenReturn("SomeOtherProvider"); + when(_storagePoolDao.findById(1L)).thenReturn(pool); + + Assert.assertFalse("canCopy should return false for non-Linstor storage", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } + + @Test + public void testCanCopySnapshotToVolumeRejectsCrossPrimary() { + DataStore srcStore = mock(DataStore.class); + when(srcStore.getRole()).thenReturn(DataStoreRole.Primary); + when(srcStore.getId()).thenReturn(1L); + + DataStore destStore = mock(DataStore.class); + when(destStore.getRole()).thenReturn(DataStoreRole.Primary); + when(destStore.getId()).thenReturn(2L); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(srcStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(destStore); + + Assert.assertFalse("canCopy should return false for SNAPSHOT -> VOLUME across different primary stores", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } + + @Test + public void testCanCopySnapshotToVolumeRejectsImageDest() { + DataStore primaryStore = mock(DataStore.class); + when(primaryStore.getRole()).thenReturn(DataStoreRole.Primary); + + DataStore imageStore = mock(DataStore.class); + when(imageStore.getRole()).thenReturn(DataStoreRole.Image); + + SnapshotInfo snapshot = mock(SnapshotInfo.class); + when(snapshot.getType()).thenReturn(DataObjectType.SNAPSHOT); + when(snapshot.getDataStore()).thenReturn(primaryStore); + + VolumeInfo volume = mock(VolumeInfo.class); + when(volume.getType()).thenReturn(DataObjectType.VOLUME); + when(volume.getDataStore()).thenReturn(imageStore); + + Assert.assertFalse("canCopy should return false when destination is Image store", + linstorPrimaryDataStoreDriver.canCopy(snapshot, volume)); + } } From a73cc9a22c09954c1631c36d2a11064764cc8355 Mon Sep 17 00:00:00 2001 From: julien-vaz <54545601+julien-vaz@users.noreply.github.com> Date: Wed, 29 Apr 2026 21:09:13 -0300 Subject: [PATCH 04/10] Improve Quota Statement (#10506) * Improve Quota Statement * Removes unused import * Fix QuotaUsageJoinDao, QuotaResponseBuilderImpl, QuotaServiceImpl e QuotaServiceImplTest * Reorganize imports * Updates QuotaStatementCmd responseBuilder scope to default * Fix log4j syntax * Address reviews + other improvements * Add missing SQL scripts and injections * Change accountid and domainid logic + add unit tests * Rename QuotaUsageDetail to QuotaTariffUsage * Fix out of bounds exception --------- Co-authored-by: Julien Hervot de Mattos Vaz Co-authored-by: Fabricio Duarte --- .../apache/cloudstack/api/ApiConstants.java | 6 + .../META-INF/db/schema-42210to42300.sql | 10 + .../db/views/cloud_usage.quota_usage_view.sql | 35 ++ .../com/cloud/utils/db/SearchCriteria.java | 6 + .../quota/dao/QuotaTariffUsageDao.java | 29 ++ .../quota/dao/QuotaTariffUsageDaoImpl.java | 56 +++ .../quota/dao/QuotaUsageJoinDao.java | 31 ++ .../quota/dao/QuotaUsageJoinDaoImpl.java | 94 +++++ .../quota/vo/QuotaTariffUsageVO.java | 86 +++++ .../cloudstack/quota/vo/QuotaUsageJoinVO.java | 179 ++++++++++ .../quota/vo/QuotaUsageResourceVO.java | 62 ++++ .../quota/spring-framework-quota-context.xml | 4 +- .../api/command/QuotaStatementCmd.java | 74 ++-- .../api/response/QuotaResponseBuilder.java | 5 +- .../response/QuotaResponseBuilderImpl.java | 334 +++++++++++++----- .../QuotaStatementItemResourceResponse.java | 61 ++++ .../response/QuotaStatementItemResponse.java | 69 ++-- .../api/response/QuotaStatementResponse.java | 65 ++-- .../apache/cloudstack/quota/QuotaService.java | 4 +- .../cloudstack/quota/QuotaServiceImpl.java | 30 +- .../api/command/QuotaStatementCmdTest.java | 23 +- .../QuotaResponseBuilderImplTest.java | 199 +++++++++++ .../quota/QuotaServiceImplTest.java | 10 +- 23 files changed, 1218 insertions(+), 254 deletions(-) create mode 100644 engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java create mode 100644 plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 7eae16a2a37..6fb7cb0612c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -157,6 +157,7 @@ public class ApiConstants { public static final String CUSTOM_ID = "customid"; public static final String CUSTOM_ACTION_ID = "customactionid"; public static final String CUSTOM_JOB_ID = "customjobid"; + public static final String CURRENCY = "currency"; public static final String CURRENT_START_IP = "currentstartip"; public static final String CURRENT_END_IP = "currentendip"; public static final String ENCRYPT = "encrypt"; @@ -541,6 +542,7 @@ public class ApiConstants { public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; public static final String SHOW_REMOVED = "showremoved"; + public static final String SHOW_RESOURCES = "showresources"; public static final String SHOW_RESOURCE_ICON = "showicon"; public static final String SHOW_INACTIVE = "showinactive"; public static final String SHOW_UNIQUE = "showunique"; @@ -606,9 +608,11 @@ public class ApiConstants { public static final String TENANT_NAME = "tenantname"; public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; + public static final String TOTAL_QUOTA = "totalquota"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; public static final String TRUST_STORE_PASSWORD = "truststorepass"; + public static final String UNIT = "unit"; public static final String URL = "url"; public static final String USAGE_INTERFACE = "usageinterface"; public static final String USED = "used"; @@ -1300,6 +1304,8 @@ public class ApiConstants { public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; public static final String QUOTA = "quota"; + public static final String QUOTA_CONSUMED = "quotaconsumed"; + public static final String QUOTA_USAGE = "quotausage"; public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 4cb9eb7cb2c..c0feb06e76a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -117,3 +117,13 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','conserve_mode', 'tin --- Disable/enable NICs CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); + +--- Quota tariff/usage mapping +CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_tariff_usage` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `tariff_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the tariff of the Quota usage detail calculated, foreign key to quota_tariff table', + `quota_usage_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the aggregation of Quota usage details, foreign key to quota_usage table', + `quota_used` decimal(20,8) NOT NULL COMMENT 'Amount of quota used', + PRIMARY KEY (`id`), + CONSTRAINT `fk_quota_tariff_usage__tariff_id` FOREIGN KEY (`tariff_id`) REFERENCES `cloud_usage`.`quota_tariff` (`id`), + CONSTRAINT `fk_quota_tariff_usage__quota_usage_id` FOREIGN KEY (`quota_usage_id`) REFERENCES `cloud_usage`.`quota_usage` (`id`)); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql new file mode 100644 index 00000000000..7ac001384e8 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud_usage`.`quota_usage_view`; + +DROP VIEW IF EXISTS `cloud_usage`.`quota_usage_view`; +CREATE VIEW `cloud_usage`.`quota_usage_view` AS +SELECT qu.id, + qu.usage_item_id, + qu.zone_id, + qu.account_id, + qu.domain_id, + qu.usage_type, + qu.quota_used, + qu.start_date, + qu.end_date, + cu.usage_id AS resource_id, + cu.network_id as network_id, + cu.offering_id as offering_id +FROM `cloud_usage`.`quota_usage` qu +INNER JOIN `cloud_usage`.`cloud_usage` cu ON (cu.id = qu.usage_item_id); diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 15807eb26d4..3323d5c4d82 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -205,6 +205,12 @@ public class SearchCriteria { } + public void setJoinParametersIfNotNull(String joinName, String conditionName, Object... params) { + if (ArrayUtils.isNotEmpty(params) && (params.length > 1 || params[0] != null)) { + setJoinParameters(joinName, conditionName, params); + } + } + public SearchCriteria getJoin(String joinName) { return _joins.get(joinName).getT(); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java new file mode 100644 index 00000000000..9684ca117b5 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java @@ -0,0 +1,29 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import java.util.List; + +public interface QuotaTariffUsageDao extends GenericDao { + + void persistQuotaTariffUsage(QuotaTariffUsageVO quotaTariffUsage); + + List listQuotaTariffUsages(Long quotaUsageId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java new file mode 100644 index 00000000000..556f552fed6 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Component +public class QuotaTariffUsageDaoImpl extends GenericDaoBase implements QuotaTariffUsageDao { + private SearchBuilder searchQuotaTariffUsages; + + @PostConstruct + public void init() { + searchQuotaTariffUsages = createSearchBuilder(); + searchQuotaTariffUsages.and("quotaUsageId", searchQuotaTariffUsages.entity().getQuotaUsageId(), SearchCriteria.Op.EQ); + searchQuotaTariffUsages.done(); + } + + @Override + public void persistQuotaTariffUsage(final QuotaTariffUsageVO quotaTariffUsage) { + logger.trace("Persisting quota tariff usage [{}].", quotaTariffUsage); + Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(quotaTariffUsage)); + } + + @Override + public List listQuotaTariffUsages(Long quotaUsageId) { + SearchCriteria sc = searchQuotaTariffUsages.create(); + sc.setParameters("quotaUsageId", quotaUsageId); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java new file mode 100644 index 00000000000..126fa11413f --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; + +import java.util.Date; +import java.util.List; + +public interface QuotaUsageJoinDao extends GenericDao { + + List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java new file mode 100644 index 00000000000..b98ea2b3a5d --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.JoinBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.Date; +import java.util.List; + +@Component +public class QuotaUsageJoinDaoImpl extends GenericDaoBase implements QuotaUsageJoinDao { + + private SearchBuilder searchQuotaUsages; + + private SearchBuilder searchQuotaUsagesJoinTariffUsages; + + @Inject + private QuotaTariffUsageDao quotaTariffUsageDao; + + @PostConstruct + public void init() { + searchQuotaUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsages); + searchQuotaUsages.done(); + + SearchBuilder searchQuotaTariffUsages = quotaTariffUsageDao.createSearchBuilder(); + searchQuotaTariffUsages.and("tariffId", searchQuotaTariffUsages.entity().getTariffId(), SearchCriteria.Op.EQ); + searchQuotaUsagesJoinTariffUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsagesJoinTariffUsages); + searchQuotaUsagesJoinTariffUsages.join("searchQuotaTariffUsages", searchQuotaTariffUsages, searchQuotaUsagesJoinTariffUsages.entity().getId(), + searchQuotaTariffUsages.entity().getQuotaUsageId(), JoinBuilder.JoinType.INNER); + searchQuotaUsagesJoinTariffUsages.done(); + } + + private void prepareQuotaUsageSearchBuilder(SearchBuilder searchBuilder) { + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("usageType", searchBuilder.entity().getUsageType(), SearchCriteria.Op.EQ); + searchBuilder.and("resourceId", searchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); + searchBuilder.and("networkId", searchBuilder.entity().getNetworkId(), SearchCriteria.Op.EQ); + searchBuilder.and("offeringId", searchBuilder.entity().getOfferingId(), SearchCriteria.Op.EQ); + searchBuilder.and("startDate", searchBuilder.entity().getStartDate(), SearchCriteria.Op.BETWEEN); + searchBuilder.and("endDate", searchBuilder.entity().getEndDate(), SearchCriteria.Op.BETWEEN); + } + + @Override + public List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId) { + SearchCriteria sc = tariffId == null ? searchQuotaUsages.create() : searchQuotaUsagesJoinTariffUsages.create(); + + sc.setParametersIfNotNull("accountId", accountId); + sc.setParametersIfNotNull("domainId", domainId); + sc.setParametersIfNotNull("usageType", usageType); + sc.setParametersIfNotNull("resourceId", resourceId); + sc.setParametersIfNotNull("networkId", networkId); + sc.setParametersIfNotNull("offeringId", offeringId); + + if (ObjectUtils.allNotNull(startDate, endDate)) { + sc.setParameters("startDate", startDate, endDate); + sc.setParameters("endDate", startDate, endDate); + } + + sc.setJoinParametersIfNotNull("searchQuotaTariffUsages", "tariffId", tariffId); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java new file mode 100644 index 00000000000..4fa9e771713 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java @@ -0,0 +1,86 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +@Entity +@Table(name = "quota_tariff_usage") +public class QuotaTariffUsageVO implements InternalIdentity { + @Id + @Column(name = "id") + private Long id; + + @Column(name = "tariff_id") + private Long tariffId; + + @Column(name = "quota_usage_id") + private Long quotaUsageId; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + public QuotaTariffUsageVO() { + quotaUsed = new BigDecimal(0); + } + + @Override + public long getId() { + return id; + } + + public Long getTariffId() { + return tariffId; + } + + public Long getQuotaUsageId() { + return quotaUsageId; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setId(Long id) { + this.id = id; + } + + public void setTariffId(Long tariffId) { + this.tariffId = tariffId; + } + + public void setQuotaUsageId(Long quotaUsageId) { + this.quotaUsageId = quotaUsageId; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + @Override + public String toString() { + return new ReflectionToStringBuilder(this, ToStringStyle.JSON_STYLE).toString(); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java new file mode 100644 index 00000000000..df9577e23c3 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java @@ -0,0 +1,179 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.math.BigDecimal; +import java.util.Date; + +@Entity +@Table(name = "quota_usage_view") +public class QuotaUsageJoinVO implements InternalIdentity { + + @Id + @Column(name = "id", updatable = false, nullable = false) + private Long id; + + @Column(name = "zone_id") + private Long zoneId = null; + + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "domain_id") + private Long domainId = null; + + @Column(name = "usage_item_id") + private Long usageItemId; + + @Column(name = "usage_type") + private int usageType; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + @Column(name = "start_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date startDate = null; + + @Column(name = "end_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date endDate = null; + + @Column(name = "resource_id") + private Long resourceId = null; + + @Column(name = "network_id") + private Long networkId = null; + + @Column(name = "offering_id") + private Long offeringId = null; + + @Override + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getZoneId() { + return zoneId; + } + + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public Long getDomainId() { + return domainId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public Long getUsageItemId() { + return usageItemId; + } + + public void setUsageItemId(Long usageItemId) { + this.usageItemId = usageItemId; + } + + public int getUsageType() { + return usageType; + } + + public void setUsageType(int usageType) { + this.usageType = usageType; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } + + public Long getNetworkId() { + return networkId; + } + + public void setNetworkId(Long networkId) { + this.networkId = networkId; + } + + public Long getOfferingId() { + return offeringId; + } + + public void setOfferingId(Long offeringId) { + this.offeringId = offeringId; + } + + public QuotaUsageJoinVO () { + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "zoneId", "accountId", "domainId", "usageItemId", "usageType", "quotaUsed", "startDate", + "endDate", "resourceId"); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java new file mode 100644 index 00000000000..92482423100 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java @@ -0,0 +1,62 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.vo; + +import java.util.Date; + +public class QuotaUsageResourceVO { + private String uuid; + private String name; + private Date removed; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public boolean isRemoved() { + return this.removed != null; + } + + public QuotaUsageResourceVO(String uuid, String name, Date removed) { + this.uuid = uuid; + this.name = name; + this.removed = removed; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 304b23b7220..5ca2679c388 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -26,7 +26,9 @@ - + + + diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index d3bd3868ed1..bfe26a9f425 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; @@ -28,24 +27,25 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementItemResponse; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; -import com.cloud.user.Account; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a quota statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - httpMethod = "GET") +@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a Quota statement for the provided Account, Project, or Domain.", + since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaStatementCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Optional, Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, + description = "Name of the Account for which the Quota statement will be generated. Deprecated, please use accountid instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "ID of the Domain for which the Quota statement will be generated. May be used individually or with account.") private Long domainId; @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End of the period of the Quota statement. " + @@ -56,15 +56,25 @@ public class QuotaStatementCmd extends BaseCmd { ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; - @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type") + @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, + description = "Consider only Quota usage records for the specified usage type in the statement.") private Integer usageType; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "ID of the Account for which the Quota statement will be generated. Can not be specified with projectid.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, + description = "ID of the Project for which the Quota statement will be generated. Can not be specified with accountid.", since = "4.23.0") + private Long projectId; + + @Parameter(name = ApiConstants.SHOW_RESOURCES, type = CommandType.BOOLEAN, description = "List the resources of each Quota type in the period.", since = "4.23.0") + private boolean showResources; + @Inject - private QuotaResponseBuilder _responseBuilder; + QuotaResponseBuilder responseBuilder; public Long getAccountId() { return accountId; @@ -99,43 +109,47 @@ public class QuotaStatementCmd extends BaseCmd { } public Date getEndDate() { - return _responseBuilder.startOfNextDay(endDate == null ? new Date() : new Date(endDate.getTime())); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; + } + + public boolean isShowResources() { + return showResources; + } + + public void setShowResources(boolean showResources) { + this.showResources = showResources; + } + + public Long getProjectId() { + return projectId; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; } - Account activeAccountByName = _accountService.getActiveAccountByName(accountName, domainId); - if (activeAccountByName != null) { - return activeAccountByName.getAccountId(); - } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaUsage(this); - - QuotaStatementResponse response = _responseBuilder.createQuotaStatementResponse(quotaUsage); - response.setStartDate(startDate == null ? null : new Date(startDate.getTime())); - response.setEndDate(endDate == null ? null : new Date(endDate.getTime())); - + QuotaStatementResponse response = responseBuilder.createQuotaStatementResponse(this); + response.setStartDate(startDate); + response.setEndDate(endDate); response.setResponseName(getCommandName()); setResponseObject(response); } - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 177fb00d4b5..bde905c487b 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -32,7 +32,6 @@ import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import java.util.Date; import java.util.List; @@ -49,7 +48,7 @@ public interface QuotaResponseBuilder { boolean isUserAllowedToSeeActivationRules(User user); - QuotaStatementResponse createQuotaStatementResponse(List quotaUsage); + QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd); QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); @@ -57,8 +56,6 @@ public interface QuotaResponseBuilder { QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); - List getQuotaUsage(QuotaStatementCmd cmd); - List getQuotaBalance(QuotaBalanceCmd cmd); QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 2d6ec3255f4..173c0723731 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -21,13 +21,11 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.math.BigDecimal; -import java.math.RoundingMode; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -36,6 +34,7 @@ import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -43,12 +42,36 @@ import java.util.stream.Collectors; import javax.inject.Inject; import com.cloud.domain.Domain; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.projects.dao.ProjectDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.SnapshotVO; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; @@ -94,7 +117,8 @@ import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.cloudstack.quota.vo.QuotaUsageResourceVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections4.CollectionUtils; @@ -106,18 +130,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.domain.DomainVO; -import com.cloud.domain.dao.DomainDao; -import com.cloud.event.ActionEvent; -import com.cloud.event.EventTypes; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.UserDao; -import com.cloud.utils.Pair; - @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { protected Logger logger = LogManager.getLogger(getClass()); @@ -140,8 +152,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject - private ProjectDao projectDao; - @Inject private QuotaAccountDao quotaAccountDao; @Inject private DomainDao domainDao; @@ -159,6 +169,21 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; + @Inject + private IPAddressDao ipAddressDao; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkOfferingDao networkOfferingDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private VMInstanceDao vmInstanceDao; + @Inject + private VMTemplateDao vmTemplateDao; + @Inject + private VolumeDao volumeDao; + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; @@ -393,78 +418,212 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } @Override - public QuotaStatementResponse createQuotaStatementResponse(final List quotaUsage) { - if (quotaUsage == null || quotaUsage.isEmpty()) { - throw new InvalidParameterValueException("There is no usage data found for period mentioned."); - } + public QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd) { + Long accountId = getAccountIdForQuotaStatement(cmd); + Long domainId = getDomainIdForQuotaStatement(cmd, accountId); + List quotaUsages = _quotaService.getQuotaUsage(accountId, null, domainId, cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); + + logger.debug("Creating quota statement from [{}] usage records for parameters [{}].", quotaUsages.size(), + ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "accountId", "projectId", "domainId", "startDate", "endDate", "usageType", "showResources")); + createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(quotaUsages, cmd.getUsageType()); + + Map> recordsPerUsageTypes = quotaUsages.stream() + .sorted(Comparator.comparingInt(QuotaUsageJoinVO::getUsageType)) + .collect(Collectors.groupingBy(QuotaUsageJoinVO::getUsageType)); + + List items = new ArrayList<>(); + recordsPerUsageTypes.forEach((key, value) -> items.add(createStatementItem(key, value, cmd.isShowResources()))); QuotaStatementResponse statement = new QuotaStatementResponse(); - - HashMap quotaTariffMap = new HashMap(); - Collection result = QuotaTypes.listQuotaTypes().values(); - - for (QuotaTypes quotaTariff : result) { - quotaTariffMap.put(quotaTariff.getQuotaType(), quotaTariff); - // add dummy record for each usage type - QuotaUsageVO dummy = new QuotaUsageVO(quotaUsage.get(0)); - dummy.setUsageType(quotaTariff.getQuotaType()); - dummy.setQuotaUsed(new BigDecimal(0)); - quotaUsage.add(dummy); - } - - if (logger.isDebugEnabled()) { - logger.debug( - "createQuotaStatementResponse Type=" + quotaUsage.get(0).getUsageType() + " usage=" + quotaUsage.get(0).getQuotaUsed().setScale(2, RoundingMode.HALF_EVEN) - + " rec.id=" + quotaUsage.get(0).getUsageItemId() + " SD=" + quotaUsage.get(0).getStartDate() + " ED=" + quotaUsage.get(0).getEndDate()); - } - - Collections.sort(quotaUsage, new Comparator() { - @Override - public int compare(QuotaUsageVO o1, QuotaUsageVO o2) { - if (o1.getUsageType() == o2.getUsageType()) { - return 0; - } - return o1.getUsageType() < o2.getUsageType() ? -1 : 1; - } - }); - - List items = new ArrayList(); - QuotaStatementItemResponse lineitem; - int type = -1; - BigDecimal usage = new BigDecimal(0); - BigDecimal totalUsage = new BigDecimal(0); - quotaUsage.add(new QuotaUsageVO());// boundary - QuotaUsageVO prev = quotaUsage.get(0); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaStatementResponse record count=" + quotaUsage.size()); - } - for (final QuotaUsageVO quotaRecord : quotaUsage) { - if (type != quotaRecord.getUsageType()) { - if (type != -1) { - lineitem = new QuotaStatementItemResponse(type); - lineitem.setQuotaUsed(usage); - lineitem.setAccountId(prev.getAccountId()); - lineitem.setDomainId(prev.getDomainId()); - lineitem.setUsageUnit(quotaTariffMap.get(type).getQuotaUnit()); - lineitem.setUsageName(quotaTariffMap.get(type).getQuotaName()); - lineitem.setObjectName("quotausage"); - items.add(lineitem); - totalUsage = totalUsage.add(usage); - usage = new BigDecimal(0); - } - type = quotaRecord.getUsageType(); - } - prev = quotaRecord; - usage = usage.add(quotaRecord.getQuotaUsed()); - } - statement.setLineItem(items); - statement.setTotalQuota(totalUsage); + statement.setTotalQuota(items.stream().map(QuotaStatementItemResponse::getQuotaUsed).reduce(BigDecimal.ZERO, BigDecimal::add)); statement.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); statement.setObjectName("statement"); + + if (accountId != null) { + Account account = _accountDao.findByIdIncludingRemoved(accountId); + statement.setAccountId(account.getUuid()); + statement.setAccountName(account.getAccountName()); + domainId = account.getDomainId(); + } + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + statement.setDomainId(domain.getUuid()); + } + return statement; } + protected Long getAccountIdForQuotaStatement(QuotaStatementCmd cmd) { + if (Account.Type.NORMAL.equals(CallContext.current().getCallingAccount().getType())) { + logger.debug("Limiting the Quota statement for the calling Account, as they are a User Account."); + return CallContext.current().getCallingAccountId(); + } + + long accountId = cmd.getEntityOwnerId(); + if (accountId != -1) { + return accountId; + } + + if (cmd.getDomainId() == null) { + logger.debug("Limiting the Quota statement for the calling Account, as 'domainid' was not informed."); + return CallContext.current().getCallingAccountId(); + } + + logger.debug("Allowing admin/domain admin to generate the Quota statement for the provided Domain."); + return null; + } + + protected Long getDomainIdForQuotaStatement(QuotaStatementCmd cmd, Long accountId) { + if (accountId != null) { + logger.debug("Quota statement is already limited to Account [{}].", accountId); + Account account = _accountDao.findByIdIncludingRemoved(accountId); + return account.getDomainId(); + } + + Long domainId = cmd.getDomainId(); + if (domainId != null) { + return domainId; + } + + logger.debug("Limiting the Quota statement for the calling Account's Domain."); + return CallContext.current().getCallingAccount().getDomainId(); + } + + protected void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(List quotaUsages, Integer usageType) { + if (usageType != null) { + logger.debug("As the usage type [{}] was informed as parameter of the API quotaStatement, we will not create dummy records.", usageType); + return; + + } + + for (Integer quotaType : QuotaTypes.listQuotaTypes().keySet()) { + QuotaUsageJoinVO dummy = new QuotaUsageJoinVO(); + dummy.setUsageType(quotaType); + dummy.setQuotaUsed(BigDecimal.ZERO); + quotaUsages.add(dummy); + } + } + + protected QuotaStatementItemResponse createStatementItem(int usageType, List usageRecords, boolean showResources) { + QuotaUsageJoinVO firstRecord = usageRecords.get(0); + int type = firstRecord.getUsageType(); + + QuotaTypes quotaType = QuotaTypes.listQuotaTypes().get(type); + + QuotaStatementItemResponse item = new QuotaStatementItemResponse(type); + item.setQuotaUsed(usageRecords.stream().map(QuotaUsageJoinVO::getQuotaUsed).filter(Objects::nonNull).reduce(BigDecimal.ZERO, BigDecimal::add)); + item.setUsageUnit(quotaType.getQuotaUnit()); + item.setUsageName(quotaType.getQuotaName()); + + setStatementItemResources(item, usageType, usageRecords, showResources); + return item; + } + + protected void setStatementItemResources(QuotaStatementItemResponse statementItem, int usageType, List quotaUsageRecords, boolean showResources) { + if (!showResources) { + return; + } + + List itemDetails = new ArrayList<>(); + + Map quotaUsagesValuesAggregatedById = quotaUsageRecords + .stream() + .filter(quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType) != null) + .collect(Collectors.groupingBy( + quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType), + Collectors.reducing(new BigDecimal(0), QuotaUsageJoinVO::getQuotaUsed, BigDecimal::add) + )); + + for (Map.Entry entry : quotaUsagesValuesAggregatedById.entrySet()) { + QuotaStatementItemResourceResponse detail = new QuotaStatementItemResourceResponse(); + + detail.setQuotaUsed(entry.getValue()); + + QuotaUsageResourceVO resource = getResourceFromIdAndType(entry.getKey(), usageType); + if (resource != null) { + detail.setResourceId(resource.getUuid()); + detail.setDisplayName(resource.getName()); + detail.setRemoved(resource.isRemoved()); + } else { + detail.setDisplayName(""); + + } + itemDetails.add(detail); + } + statementItem.setResources(itemDetails); + } + + protected Long getResourceIdByUsageType(QuotaUsageJoinVO quotaUsageJoinVo, int usageType) { + switch (usageType) { + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + return quotaUsageJoinVo.getNetworkId(); + case QuotaTypes.NETWORK_OFFERING: + return quotaUsageJoinVo.getOfferingId(); + default: + return quotaUsageJoinVo.getResourceId(); + } + } + + protected QuotaUsageResourceVO getResourceFromIdAndType(long resourceId, int usageType) { + switch (usageType) { + case QuotaTypes.ALLOCATED_VM: + case QuotaTypes.RUNNING_VM: + VMInstanceVO vmInstance = vmInstanceDao.findByIdIncludingRemoved(resourceId); + if (vmInstance != null) { + return new QuotaUsageResourceVO(vmInstance.getUuid(), vmInstance.getHostName(), vmInstance.getRemoved()); + } + break; + case QuotaTypes.VOLUME: + case QuotaTypes.VOLUME_SECONDARY: + case QuotaTypes.VM_DISK_BYTES_READ: + case QuotaTypes.VM_DISK_BYTES_WRITE: + case QuotaTypes.VM_DISK_IO_READ: + case QuotaTypes.VM_DISK_IO_WRITE: + VolumeVO volume = volumeDao.findByIdIncludingRemoved(resourceId); + if (volume != null) { + return new QuotaUsageResourceVO(volume.getUuid(), volume.getName(), volume.getRemoved()); + } + break; + case QuotaTypes.VM_SNAPSHOT_ON_PRIMARY: + case QuotaTypes.VM_SNAPSHOT: + case QuotaTypes.SNAPSHOT: + SnapshotVO snapshot = snapshotDao.findByIdIncludingRemoved(resourceId); + if (snapshot != null) { + return new QuotaUsageResourceVO(snapshot.getUuid(), snapshot.getName(), snapshot.getRemoved()); + } + break; + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + NetworkVO network = networkDao.findByIdIncludingRemoved(resourceId); + if (network != null) { + return new QuotaUsageResourceVO(network.getUuid(), network.getName(), network.getRemoved()); + } + break; + case QuotaTypes.TEMPLATE: + case QuotaTypes.ISO: + VMTemplateVO vmTemplate = vmTemplateDao.findByIdIncludingRemoved(resourceId); + if (vmTemplate != null) { + return new QuotaUsageResourceVO(vmTemplate.getUuid(), vmTemplate.getName(), vmTemplate.getRemoved()); + } + break; + case QuotaTypes.NETWORK_OFFERING: + NetworkOfferingVO networkOffering = networkOfferingDao.findByIdIncludingRemoved(resourceId); + if (networkOffering != null) { + return new QuotaUsageResourceVO(networkOffering.getUuid(), networkOffering.getName(), networkOffering.getRemoved()); + } + break; + case QuotaTypes.IP_ADDRESS: + IPAddressVO ipAddress = ipAddressDao.findByIdIncludingRemoved(resourceId); + if (ipAddress != null) { + return new QuotaUsageResourceVO(ipAddress.getUuid(), ipAddress.getName(), ipAddress.getRemoved()); + } + break; + } + return null; + } + @Override public Pair, Integer> listQuotaTariffPlans(final QuotaTariffListCmd cmd) { Date startDate = cmd.getEffectiveDate(); @@ -518,14 +677,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } protected void warnQuotaTariffUpdateDeprecatedFields(QuotaTariffUpdateCmd cmd) { - String warnMessage = "The parameter 's%s' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; + String warnMessage = "The parameter '{}' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; if (cmd.getStartDate() != null) { - logger.warn(String.format(warnMessage,"startdate")); + logger.warn(warnMessage, "startdate"); } if (cmd.getUsageType() != null) { - logger.warn(String.format(warnMessage,"usagetype")); + logger.warn(warnMessage, "usagetype"); } } @@ -712,11 +871,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { return resp; } - @Override - public List getQuotaUsage(QuotaStatementCmd cmd) { - return _quotaService.getQuotaUsage(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); - } - @Override public List getQuotaBalance(QuotaBalanceCmd cmd) { return _quotaService.findQuotaBalanceVO(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getStartDate(), cmd.getEndDate()); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java new file mode 100644 index 00000000000..3e052f73339 --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java @@ -0,0 +1,61 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import java.math.BigDecimal; + +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; + +public class QuotaStatementItemResourceResponse extends BaseResponse { + + @SerializedName(ApiConstants.QUOTA_CONSUMED) + @Param(description = "Quota consumed.") + private BigDecimal quotaUsed; + + @SerializedName(ApiConstants.RESOURCE_ID) + @Param(description = "Resources's ID.") + private String resourceId; + + @SerializedName(ApiConstants.DISPLAY_NAME) + @Param(description = "Resource's display name.") + private String displayName; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "Indicates whether the resource is removed or active.") + private boolean removed; + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setRemoved(boolean removed) { + this.removed = removed; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java index c370d82b3cc..0747c5a9172 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java @@ -17,72 +17,41 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; +import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; public class QuotaStatementItemResponse extends BaseResponse { - @SerializedName("type") - @Param(description = "Usage type") + @SerializedName(ApiConstants.TYPE) + @Param(description = "Usage type.") private int usageType; - @SerializedName("accountid") - @Param(description = "Account id") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain id") - private Long domainId; - - @SerializedName("name") - @Param(description = "Usage type name") + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Usage type.") private String usageName; - @SerializedName("unit") - @Param(description = "Usage unit") + @SerializedName(ApiConstants.UNIT) + @Param(description = "Unit of the Usage type.") private String usageUnit; - @SerializedName("quota") - @Param(description = "Quota consumed") + @SerializedName(ApiConstants.QUOTA) + @Param(description = "Quota consumed.") private BigDecimal quotaUsed; + @SerializedName(ApiConstants.RESOURCES) + @Param(description = "Item's resources.") + private List resources; + public QuotaStatementItemResponse(final int usageType) { this.usageType = usageType; } - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } - - public void setAccountName(String accountName) { - this.accountName = accountName; - } - - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - public String getUsageName() { return usageName; } @@ -112,7 +81,15 @@ public class QuotaStatementItemResponse extends BaseResponse { } public void setQuotaUsed(BigDecimal quotaUsed) { - this.quotaUsed = quotaUsed.setScale(2, RoundingMode.HALF_EVEN); + this.quotaUsed = quotaUsed; + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + this.resources = resources; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java index 0a7ba4dbeb9..81cb1011182 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java @@ -18,56 +18,56 @@ package org.apache.cloudstack.api.response; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import java.util.List; public class QuotaStatementResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; + @SerializedName(ApiConstants.ACCOUNT_ID) + @Param(description = "ID of the Account.") + private String accountId; - @SerializedName("account") - @Param(description = "Account name") + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "Name of the Account.") private String accountName; - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "ID of the Domain.") + private String domainId; - @SerializedName("quotausage") - @Param(description = "List of quota usage under various types", responseObject = QuotaStatementItemResponse.class) + @SerializedName(ApiConstants.QUOTA_USAGE) + @Param(description = "List of Quota usage under various types.", responseObject = QuotaStatementItemResponse.class) private List lineItem; - @SerializedName("totalquota") - @Param(description = "Total quota used during this period") + @SerializedName(ApiConstants.TOTAL_QUOTA) + @Param(description = "Total Quota consumed during this period.") private BigDecimal totalQuota; - @SerializedName("startdate") - @Param(description = "Start date") + @SerializedName(ApiConstants.START_DATE) + @Param(description = "Start date of the Quota statement.") private Date startDate = null; - @SerializedName("enddate") - @Param(description = "End date") + @SerializedName(ApiConstants.END_DATE) + @Param(description = "End date of the Quota statement.") private Date endDate = null; - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Currency of the Quota statement.") private String currency; public QuotaStatementResponse() { super(); } - public Long getAccountId() { + public String getAccountId() { return accountId; } - public void setAccountId(Long accountId) { + public void setAccountId(String accountId) { this.accountId = accountId; } @@ -79,45 +79,36 @@ public class QuotaStatementResponse extends BaseResponse { this.accountName = accountName; } - public Long getDomainId() { + public String getDomainId() { return domainId; } - public void setDomainId(Long domainId) { + public void setDomainId(String domainId) { this.domainId = domainId; } - public List getLineItem() { - return lineItem; - } - public void setLineItem(List lineItem) { this.lineItem = lineItem; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - - public BigDecimal getTotalQuota() { - return totalQuota; + this.endDate = endDate; } public void setTotalQuota(BigDecimal totalQuota) { - this.totalQuota = totalQuota.setScale(2, RoundingMode.HALF_EVEN); + this.totalQuota = totalQuota; } public String getCurrency() { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index a421d0f066a..78acfc11682 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -21,14 +21,14 @@ import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import com.cloud.user.AccountVO; import com.cloud.utils.component.PluggableService; public interface QuotaService extends PluggableService { - List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); + List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index 2d7d623d1d9..a0ba2fbc751 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -53,10 +53,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; -import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.commons.lang3.ObjectUtils; import org.springframework.stereotype.Component; @@ -80,7 +80,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private QuotaAccountDao _quotaAcc; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageJoinDao quotaUsageJoinDao; @Inject private DomainDao _domainDao; @Inject @@ -213,27 +213,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi } @Override - public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { - // if accountId is not specified, use accountName and domainId - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } - } - + public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { if (startDate.after(endDate)) { throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); } @@ -241,7 +221,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi logger.debug("Getting quota records of type [{}] for account [{}] in domain [{}], between [{}] and [{}].", usageType, accountId, domainId, startDate, endDate); - return _quotaUsageDao.findQuotaUsage(accountId, domainId, usageType, startDate, endDate); + return quotaUsageJoinDao.findQuotaUsage(accountId, domainId, usageType, null, null, null, startDate, endDate, null); } @Override diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java index d6f9f747fa8..e67638db632 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java @@ -16,38 +16,29 @@ // under the License. package org.apache.cloudstack.api.command; -import junit.framework.TestCase; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - @RunWith(MockitoJUnitRunner.class) -public class QuotaStatementCmdTest extends TestCase { +public class QuotaStatementCmdTest { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder responseBuilderMock; @Test - public void testQuotaStatementCmd() throws NoSuchFieldException, IllegalAccessException { + public void executeTestVerifyCalls() { QuotaStatementCmd cmd = new QuotaStatementCmd(); cmd.setAccountName("admin"); + cmd.responseBuilder = responseBuilderMock; - Field rbField = QuotaStatementCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + Mockito.doReturn(new QuotaStatementResponse()).when(responseBuilderMock).createQuotaStatementResponse(Mockito.any()); - List quotaUsageVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaUsage(Mockito.eq(cmd))).thenReturn(quotaUsageVOList); - Mockito.when(responseBuilder.createQuotaStatementResponse(Mockito.eq(quotaUsageVOList))).thenReturn(new QuotaStatementResponse()); cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).getQuotaUsage(Mockito.eq(cmd)); + + Mockito.verify(responseBuilderMock).createQuotaStatementResponse(cmd); } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index ea88a106b84..81b4992082d 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; @@ -42,6 +44,7 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; @@ -67,6 +70,7 @@ import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.commons.lang3.time.DateUtils; @@ -914,4 +918,199 @@ public class QuotaResponseBuilderImplTest extends TestCase { Assert.assertTrue(formattedVariables.containsValue("accountname")); Assert.assertTrue(formattedVariables.containsValue("zonename")); } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeDifferentFromNullDoNothing() { + List listUsage = new ArrayList<>(); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, 1); + + Assert.assertTrue(listUsage.isEmpty()); + } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeIsNullAddDummyForAllQuotaTypes() { + List listUsage = new ArrayList<>(); + listUsage.add(new QuotaUsageJoinVO()); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, null); + + Assert.assertEquals(QuotaTypes.listQuotaTypes().size() + 1, listUsage.size()); + + QuotaTypes.listQuotaTypes().entrySet().forEach(entry -> { + Assert.assertTrue(listUsage.stream().anyMatch(usage -> usage.getUsageType() == entry.getKey() && usage.getQuotaUsed().equals(BigDecimal.ZERO))); + }); + } + + private List getQuotaUsagesForTest() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + List quotaUsages = new ArrayList<>(); + + QuotaUsageJoinVO quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(1l); + quotaUsage.setDomainId(2l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(10)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-01")); + quotaUsage.setEndDate(sdf.parse("2022-01-02")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(4l); + quotaUsage.setDomainId(5l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(null); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-03")); + quotaUsage.setEndDate(sdf.parse("2022-01-04")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(6l); + quotaUsage.setDomainId(7l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(5)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-05")); + quotaUsage.setEndDate(sdf.parse("2022-01-06")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + return quotaUsages; + } + + @Test + public void createStatementItemTestReturnItem() { + List quotaUsages = getQuotaUsagesForTest(); + Mockito.doNothing().when(quotaResponseBuilderSpy).setStatementItemResources(Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyBoolean()); + + QuotaStatementItemResponse result = quotaResponseBuilderSpy.createStatementItem(0, quotaUsages, false); + + QuotaUsageJoinVO expected = quotaUsages.get(0); + QuotaTypes quotaTypeExpected = QuotaTypes.listQuotaTypes().get(expected.getUsageType()); + Assert.assertEquals(BigDecimal.valueOf(15), result.getQuotaUsed()); + Assert.assertEquals(quotaTypeExpected.getQuotaUnit(), result.getUsageUnit()); + Assert.assertEquals(quotaTypeExpected.getQuotaName(), result.getUsageName()); + } + + @Test + public void setStatementItemResourcesTestDoNotShowResourcesDoNothing() { + QuotaStatementItemResponse item = new QuotaStatementItemResponse(1); + + quotaResponseBuilderSpy.setStatementItemResources(item, 0, getQuotaUsagesForTest(), false); + + Assert.assertNull(item.getResources()); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountForNormalUser() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsEntityOwnerIdWhenProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(42L).when(cmd).getEntityOwnerId(); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(42L), result); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountWhenCallerIsAdminAndDomainIsNotProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(null).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsAdminAndDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(10L).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertNull(result); + } + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsAccountDomainIdWhenAccountIdIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + AccountVO account = Mockito.mock(AccountVO.class); + + Mockito.doReturn(account).when(accountDaoMock).findByIdIncludingRemoved(55L); + Mockito.doReturn(77L).when(account).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, 55L); + + Assert.assertEquals(Long.valueOf(77L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsProvidedDomainIdWhenAccountIdIsNull() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(99L).when(cmd).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(Long.valueOf(99L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestFallsBackToCallingAccountDomainIdWhenNeitherAccountNorDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + Account account = Mockito.mock(Account.class); + + Mockito.doReturn(null).when(cmd).getDomainId(); + Mockito.doReturn(123L).when(account).getDomainId(); + Mockito.doReturn(account).when(callContextMock).getCallingAccount(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(123L, result.longValue()); + } + } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 259264f3b0e..a0fe63de851 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.joda.time.DateTime; @@ -63,6 +64,8 @@ public class QuotaServiceImplTest extends TestCase { @Mock QuotaBalanceDao quotaBalanceDao; @Mock + QuotaUsageJoinDao quotaUsageJoinDaoMock; + @Mock QuotaResponseBuilder respBldr; @Mock private AccountVO accountVoMock; @@ -85,9 +88,9 @@ public class QuotaServiceImplTest extends TestCase { quotaAccountDaoField.setAccessible(true); quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); - Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); + Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("quotaUsageJoinDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageJoinDaoMock); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); @@ -142,7 +145,8 @@ public class QuotaServiceImplTest extends TestCase { final Date endDate = new Date(); quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); - Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); + Mockito.verify(quotaUsageJoinDaoMock, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); } @Test From 92d82989e327470c48d3ad2123a93011c0fb5ae3 Mon Sep 17 00:00:00 2001 From: Gean Jair Silva <89494158+GeanJS@users.noreply.github.com> Date: Thu, 30 Apr 2026 05:46:26 -0300 Subject: [PATCH 05/10] Correction of the user responsible for the event (#13066) Co-authored-by: gean.silva --- .../java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 9e7c644cf65..fad2da89cf2 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -1037,7 +1037,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(ownerId, ownerType, resourceType, tag); - ActionEventUtils.onActionEvent(caller.getId(), caller.getAccountId(), + Long callingUserId = CallContext.current().getCallingUserId(); + ActionEventUtils.onActionEvent(callingUserId, caller.getAccountId(), caller.getDomainId(), EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, "Resource limit updated. Resource Type: " + resourceType + ", New Value: " + max, ownerResourceId, ownerResourceType.toString()); From c45596cca3ec5d3c98fd3bdc1fb3bca53a3352a1 Mon Sep 17 00:00:00 2001 From: Bryan Lima <42067040+BryanMLima@users.noreply.github.com> Date: Thu, 30 Apr 2026 15:30:02 +0200 Subject: [PATCH 06/10] Refactor of Allocator classes (#9074) * Refactoring Allocator classes * Break into smaller methods random and firfit allocators. * Added unit tests for random and firstfit allocators * Move random allocator from cloud-plugins to cloud-server * Add BaseAllocator abstract class for duplicate code * Add missing license * Add missing license to unit test file * Remove host allocator random dependency * Change exception message on smoke tests * Remove conditional as it was never actually reached in the original flow * Fix tests * Fix flipped parameters * Fix NPE while listing hosts for migration when suitableHosts is null * Remove unnecessary stubbings * Fix checkstyle * Remove unnecessary file * Rename exception error messages * Apply suggestions from code review Co-authored-by: Fabricio Duarte * Rename UserVmDetailVO references to VMInstanceDetailVO * Remove unused imports * Add new line at EOF * Remove unnecessary random allocator pom * Fix GPU allocation mistake * Fix failing tests --------- Co-authored-by: Fabricio Duarte Co-authored-by: Fabricio Duarte --- .../manager/allocator/HostAllocator.java | 33 - .../com/cloud/deploy/DeploymentPlanner.java | 2 +- .../admin/host/FindHostsForMigrationCmd.java | 2 +- .../api/command/admin/host/ListHostsCmd.java | 2 +- client/pom.xml | 5 - .../com/cloud/vm/VirtualMachineManager.java | 9 - .../cloud/vm/VirtualMachineManagerImpl.java | 18 - .../main/java/com/cloud/host/dao/HostDao.java | 2 +- .../java/com/cloud/host/dao/HostDaoImpl.java | 4 +- plugins/host-allocators/random/pom.xml | 30 - .../allocator/impl/RandomAllocator.java | 196 ----- .../host-allocator-random/module.properties | 18 - .../spring-host-allocator-random-context.xml | 34 - .../allocator/impl/RandomAllocatorTest.java | 80 -- plugins/pom.xml | 2 - .../manager/allocator/impl/BaseAllocator.java | 90 +++ .../allocator/impl/FirstFitAllocator.java | 660 +++++++---------- .../allocator/impl/RandomAllocator.java | 132 ++++ .../allocator/impl/TestingAllocator.java | 21 +- .../deploy/DeploymentPlanningManagerImpl.java | 22 +- .../cloud/server/ManagementServerImpl.java | 10 +- .../spring-server-allocator-context.xml | 3 + .../allocator/impl/BaseAllocatorTest.java | 219 ++++++ .../allocator/impl/FirstFitAllocatorTest.java | 690 ++++++++++++++---- .../allocator/impl/RandomAllocatorTest.java | 332 +++++++++ .../smoke/test_vm_strict_host_tags.py | 2 +- 26 files changed, 1615 insertions(+), 1003 deletions(-) delete mode 100644 plugins/host-allocators/random/pom.xml delete mode 100644 plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java delete mode 100644 plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties delete mode 100644 plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml delete mode 100644 plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java diff --git a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java index 604720aaa29..5d028d31d5b 100644 --- a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java @@ -22,19 +22,11 @@ import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.Adapter; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { - /** - * @param UserVm vm - * @param ServiceOffering offering - **/ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - /** * Determines which physical hosts are suitable to * allocate the guest virtual machines on @@ -49,31 +41,6 @@ public interface HostAllocator extends Adapter { public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); - /** - * Determines which physical hosts are suitable to allocate the guest - * virtual machines on - * - * Allocators must set any other hosts not considered for allocation in the - * ExcludeList avoid. Thus the avoid set and the list of hosts suitable, - * together must cover the entire host set in the cluster. - * - * @param VirtualMachineProfile - * vmProfile - * @param DeploymentPlan - * plan - * @param GuestType - * type - * @param ExcludeList - * avoid - * @param int returnUpTo (use -1 to return all possible hosts) - * @param boolean considerReservedCapacity (default should be true, set to - * false if host capacity calculation should not look at reserved - * capacity) - * @return List List of hosts that are suitable for VM allocation - **/ - - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity); - /** * Determines which physical hosts are suitable to allocate the guest * virtual machines on diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 8f7e773070f..22d796d4a77 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing; + random, firstfit, userdispersing, firstfitleastconsumed; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java index abca619f82a..4d6ef741961 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java @@ -78,7 +78,7 @@ public class FindHostsForMigrationCmd extends BaseListCmd { for (Host host : result.first()) { HostForMigrationResponse hostResponse = _responseGenerator.createHostForMigrationResponse(host); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index e202dfad77b..8f5e6c784d6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -252,7 +252,7 @@ public class ListHostsCmd extends BaseListCmd { for (Host host : result.first()) { HostResponse hostResponse = _responseGenerator.createHostResponse(host, getDetails()); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/client/pom.xml b/client/pom.xml index 7118f455ab5..55123de0f98 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -377,11 +377,6 @@ cloud-plugin-explicit-dedication ${project.version} - - org.apache.cloudstack - cloud-plugin-host-allocator-random - ${project.version} - org.apache.cloudstack cloud-plugin-outofbandmanagement-driver-ipmitool diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 57dc1b7bf72..e871bd8672f 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -181,15 +181,6 @@ public interface VirtualMachineManager extends Manager { void advanceReboot(String vmUuid, Map params) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; - /** - * Check to see if a virtual machine can be upgraded to the given service offering - * - * @param vm - * @param offering - * @return true if the host can handle the upgrade, false otherwise - */ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - VirtualMachine findById(long vmId); void storageMigration(String vmUuid, Map volumeToPool); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 71ecc73f325..17ddf870670 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -4017,19 +4017,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - @Override - public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { - boolean isMachineUpgradable = true; - for (final HostAllocator allocator : hostAllocators) { - isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (!isMachineUpgradable) { - break; - } - } - - return isMachineUpgradable; - } - @Override public void reboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ResourceUnavailableException { try { @@ -4469,11 +4456,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { - throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + - newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); - } - final List currentTags = StringUtils.csvTagsToList(currentDiskOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newDiskOffering.getTags()); if (VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(vmInstance.getDataCenterId())) { diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 090b019334f..5f5b2affee0 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -218,7 +218,7 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); - List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags); List findClustersThatMatchHostTagRule(String computeOfferingTags); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 2d8fcca6cdb..99c9a979c3b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -1520,7 +1520,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao } } - public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + public List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags) { List hostTagVOList = _hostTagsDao.findHostRuleTags(); List result = new ArrayList<>(); for (HostTagVO rule: hostTagVOList) { @@ -1534,7 +1534,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao public List findClustersThatMatchHostTagRule(String computeOfferingTags) { Set result = new HashSet<>(); - List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + List hosts = findHostsWithTagRuleThatMatchComputeOfferingTags(computeOfferingTags); for (HostVO host: hosts) { result.add(host.getClusterId()); } diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml deleted file mode 100644 index 061caf1573f..00000000000 --- a/plugins/host-allocators/random/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - cloud-plugin-host-allocator-random - Apache CloudStack Plugin - Host Allocator Random - - org.apache.cloudstack - cloudstack-plugins - 4.23.0.0-SNAPSHOT - ../../pom.xml - - diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java deleted file mode 100644 index 42129944a19..00000000000 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; - -@Component -public class RandomAllocator extends AdapterBase implements HostAllocator { - @Inject - private HostDao _hostDao; - @Inject - private ResourceManager _resourceMgr; - @Inject - private ClusterDao clusterDao; - @Inject - private ClusterDetailsDao clusterDetailsDao; - @Inject - private CapacityManager capacityManager; - - protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - List taggedHosts = new ArrayList<>(); - if (offeringHostTag != null) { - taggedHosts.addAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag)); - } - if (templateTag != null) { - List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - if (taggedHosts.isEmpty()) { - taggedHosts = templateTaggedHosts; - } else { - taggedHosts.retainAll(templateTaggedHosts); - } - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", - taggedHosts.size(), - (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::toString).toArray(), ","))), - type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); - } - return taggedHosts; - } - private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - List hostsCopy = null; - List suitableHosts = new ArrayList<>(); - - if (type == Host.Type.Storage) { - return suitableHosts; - } - String offeringHostTag = offering.getHostTag(); - - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - String templateTag = template.getTemplateTag(); - String hostTag = null; - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostTag = ObjectUtils.allNotNull(offeringHostTag, templateTag) ? - String.format("%s, %s", offeringHostTag, templateTag) : - ObjectUtils.firstNonNull(offeringHostTag, templateTag); - logger.debug("Looking for hosts in dc [{}], pod [{}], cluster [{}] and complying with host tag(s): [{}]", dcId, podId, clusterId, hostTag); - } else { - logger.debug("Looking for hosts in dc: {} pod: {} cluster: {}", dcId , podId, clusterId); - } - if (hosts != null) { - // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList<>(hosts); - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); - } else { - hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); - } - } else { - // list all computing hosts, regardless of whether they support routing...it's random after all - if (offeringHostTag != null) { - hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); - } else { - hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); - } - } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); - - if (hostsCopy.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); - return null; - } - - logger.debug("Random Allocator found {} hosts", hostsCopy.size()); - if (hostsCopy.isEmpty()) { - return suitableHosts; - } - - Collections.shuffle(hostsCopy); - for (Host host : hostsCopy) { - if (suitableHosts.size() == returnUpTo) { - break; - } - if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Host %s is in avoid set, skipping this and trying other available hosts", host)); - } - continue; - } - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Not using host %s; host has cpu capability? %s, host has capacity? %s", host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second())); - } - continue; - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found a suitable host, adding to list: %s", host)); - } - suitableHosts.add(host); - } - if (logger.isDebugEnabled()) { - logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - if (CollectionUtils.isEmpty(hosts)) { - if (logger.isDebugEnabled()) { - logger.debug("Random Allocator found 0 hosts as given host list is empty"); - } - return new ArrayList<>(); - } - return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, - Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity); - } - - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } -} diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties deleted file mode 100644 index dcfe8d3537f..00000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/module.properties +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -name=host-allocator-random -parent=allocator diff --git a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml b/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml deleted file mode 100644 index d84eaafaa5a..00000000000 --- a/plugins/host-allocators/random/src/main/resources/META-INF/cloudstack/host-allocator-random/spring-host-allocator-random-context.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java deleted file mode 100644 index 538d7157184..00000000000 --- a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.collections.CollectionUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; - -@RunWith(MockitoJUnitRunner.class) -public class RandomAllocatorTest { - - @Mock - HostDao hostDao; - @InjectMocks - RandomAllocator randomAllocator; - - @Test - public void testListHostsByTags() { - Host.Type type = Host.Type.Routing; - Long id = 1L; - String templateTag = "tag1"; - String offeringTag = "tag2"; - HostVO host1 = Mockito.mock(HostVO.class); - HostVO host2 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, offeringTag)).thenReturn(List.of(host1, host2)); - - // No template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - List result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Different template tagged host - HostVO host3 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Matching template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - - // No template tag - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, null); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(2, result.size()); - - // No offering tag - result = randomAllocator.listHostsByTags(type, id, id, id, null, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - } -} diff --git a/plugins/pom.xml b/plugins/pom.xml index e4904ccdf40..3262404032f 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -84,8 +84,6 @@ ha-planners/skip-heurestics - host-allocators/random - hypervisors/baremetal hypervisors/external hypervisors/hyperv diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java new file mode 100644 index 00000000000..58fcc62cdc3 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/BaseAllocator.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.capacity.CapacityManager; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import org.apache.commons.collections.CollectionUtils; + +import javax.inject.Inject; +import java.util.List; + +public abstract class BaseAllocator extends AdapterBase implements HostAllocator { + + @Inject + protected HostDao hostDao; + + @Inject + protected CapacityManager capacityManager; + + protected void retainHostsMatchingServiceOfferingAndTemplateTags(List availableHosts, Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { + logger.debug("Hosts {} will be checked for template and host tags compatibility.", availableHosts); + + if (offeringHostTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Service Offering.", offeringHostTag); + List hostsWithHostTag = hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag); + logger.debug("Retaining hosts {} because they match the offering host tag {}.", hostsWithHostTag, offeringHostTag); + availableHosts.retainAll(hostsWithHostTag); + } + + if (templateTag != null) { + logger.debug("Looking for hosts having the tag [{}] specified in the Template.", templateTag); + List hostsWithTemplateTag = hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); + logger.debug("Retaining hosts {} because they match the template tag {}.", hostsWithTemplateTag, templateTag); + availableHosts.retainAll(hostsWithTemplateTag); + } + + logger.debug("Remaining hosts after template tag and host tags validations are {}.", availableHosts); + } + + protected void addHostsBasedOnTagRules(String hostTagOnOffering, List clusterHosts) { + List hostsWithTagRules = hostDao.findHostsWithTagRuleThatMatchComputeOfferingTags(hostTagOnOffering); + + if (CollectionUtils.isEmpty(hostsWithTagRules)) { + logger.info("No hosts found with tag rules matching the compute offering tag [{}].", hostTagOnOffering); + return; + } + + logger.info("Found hosts {} with tag rules matching the compute offering tag [{}].", hostsWithTagRules, hostTagOnOffering); + clusterHosts.addAll(hostsWithTagRules); + } + + /** + * Adds hosts with enough CPU capability and enough CPU capacity to the suitable hosts list. + */ + protected boolean hostHasCpuCapabilityAndCapacity(boolean considerReservedCapacity, ServiceOffering offering, Host host) { + logger.debug("Looking for CPU frequency {} MHz and RAM {} MB.", () -> offering.getCpu() * offering.getSpeed(), offering::getRamSize); + Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); + Boolean hasCpuCapability = cpuCapabilityAndCapacity.first(); + Boolean hasCpuCapacity = cpuCapabilityAndCapacity.second(); + + if (hasCpuCapability && hasCpuCapacity) { + logger.debug("Host {} is a suitable host as it has enough CPU capability and CPU capacity.", () -> host); + return true; + } + + logger.debug("Cannot use host {}. Does the host have CPU capability? {}. Does the host have CPU capacity? {}..", () -> host, () -> hasCpuCapability, () -> hasCpuCapacity); + return false; + } + +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 1110804959e..8b4ca318bf7 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -16,6 +16,41 @@ // under the License. package com.cloud.agent.manager.allocator.impl; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.firstfitleastconsumed; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.random; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.userdispersing; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.configuration.Config; +import com.cloud.configuration.ConfigurationManager; +import com.cloud.deploy.DeploymentClusterPlanner; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.deploy.FirstFitPlanner; +import com.cloud.gpu.GPU; +import com.cloud.host.DetailVO; +import com.cloud.host.Host; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDetailsDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.user.Account; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.vm.VMInstanceDetailVO; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; + import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; @@ -29,59 +64,19 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.gpu.dao.VgpuProfileDao; - +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.Capacity; -import com.cloud.capacity.CapacityManager; -import com.cloud.capacity.CapacityVO; -import com.cloud.capacity.dao.CapacityDao; -import com.cloud.configuration.Config; -import com.cloud.configuration.ConfigurationManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentClusterPlanner; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.deploy.FirstFitPlanner; -import com.cloud.gpu.GPU; -import com.cloud.host.DetailVO; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.host.dao.HostDetailsDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.service.ServiceOfferingDetailsVO; -import com.cloud.service.dao.ServiceOfferingDetailsDao; -import com.cloud.storage.GuestOSCategoryVO; -import com.cloud.storage.GuestOSVO; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.dao.GuestOSCategoryDao; -import com.cloud.storage.dao.GuestOSDao; -import com.cloud.user.Account; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VMInstanceDetailVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; -import com.cloud.vm.dao.VMInstanceDetailsDao; -import com.cloud.vm.dao.VMInstanceDao; - +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.jetbrains.annotations.NotNull; +import org.springframework.stereotype.Component; /** * An allocator that tries to find a fit on a computing host. This allocator does not care whether or not the host supports routing. */ @Component -public class FirstFitAllocator extends AdapterBase implements HostAllocator { - @Inject - protected HostDao _hostDao = null; +public class FirstFitAllocator extends BaseAllocator { @Inject HostDetailsDao _hostDetailsDao = null; @Inject @@ -95,289 +90,183 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { @Inject protected ResourceManager _resourceMgr; @Inject - ClusterDao _clusterDao; - @Inject - ClusterDetailsDao _clusterDetailsDao; - @Inject ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @Inject - CapacityManager _capacityMgr; - @Inject CapacityDao _capacityDao; @Inject - VMInstanceDetailsDao _vmInstanceDetailsDao; - @Inject - private VgpuProfileDao vgpuProfileDao; + VMInstanceDetailsDao vmInstanceDetailsDao; boolean _checkHvm = true; + static DecimalFormat decimalFormat = new DecimalFormat("#.##"); @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - Account account = vmProfile.getOwner(); - - boolean isVMDeployedWithUefi = false; - VMInstanceDetailVO vmInstanceDetailVO = _vmInstanceDetailsDao.findDetail(vmProfile.getId(), "UEFI"); - if(vmInstanceDetailVO != null){ - if ("secure".equalsIgnoreCase(vmInstanceDetailVO.getValue()) || "legacy".equalsIgnoreCase(vmInstanceDetailVO.getValue())) { - isVMDeployedWithUefi = true; - } - } - logger.info(" Guest VM is requested with Custom[UEFI] Boot Type "+ isVMDeployedWithUefi); - - - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of routing or not - return new ArrayList<>(); - } - String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - logger.debug("Looking for hosts in {}", paramAsStringToLog); - - String hostTagOnOffering = offering.getHostTag(); - String hostTagOnTemplate = template.getTemplateTag(); - String hostTagUefi = "UEFI"; - - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; - - List clusterHosts = new ArrayList<>(); - List hostsMatchingUefiTag = new ArrayList<>(); - if(isVMDeployedWithUefi){ - hostsMatchingUefiTag = _hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagUefi + "' are:" + hostsMatchingUefiTag); - } - } - - - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - if (haVmTag != null) { - clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag); - } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - clusterHosts = _resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId); - } else { - List hostsMatchingOfferingTag = new ArrayList<>(); - List hostsMatchingTemplateTag = new ArrayList<>(); - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsMatchingOfferingTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsMatchingOfferingTag); - } - } - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } - hostsMatchingTemplateTag = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsMatchingTemplateTag); - } - } - - if (hasSvcOfferingTag && hasTemplateTag) { - hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - if (logger.isDebugEnabled()) { - logger.debug("Found " + hostsMatchingOfferingTag.size() + " Hosts satisfying both tags, host ids are:" + hostsMatchingOfferingTag); - } - - clusterHosts = hostsMatchingOfferingTag; - } else { - if (hasSvcOfferingTag) { - clusterHosts = hostsMatchingOfferingTag; - } else { - clusterHosts = hostsMatchingTemplateTag; - } - } - } - } - - if (isVMDeployedWithUefi) { - clusterHosts.retainAll(hostsMatchingUefiTag); - } - - clusterHosts.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); - - - if (clusterHosts.isEmpty()) { - logger.warn("No suitable host found for VM [{}] with tags {} in {}.", vmProfile, hostTagOnOffering, paramAsStringToLog); - return null; - } - // add all hosts that we are not considering to the avoid list - List allhostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); - allhostsInCluster.removeAll(clusterHosts); - - logger.debug(() -> String.format("Adding hosts [%s] to the avoid set because these hosts do not support HA.", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allhostsInCluster, "uuid", "name"))); - - for (HostVO host : allhostsInCluster) { - avoid.addHost(host.getId()); - } - - return allocateTo(vmProfile, plan, offering, template, avoid, clusterHosts, returnUpTo, considerReservedCapacity, account); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { + boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + long dcId = plan.getDataCenterId(); Long podId = plan.getPodId(); Long clusterId = plan.getClusterId(); ServiceOffering offering = vmProfile.getServiceOffering(); - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); Account account = vmProfile.getOwner(); - List suitableHosts = new ArrayList<>(); - List hostsCopy = new ArrayList<>(hosts); - - if (type == Host.Type.Storage) { - // FirstFitAllocator should be used for user VMs only since it won't care whether the host is capable of - // routing or not. - return suitableHosts; - } String hostTagOnOffering = offering.getHostTag(); String hostTagOnTemplate = template.getTemplateTag(); - boolean hasSvcOfferingTag = hostTagOnOffering != null ? true : false; - boolean hasTemplateTag = hostTagOnTemplate != null ? true : false; + String paramAsStringToLog = String.format("zone [%s], pod [%s], cluster [%s]", dcId, podId, clusterId); - String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); - if (haVmTag != null) { - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + List suitableHosts = retrieveHosts(vmProfile, type, (List) hosts, clusterId, podId, dcId, hostTagOnOffering, hostTagOnTemplate); + + if (suitableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in {}.", vmProfile, paramAsStringToLog); + return null; + } + + if (CollectionUtils.isEmpty(hosts)) { + addHostsToAvoidSet(type, avoid, clusterId, podId, dcId, suitableHosts); + } + + return allocateTo(vmProfile, plan, offering, template, avoid, suitableHosts, returnUpTo, considerReservedCapacity, account); + } + + protected List retrieveHosts(VirtualMachineProfile vmProfile, Type type, List hostsToFilter, Long clusterId, Long podId, long dcId, String hostTagOnOffering, + String hostTagOnTemplate) { + String haVmTag = (String) vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + List clusterHosts; + + if (CollectionUtils.isNotEmpty(hostsToFilter)) { + clusterHosts = new ArrayList<>(hostsToFilter); } else { - if (hostTagOnOffering == null && hostTagOnTemplate == null) { - hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); - } else { - if (hasSvcOfferingTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on SvcOffering:" + hostTagOnOffering); - } - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering)); - - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnOffering + "' are:" + hostsCopy); - } - } - - if (hasTemplateTag) { - if (logger.isDebugEnabled()) { - logger.debug("Looking for hosts having tag specified on Template:" + hostTagOnTemplate); - } - - hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate)); - - if (logger.isDebugEnabled()) { - logger.debug("Hosts with tag '" + hostTagOnTemplate + "' are:" + hostsCopy); - } - } - } + clusterHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); } - hostsCopy.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); - - if (!hostsCopy.isEmpty()) { - suitableHosts = allocateTo(vmProfile, plan, offering, template, avoid, hostsCopy, returnUpTo, considerReservedCapacity, account); + if (haVmTag != null) { + clusterHosts.retainAll(hostDao.listByHostTag(type, clusterId, podId, dcId, haVmTag)); + } else if (ObjectUtils.allNull(hostTagOnOffering, hostTagOnTemplate)) { + clusterHosts.retainAll(_resourceMgr.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId)); + } else { + retainHostsMatchingServiceOfferingAndTemplateTags(clusterHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); } + filterHostsWithUefiEnabled(type, vmProfile, clusterId, podId, dcId, clusterHosts); + + addHostsBasedOnTagRules(hostTagOnOffering, clusterHosts); + + return clusterHosts; + + } + + /** + * Add all hosts to the avoid set that were not considered during the allocation + */ + protected void addHostsToAvoidSet(Type type, ExcludeList avoid, Long clusterId, Long podId, long dcId, List suitableHosts) { + List allHostsInCluster = hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); + + allHostsInCluster.removeAll(suitableHosts); + + logger.debug("Adding hosts [{}] to the avoid set because these hosts were not considered for allocation.", + () -> ReflectionToStringBuilderUtils.reflectOnlySelectedFields(allHostsInCluster, "uuid", "name")); + + for (HostVO host : allHostsInCluster) { + avoid.addHost(host.getId()); + } + } + + protected void filterHostsWithUefiEnabled(Type type, VirtualMachineProfile vmProfile, Long clusterId, Long podId, long dcId, List clusterHosts) { + VMInstanceDetailVO vmInstanceDetailVO = vmInstanceDetailsDao.findDetail(vmProfile.getId(), "UEFI"); + + if (vmInstanceDetailVO == null) { + return; + } + + if (!StringUtils.equalsAnyIgnoreCase(vmInstanceDetailVO.getValue(), ApiConstants.BootMode.SECURE.toString(), ApiConstants.BootMode.LEGACY.toString())) { + return; + } + + logger.info("Guest VM is requested with Custom[UEFI] Boot Type enabled."); + + List hostsMatchingUefiTag = hostDao.listByHostCapability(type, clusterId, podId, dcId, Host.HOST_UEFI_ENABLE); + + logger.debug("Hosts with UEFI enabled are {}.", hostsMatchingUefiTag); + clusterHosts.retainAll(hostsMatchingUefiTag); + } + + protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity, Account account) { + String vmAllocationAlgorithm = DeploymentClusterPlanner.VmAllocationAlgorithm.value(); + if (random.toString().equals(vmAllocationAlgorithm)) { + Collections.shuffle(hosts); + } else if (userdispersing.toString().equals(vmAllocationAlgorithm)) { + hosts = reorderHostsByNumberOfVms(plan, hosts, account); + } else if (firstfitleastconsumed.toString().equals(vmAllocationAlgorithm)) { + hosts = reorderHostsByCapacity(plan, hosts); + } + + logger.debug("FirstFitAllocator has {} hosts to check for allocation {}.", hosts.size(), hosts); + hosts = prioritizeHosts(template, offering, hosts); + logger.debug("Found {} hosts for allocation after prioritization: {}.", hosts.size(), hosts); + + List suitableHosts = checkHostsCompatibilities(offering, vmProfile, avoid, hosts, returnUpTo, considerReservedCapacity); + logger.debug("Host Allocator returning {} suitable hosts.", suitableHosts.size()); + return suitableHosts; } - protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, ServiceOffering offering, VMTemplateVO template, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity, Account account) { - String vmAllocationAlgorithm = DeploymentClusterPlanner.VmAllocationAlgorithm.value(); - if (vmAllocationAlgorithm.equals("random")) { - // Shuffle this so that we don't check the hosts in the same order. - Collections.shuffle(hosts); - } else if (vmAllocationAlgorithm.equals("userdispersing")) { - hosts = reorderHostsByNumberOfVms(plan, hosts, account); - } else if(vmAllocationAlgorithm.equals("firstfitleastconsumed")){ - hosts = reorderHostsByCapacity(plan, hosts); - } - - if (logger.isDebugEnabled()) { - logger.debug("FirstFitAllocator has " + hosts.size() + " hosts to check for allocation: " + hosts); - } - - // We will try to reorder the host lists such that we give priority to hosts that have - // the minimums to support a VM's requirements - hosts = prioritizeHosts(template, offering, hosts); - - if (logger.isDebugEnabled()) { - logger.debug("Found " + hosts.size() + " hosts for allocation after prioritization: " + hosts); - } - - if (logger.isDebugEnabled()) { - logger.debug("Looking for speed=" + (offering.getCpu() * offering.getSpeed()) + "Mhz, Ram=" + offering.getRamSize() + " MB"); - } - - long serviceOfferingId = offering.getId(); + protected List checkHostsCompatibilities(ServiceOffering offering, VirtualMachineProfile vmProfile, ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { List suitableHosts = new ArrayList<>(); - ServiceOfferingDetailsVO offeringDetails = null; + logger.debug("Checking compatibility for the following hosts {}.", suitableHosts); for (Host host : hosts) { if (suitableHosts.size() == returnUpTo) { break; } + if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug("Host: {} is in avoid set, skipping this and trying other available hosts", host); - } + logger.debug("Host [{}] is in avoid set, skipping this and trying other available hosts", host); continue; } - //find number of guest VMs occupying capacity on this host. - if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { + if (capacityManager.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Adding host [{}] to the avoid set because this host already has the max number of running (user and/or system) VMs.", host); avoid.addHost(host.getId()); continue; } - // Check if GPU device is required by offering and host has the availability - if (_resourceMgr.isGPUDeviceAvailable(offering, host, vmProfile.getId())) { - logger.debug("Host [{}] has required GPU devices available.", host); - } else { - // If GPU is not available, skip this host - logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", host); - avoid.addHost(host.getId()); + if (offeringRequestedVGpuAndHostDoesNotHaveIt(offering, vmProfile, avoid, host)) { continue; } - Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (cpuCapabilityAndCapacity.first() && cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug("Found a suitable host, adding to list: {}", host); - } + if (hostHasCpuCapabilityAndCapacity(considerReservedCapacity, offering, host)) { suitableHosts.add(host); - } else { - if (logger.isDebugEnabled()) { - logger.debug("Not using host {}; host has cpu capability? {}, host has capacity?{}", - host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second()); - } - avoid.addHost(host.getId()); + continue; } + avoid.addHost(host.getId()); } - - if (logger.isDebugEnabled()) { - logger.debug("Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; } - // Reorder hosts in the decreasing order of free capacity. + protected boolean offeringRequestedVGpuAndHostDoesNotHaveIt(ServiceOffering offering, VirtualMachineProfile vmProfile, ExcludeList avoid, Host host) { + if (_resourceMgr.isGPUDeviceAvailable(offering, host, vmProfile.getId())) { + logger.debug("Host [{}] has required GPU devices available.", () -> host); + return false; + } + + logger.debug("Adding host [{}] to avoid set, because this host does not have required GPU devices available.", () -> host); + avoid.addHost(host.getId()); + return true; + } + + /** + * Reorder hosts in the decreasing order of free capacity. + */ private List reorderHostsByCapacity(DeploymentPlan plan, List hosts) { Long zoneId = plan.getDataCenterId(); Long clusterId = plan.getClusterId(); @@ -388,26 +277,10 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { .sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())) .collect(Collectors.toMap(Map.Entry::getKey, entry -> decimalFormat.format(entry.getValue() * 100) + "%", (e1, e2) -> e1, LinkedHashMap::new)); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts: [{}] in descending order of free capacity (percentage) in the cluster: {}", - hostIdsByFreeCapacity, sortedHostByCapacity); - } + logger.debug("List of hosts: [{}] in descending order of free capacity (percentage) in the cluster: {}.", + hostIdsByFreeCapacity, sortedHostByCapacity); - //now filter the given list of Hosts by this ordered list - Map hostMap = new HashMap<>(); - for (Host host : hosts) { - hostMap.put(host.getId(), host); - } - List matchingHostIds = new ArrayList<>(hostMap.keySet()); - - hostIdsByFreeCapacity.retainAll(matchingHostIds); - - List reorderedHosts = new ArrayList<>(); - for(Long id: hostIdsByFreeCapacity){ - reorderedHosts.add(hostMap.get(id)); - } - - return reorderedHosts; + return filterHosts(hosts, hostIdsByFreeCapacity); } private Pair, Map> getOrderedHostsByCapacity(Long zoneId, Long clusterId) { @@ -450,116 +323,154 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { Long clusterId = plan.getClusterId(); List hostIdsByVmCount = _vmInstanceDao.listHostIdsByVmCount(dcId, podId, clusterId, account.getAccountId()); - if (logger.isDebugEnabled()) { - logger.debug("List of hosts in ascending order of number of VMs: " + hostIdsByVmCount); - } + logger.debug("List of hosts in ascending order of number of VMs: {}.", hostIdsByVmCount); - //now filter the given list of Hosts by this ordered list + return filterHosts(hosts, hostIdsByVmCount); + } + + /** + * Filter the given list of Hosts considering the ordered list + */ + private List filterHosts(List hosts, List orderedHostIdsList) { Map hostMap = new HashMap<>(); + for (Host host : hosts) { hostMap.put(host.getId(), host); } List matchingHostIds = new ArrayList<>(hostMap.keySet()); - - hostIdsByVmCount.retainAll(matchingHostIds); + orderedHostIdsList.retainAll(matchingHostIds); List reorderedHosts = new ArrayList<>(); - for (Long id : hostIdsByVmCount) { + for(Long id: orderedHostIdsList){ reorderedHosts.add(hostMap.get(id)); } return reorderedHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - + /** + * Reorder the host list giving priority to hosts that have the minimum to support the VM's requirements. + */ protected List prioritizeHosts(VMTemplateVO template, ServiceOffering offering, List hosts) { if (template == null) { return hosts; } - // Determine the guest OS category of the template - String templateGuestOSCategory = getTemplateGuestOSCategory(template); + List hostsToCheck = filterHostWithNoHvmIfTemplateRequested(template, hosts); List prioritizedHosts = new ArrayList<>(); - List noHvmHosts = new ArrayList<>(); - - // If a template requires HVM and a host doesn't support HVM, remove it from consideration - List hostsToCheck = new ArrayList<>(); - if (template.isRequiresHvm()) { - for (Host host : hosts) { - if (hostSupportsHVM(host)) { - hostsToCheck.add(host); - } else { - noHvmHosts.add(host); - } - } - } else { - hostsToCheck.addAll(hosts); - } - - if (logger.isDebugEnabled()) { - if (noHvmHosts.size() > 0) { - logger.debug("Not considering hosts: " + noHvmHosts + " to deploy template: " + template + " as they are not HVM enabled"); - } - } - // If a host is tagged with the same guest OS category as the template, move it to a high priority list - // If a host is tagged with a different guest OS category than the template, move it to a low priority list List highPriorityHosts = new ArrayList<>(); List lowPriorityHosts = new ArrayList<>(); - for (Host host : hostsToCheck) { - String hostGuestOSCategory = getHostGuestOSCategory(host); - if (hostGuestOSCategory == null) { - continue; - } else if (templateGuestOSCategory != null && templateGuestOSCategory.equals(hostGuestOSCategory)) { - highPriorityHosts.add(host); - } else { - lowPriorityHosts.add(host); - } - } + prioritizeHostsWithMatchingGuestOs(template, hostsToCheck, highPriorityHosts, lowPriorityHosts); hostsToCheck.removeAll(highPriorityHosts); hostsToCheck.removeAll(lowPriorityHosts); - // Prioritize the remaining hosts by HVM capability - for (Host host : hostsToCheck) { - if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { - // Host and template both do not support hvm, put it as first consideration - prioritizedHosts.add(0, host); - } else { - // Template doesn't require hvm, but the machine supports it, make it last for consideration - prioritizedHosts.add(host); - } - } - - // Merge the lists + prioritizeHostsByHvmCapability(template, hostsToCheck, prioritizedHosts); prioritizedHosts.addAll(0, highPriorityHosts); prioritizedHosts.addAll(lowPriorityHosts); - // if service offering is not GPU enabled then move all the GPU enabled hosts to the end of priority list. - if (_serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) == null && offering.getVgpuProfileId() == null) { + prioritizeHostsByGpuEnabled(offering, prioritizedHosts); - List gpuEnabledHosts = new ArrayList<>(); - // Check for GPU enabled hosts. - for (Host host : prioritizedHosts) { - if (_resourceMgr.isHostGpuEnabled(host.getId())) { - gpuEnabledHosts.add(host); - } - } - // Move GPU enabled hosts to the end of list - if(!gpuEnabledHosts.isEmpty()) { - prioritizedHosts.removeAll(gpuEnabledHosts); - prioritizedHosts.addAll(gpuEnabledHosts); - } - } return prioritizedHosts; } + + /** + * If a template requires HVM and a host doesn't support HVM, remove it from consideration. + */ + protected List filterHostWithNoHvmIfTemplateRequested(VMTemplateVO template, List hosts) { + List hostsToCheck = new ArrayList<>(); + + if (!template.isRequiresHvm()) { + logger.debug("Template [{}] does not require HVM, therefore, the hosts {} will not be checked for HVM compatibility.", template, hostsToCheck); + hostsToCheck.addAll(hosts); + return hostsToCheck; + } + + List noHvmHosts = new ArrayList<>(); + logger.debug("Template [{}] requires HVM, therefore, the hosts %s will be checked for HVM compatibility.", template, hostsToCheck); + + for (Host host : hosts) { + if (hostSupportsHVM(host)) { + hostsToCheck.add(host); + } else { + noHvmHosts.add(host); + } + } + + if (!noHvmHosts.isEmpty()) { + logger.debug("Not considering hosts {} to deploy VM using template {} as they are not HVM enabled.", noHvmHosts, template); + } + + return hostsToCheck; + } + + + /** + * If service offering did not request for vGPU, then move all host with GPU to the end of the host priority list. + */ + protected void prioritizeHostsByGpuEnabled(ServiceOffering offering, List prioritizedHosts) { + boolean serviceOfferingRequestedVGpu = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString()) != null || offering.getVgpuProfileId() != null; + + if (serviceOfferingRequestedVGpu) { + return; + } + + List gpuEnabledHosts = new ArrayList<>(); + + for (Host host : prioritizedHosts) { + if (_resourceMgr.isHostGpuEnabled(host.getId())) { + gpuEnabledHosts.add(host); + } + } + + if (!gpuEnabledHosts.isEmpty()) { + prioritizedHosts.removeAll(gpuEnabledHosts); + prioritizedHosts.addAll(gpuEnabledHosts); + } + } + + /** + * Prioritize remaining host by HVM capability. + * + *
    + *
  • If host and template both do not support HVM, put it at the start of the list.
    • + *
  • If the template doesn't require HVM, but the machine supports it, append it to the list.
    • + *
+ */ + protected void prioritizeHostsByHvmCapability(VMTemplateVO template, List hostsToCheck, List prioritizedHosts) { + for (Host host : hostsToCheck) { + if (!template.isRequiresHvm() && !hostSupportsHVM(host)) { + prioritizedHosts.add(0, host); + } else { + prioritizedHosts.add(host); + } + } + } + + + /** + *
    + *
  • If a host is tagged with the same guest OS category as the template, move it to a high priority list.
    • + *
  • If a host is tagged with a different guest OS category than the template, move it to a low priority list.
    • + *
+ */ + protected void prioritizeHostsWithMatchingGuestOs(VMTemplateVO template, List hostsToCheck, List highPriorityHosts, List lowPriorityHosts) { + String templateGuestOSCategory = getTemplateGuestOSCategory(template); + + for (Host host : hostsToCheck) { + String hostGuestOSCategory = getHostGuestOSCategory(host); + + if (StringUtils.equals(templateGuestOSCategory, hostGuestOSCategory)) { + highPriorityHosts.add(host); + } else if (hostGuestOSCategory != null) { + lowPriorityHosts.add(host); + } + } + } + + protected boolean hostSupportsHVM(Host host) { if (!_checkHvm) { return true; @@ -621,19 +532,8 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (_configDao != null) { Map configs = _configDao.getConfiguration(params); String value = configs.get("xenserver.check.hvm"); - _checkHvm = value == null ? true : Boolean.parseBoolean(value); + _checkHvm = value == null || Boolean.parseBoolean(value); } return true; } - - @Override - public boolean start() { - return true; - } - - @Override - public boolean stop() { - return true; - } - } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java new file mode 100644 index 00000000000..96b4255b876 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.host.Host; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.VMTemplateVO; +import com.cloud.vm.VirtualMachineProfile; + +@Component +public class RandomAllocator extends BaseAllocator { + @Inject + private ResourceManager _resourceMgr; + + protected List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity) { + if (type == Host.Type.Storage) { + return null; + } + + long dcId = plan.getDataCenterId(); + Long podId = plan.getPodId(); + Long clusterId = plan.getClusterId(); + ServiceOffering offering = vmProfile.getServiceOffering(); + + String offeringHostTag = offering.getHostTag(); + VMTemplateVO template = (VMTemplateVO) vmProfile.getTemplate(); + logger.debug("Looking for hosts in zone [{}], pod [{}], cluster [{}].", dcId, podId, clusterId); + + List availableHosts = retrieveHosts(type, (List) hosts, template, offeringHostTag, clusterId, podId, dcId); + + if (availableHosts.isEmpty()) { + logger.info("No suitable host found for VM [{}] in zone [{}], pod [{}], cluster [{}].", vmProfile, dcId, podId, clusterId); + return null; + } + + return filterAvailableHosts(avoid, returnUpTo, considerReservedCapacity, availableHosts, offering); + } + + protected List filterAvailableHosts(ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity, List availableHosts, ServiceOffering offering) { + logger.debug("Random Allocator found [{}] available hosts. They will be checked if they are in the avoid set and for CPU capability and capacity.", availableHosts::size); + List suitableHosts = new ArrayList<>(); + + Collections.shuffle(availableHosts); + for (Host host : availableHosts) { + if (suitableHosts.size() == returnUpTo) { + break; + } + + if (avoid.shouldAvoid(host)) { + logger.debug("Host [{}] is in the avoid set, skipping it and trying other available hosts.", () -> host); + continue; + } + + if (!hostHasCpuCapabilityAndCapacity(considerReservedCapacity, offering, host)) { + continue; + } + + logger.debug("Found the suitable host [{}], adding to list.", () -> host); + suitableHosts.add(host); + } + + logger.debug("Random Host Allocator returning {} suitable hosts.", suitableHosts::size); + return suitableHosts; + } + + /** + * @return all computing hosts, regardless of whether they support routing. + */ + protected List retrieveHosts(Type type, List hosts, VMTemplateVO template, String offeringHostTag, Long clusterId, Long podId, long dcId) { + List availableHosts; + String templateTag = template.getTemplateTag(); + + if (CollectionUtils.isNotEmpty(hosts)) { + availableHosts = new ArrayList<>(hosts); + } else { + availableHosts = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + } + + if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { + retainHostsMatchingServiceOfferingAndTemplateTags(availableHosts, type, dcId, podId, clusterId, offeringHostTag, templateTag); + } else { + List hostsWithNoRuleTag = hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); + logger.debug("Retaining hosts {} because they do not have rule tags.", hostsWithNoRuleTag); + availableHosts.retainAll(hostsWithNoRuleTag); + } + + addHostsBasedOnTagRules(offeringHostTag, availableHosts); + + return availableHosts; + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, + boolean considerReservedCapacity) { + return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); + } +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java index fd8e65e2476..04a0b182b7b 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/TestingAllocator.java @@ -28,32 +28,24 @@ import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public class TestingAllocator extends AdapterBase implements HostAllocator { @Inject HostDao _hostDao; - Long _computingHost; Long _storageHost; Long _routingHost; @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); + return allocateTo(vmProfile, plan, type, avoid, null, returnUpTo, true); } @Override public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - List availableHosts = new ArrayList(); + boolean considerReservedCapacity) { + List availableHosts = new ArrayList<>(); Host host = null; if (type == Host.Type.Routing && _routingHost != null) { host = _hostDao.findById(_routingHost); @@ -66,13 +58,6 @@ public class TestingAllocator extends AdapterBase implements HostAllocator { return availableHosts; } - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } - @Override public boolean configure(String name, Map params) { String value = (String)params.get(Host.Type.Routing.toString()); diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index f163a3d52a5..7fb4a97e149 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -20,7 +20,6 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -1714,20 +1713,19 @@ StateListener, Configurable { @Override public void reorderHostsByPriority(Map priorities, List hosts) { - logger.debug("Re-ordering hosts {} by priorities {}", hosts, priorities); + if (CollectionUtils.isEmpty(hosts)){ + logger.debug("Hosts list is empty; therefore, there is nothing to reorder."); + return; + } + logger.info("Re-ordering hosts [{}] by priorities [{}].", hosts, priorities); hosts.removeIf(host -> DataCenterDeployment.PROHIBITED_HOST_PRIORITY.equals(getHostPriority(priorities, host.getId()))); + hosts.sort((host1, host2) -> { + int res = getHostPriority(priorities, host1.getId()).compareTo(getHostPriority(priorities, host2.getId())); + return -res; + }); - Collections.sort(hosts, new Comparator<>() { - @Override - public int compare(Host host1, Host host2) { - int res = getHostPriority(priorities, host1.getId()).compareTo(getHostPriority(priorities, host2.getId())); - return -res; - } - } - ); - - logger.debug("Hosts after re-ordering are: {}", hosts); + logger.info("Hosts after re-ordering are: [{}].", hosts); } private Integer getHostPriority(Map priorities, Long hostId) { diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index d4266496e98..337bdbbbc63 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1696,11 +1696,7 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen List suitableHosts = new ArrayList<>(); for (final HostAllocator allocator : hostAllocators) { - if (CollectionUtils.isNotEmpty(compatibleHosts)) { - suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, compatibleHosts, HostAllocator.RETURN_UPTO_ALL, false); - } else { - suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, HostAllocator.RETURN_UPTO_ALL, false); - } + suitableHosts = allocator.allocateTo(vmProfile, plan, Host.Type.Routing, excludes, compatibleHosts, HostAllocator.RETURN_UPTO_ALL, false); if (CollectionUtils.isNotEmpty(suitableHosts)) { break; @@ -1709,10 +1705,10 @@ public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implemen _dpMgr.reorderHostsByPriority(plan.getHostPriorities(), suitableHosts); - if (suitableHosts.isEmpty()) { + if (CollectionUtils.isEmpty(suitableHosts)) { logger.warn("No suitable hosts found."); } else { - logger.debug("Hosts having capacity and suitable for migration: {}", suitableHosts); + logger.debug("Hosts having capacity and are suitable for migration: {}", suitableHosts); } // Only list hosts of the same architecture as the source Host in a multi-arch zone diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index d3781649845..28b96b8d194 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -34,6 +34,9 @@ + + diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java new file mode 100644 index 00000000000..ddd1e3959d5 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/BaseAllocatorTest.java @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import com.cloud.capacity.CapacityManager; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.utils.Pair; +import com.cloud.vm.VirtualMachineProfile; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(MockitoJUnitRunner.class) +public class BaseAllocatorTest { + @Mock + HostDao hostDaoMock; + + @Mock + CapacityManager capacityManagerMock; + + @InjectMocks + @Spy + BaseAllocator baseAllocator = new MockBaseAllocator(); + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 2L; + + private final Long dcId = 3L; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final String hostTag = "hostTag"; + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagShouldRetainHostsWithServiceOfferingTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = null; + + Mockito.doReturn(hostsWithMathingTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasServiceOfferingTagAndHasHostTagOnTemplateShouldRetainHostsWithServiceOfferingTagAndTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + List hostsWithMathingTemplateTags = new ArrayList<>(Arrays.asList(host1, host2)); + String hostTagOnTemplate = "hostTagOnTemplate"; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingTemplateTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestHasHostTagOnTemplateShouldRetainHostsWithTemplateTag() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithMathingServiceTags = new ArrayList<>(Arrays.asList(host1, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = "hostTagOnOffering"; + + Mockito.doReturn(hostsWithMathingServiceTags).when(hostDaoMock).listByHostTag(type, clusterId, podId, dcId, hostTagOnOffering); + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void retainHostsMatchingServiceOfferingAndTemplateTagsTestNoServiceTagAndNoTemplateTagShouldHaveAllSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + String hostTagOnTemplate = null; + String hostTagOnOffering = null; + + baseAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(suitableHosts, type, dcId, podId, clusterId, hostTagOnOffering, hostTagOnTemplate); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsEmptyShouldNotAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List emptyList = new ArrayList<>(); + + Mockito.doReturn(emptyList).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + } + + @Test + public void addHostsBasedOnTagRulesTestHostsWithTagRuleIsNotEmptyShouldAddToSuitableHosts() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsMatchingRuleTag = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsMatchingRuleTag).when(hostDaoMock).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.anyString()); + baseAllocator.addHostsBasedOnTagRules(hostTag, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityAndCpuCapacityShouldReturnTrue() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertTrue(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostHasCpuCapabilityButNoCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = true; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityButHasCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = true; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + @Test + public void hostHasCpuCapabilityAndCapacityTestHostDoesNotHaveCpuCapabilityAndCpuCapacityShouldReturnFalse() { + Boolean hasCpuCapability = false; + Boolean hasCpuCapacity = false; + Pair pair = new Pair<>(hasCpuCapability, hasCpuCapacity); + + Mockito.doReturn(pair).when(capacityManagerMock).checkIfHostHasCpuCapabilityAndCapacity(Mockito.any(Host.class), Mockito.any(ServiceOffering.class), Mockito.anyBoolean()); + boolean result = baseAllocator.hostHasCpuCapabilityAndCapacity(true, serviceOffering, host1); + + Assert.assertFalse(result); + } + + class MockBaseAllocator extends BaseAllocator { + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, int returnUpTo) { + return null; + } + + @Override + public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Host.Type type, DeploymentPlanner.ExcludeList avoid, List hosts, int returnUpTo, boolean considerReservedCapacity) { + return null; + } + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java index 0d6a6fa8ff9..00cac5bbd52 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocatorTest.java @@ -18,214 +18,596 @@ package com.cloud.agent.manager.allocator.impl; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.cloud.agent.manager.allocator.HostAllocator; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.CapacityVO; import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner; import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingDetailsVO; +import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; import com.cloud.utils.Pair; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.VMInstanceDetailsDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyMap; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - - +@RunWith(MockitoJUnitRunner.class) public class FirstFitAllocatorTest { - private static final double TOLERANCE = 0.0001; - private FirstFitAllocator allocator; - private CapacityManager capacityMgr; - private ServiceOfferingDetailsDao offeringDetailsDao; - private ResourceManager resourceMgr; + private static final double TOLERANCE = 0.0001; - private DeploymentPlan plan; - private ServiceOffering offering; - private DeploymentPlanner.ExcludeList avoid; - private Account account; + @Mock + HostDao hostDaoMock; - private Host host1; - private Host host2; - private VirtualMachineProfile vmProfile; - ConfigurationDao configDao; + @Mock + ResourceManager resourceManagerMock; - @Before - public void setUp() { - allocator = new FirstFitAllocator(); - capacityMgr = mock(CapacityManager.class); - offeringDetailsDao = mock(ServiceOfferingDetailsDao.class); - resourceMgr = mock(ResourceManager.class); - configDao = mock(ConfigurationDao.class); + @Mock + VMInstanceDetailsDao userVmDetailsDaoMock; - allocator._capacityMgr = capacityMgr; - allocator._serviceOfferingDetailsDao = offeringDetailsDao; - allocator._resourceMgr = resourceMgr; - allocator._configDao = configDao; + @Mock + ServiceOfferingDetailsDao serviceOfferingDetailsDao; - plan = mock(DeploymentPlan.class); - offering = mock(ServiceOffering.class); - avoid = mock(DeploymentPlanner.ExcludeList.class); - account = mock(Account.class); + @Mock + CapacityManager capacityMgr; - host1 = mock(Host.class); - host2 = mock(Host.class); + @Mock + ConfigurationDao configDao; - vmProfile = mock(VirtualMachineProfile.class); - when(vmProfile.getId()).thenReturn(1L); + @Spy + @InjectMocks + FirstFitAllocator firstFitAllocatorSpy; - when(plan.getDataCenterId()).thenReturn(1L); - when(offering.getCpu()).thenReturn(2); - when(offering.getSpeed()).thenReturn(1000); - when(offering.getRamSize()).thenReturn(2048); - when(offering.getId()).thenReturn(123L); - when(offering.getHostTag()).thenReturn(null); - when(offering.getVgpuProfileId()).thenReturn(null); - } + private final Host.Type type = Host.Type.Routing; - @Test - public void testConfigure() throws Exception { - when(configDao.getConfiguration(anyMap())).thenReturn(new HashMap<>()); - assertTrue(allocator._checkHvm); - assertTrue(allocator.configure("test", new HashMap<>())); - } + private final Long clusterId = 1L; - @Test - public void testAllocateTo_SuccessfulMatch() { - List inputHosts = Arrays.asList(host1, host2); + private final Long podId = 2L; - // All hosts are allowed - when(avoid.shouldAvoid(host1)).thenReturn(false); - when(avoid.shouldAvoid(host2)).thenReturn(false); + private final Long dcId = 3L; - // No GPU requirement - when(offeringDetailsDao.findDetail(eq(123L), anyString())).thenReturn(null); + private final List emptyList = new ArrayList<>(); - // CPU capability and capacity is met - when(capacityMgr.checkIfHostReachMaxGuestLimit(any())).thenReturn(false); - when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(eq(host1), eq(offering), eq(true))) - .thenReturn(new Pair<>(true, true)); - when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(eq(host2), eq(offering), eq(true))) - .thenReturn(new Pair<>(true, false)); + private final String hostTag = "hostTag"; + private final String templateTag = "templateTag"; - when(resourceMgr.isGPUDeviceAvailable(offering, host1, vmProfile.getId())).thenReturn(true); - when(resourceMgr.isGPUDeviceAvailable(offering, host2, vmProfile.getId())).thenReturn(true); + private final HostVO host1 = mock(HostVO.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 2, true, account); + private final HostVO host2 = mock(HostVO.class); - // Only host1 should be returned - assertEquals(1, result.size()); - assertTrue(result.contains(host1)); - assertFalse(result.contains(host2)); - } + private final HostVO host3 = mock(HostVO.class); - @Test - public void testAllocateTo_AvoidSetAndGuestLimit() { - List inputHosts = Arrays.asList(host1, host2); + private final ServiceOfferingVO serviceOffering = mock(ServiceOfferingVO.class); - when(avoid.shouldAvoid(host1)).thenReturn(true); // Avoided - when(avoid.shouldAvoid(host2)).thenReturn(false); + private final DeploymentPlanner.ExcludeList excludeList = mock(DeploymentPlanner.ExcludeList.class); - when(capacityMgr.checkIfHostReachMaxGuestLimit(host2)).thenReturn(true); // Reached limit + private final VirtualMachineProfile virtualMachineProfile = mock(VirtualMachineProfile.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 2, true, account); + private final VMTemplateVO vmTemplateVO = mock(VMTemplateVO.class); - assertTrue(result.isEmpty()); - } + private final Account account = mock(Account.class); - @Test - public void testAllocateTo_GPUNotAvailable() { - List inputHosts = Arrays.asList(host1); - when(avoid.shouldAvoid(host1)).thenReturn(false); + private final DeploymentPlan deploymentPlan = mock(DeploymentPlan.class); - // GPU required but not available - var vgpuDetail = mock(com.cloud.service.ServiceOfferingDetailsVO.class); - var pciDetail = mock(com.cloud.service.ServiceOfferingDetailsVO.class); - when(offeringDetailsDao.findDetail(eq(123L), eq("vgpuType"))).thenReturn(vgpuDetail); - when(offeringDetailsDao.findDetail(eq(123L), eq("pciDevice"))).thenReturn(pciDetail); - when(pciDetail.getValue()).thenReturn("NVIDIA"); - when(vgpuDetail.getValue()).thenReturn("GRID"); + private final DeploymentPlanner.ExcludeList avoid = mock(DeploymentPlanner.ExcludeList.class); - when(resourceMgr.isGPUDeviceAvailable(offering, host1, vmProfile.getId())).thenReturn(false); + private final ServiceOffering offering = mock(ServiceOffering.class); - List result = allocator.allocateTo(vmProfile, plan, offering, null, avoid, inputHosts, 1, true, account); + private final boolean considerReservedCapacity = true; - assertTrue(result.isEmpty()); - } + @Test + public void testConfigure() throws Exception { + when(configDao.getConfiguration(Mockito.anyMap())).thenReturn(new HashMap<>()); + Assert.assertTrue(firstFitAllocatorSpy._checkHvm); + Assert.assertTrue(firstFitAllocatorSpy.configure("test", new HashMap<>())); + } - @Test - public void testHostByCombinedCapacityOrder() { - // Test scenario 1: Default capacity usage (0.5 weight) - List mockCapacity = getHostCapacities(); - Map hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.5); + @Test + public void testAllocateTo_SuccessfulMatch() { + List inputHosts = Arrays.asList(host1, host2); - // Verify host ordering and capacity values - Long firstHostId = hostByCombinedCapacity.keySet().iterator().next(); - Assert.assertEquals("Host with ID 1 should be first in ordering", Long.valueOf(1L), firstHostId); - Assert.assertEquals("Host 1 combined capacity should match expected value", - 0.9609375, hostByCombinedCapacity.get(1L), TOLERANCE); - Assert.assertEquals("Host 2 combined capacity should match expected value", - 0.9296875, hostByCombinedCapacity.get(2L), TOLERANCE); + // All hosts are allowed + when(avoid.shouldAvoid(host1)).thenReturn(false); + when(avoid.shouldAvoid(host2)).thenReturn(false); - // Test scenario 2: Modified capacity usage (0.7 weight) - when(mockCapacity.get(0).getUsedCapacity()).thenReturn(1500L); - hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.7); + // CPU capability and capacity is met + when(capacityMgr.checkIfHostReachMaxGuestLimit(Mockito.any(Host.class))).thenReturn(false); + when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host1, offering, true)) + .thenReturn(new Pair<>(true, true)); + when(capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(host2, offering,true)) + .thenReturn(new Pair<>(true, false)); - // Verify new ordering after capacity change - firstHostId = hostByCombinedCapacity.keySet().iterator().next(); - Assert.assertEquals("Host with ID 2 should be first after capacity change", Long.valueOf(2L), firstHostId); - Assert.assertEquals("Host 2 combined capacity should match expected value after change", - 0.9515625, hostByCombinedCapacity.get(2L), TOLERANCE); - Assert.assertEquals("Host 1 combined capacity should match expected value after change", - 0.9484375, hostByCombinedCapacity.get(1L), TOLERANCE); - } + when(resourceManagerMock.isGPUDeviceAvailable(offering, host1, virtualMachineProfile.getId())).thenReturn(true); + when(resourceManagerMock.isGPUDeviceAvailable(offering, host2, virtualMachineProfile.getId())).thenReturn(true); - List getHostCapacities() { - CapacityVO cpuCapacity1 = mock(CapacityVO.class); - when(cpuCapacity1.getHostOrPoolId()).thenReturn(1L); - when(cpuCapacity1.getTotalCapacity()).thenReturn(32000L); - when(cpuCapacity1.getReservedCapacity()).thenReturn(0L); - when(cpuCapacity1.getUsedCapacity()).thenReturn(500L); - when(cpuCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 2, true, account); - CapacityVO cpuCapacity2 = mock(CapacityVO.class); - when(cpuCapacity2.getHostOrPoolId()).thenReturn(2L); - when(cpuCapacity2.getTotalCapacity()).thenReturn(32000L); - when(cpuCapacity2.getReservedCapacity()).thenReturn(0L); - when(cpuCapacity2.getUsedCapacity()).thenReturn(500L); - when(cpuCapacity2.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + // Only host1 should be returned + Assert.assertEquals(1, result.size()); + Assert.assertTrue(result.contains(host1)); + Assert.assertFalse(result.contains(host2)); + } - CapacityVO memCapacity1 = mock(CapacityVO.class); - when(memCapacity1.getHostOrPoolId()).thenReturn(1L); - when(memCapacity1.getTotalCapacity()).thenReturn(8589934592L); - when(memCapacity1.getReservedCapacity()).thenReturn(0L); - when(memCapacity1.getUsedCapacity()).thenReturn(536870912L); - when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + @Test + public void testAllocateTo_AvoidSetAndGuestLimit() { + List inputHosts = Arrays.asList(host1, host2); - CapacityVO memCapacity2 = mock(CapacityVO.class); - when(memCapacity2.getHostOrPoolId()).thenReturn(2L); - when(memCapacity2.getTotalCapacity()).thenReturn(8589934592L); - when(memCapacity2.getReservedCapacity()).thenReturn(0L); - when(memCapacity2.getUsedCapacity()).thenReturn(1073741824L); - when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); - return Arrays.asList(cpuCapacity1, memCapacity1, cpuCapacity2, memCapacity2); - } + when(avoid.shouldAvoid(host1)).thenReturn(true); // Avoided + when(avoid.shouldAvoid(host2)).thenReturn(false); + + when(capacityMgr.checkIfHostReachMaxGuestLimit(host2)).thenReturn(true); // Reached limit + + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 2, true, account); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testAllocateTo_GPUNotAvailable() { + List inputHosts = Arrays.asList(host1); + when(avoid.shouldAvoid(host1)).thenReturn(false); + when(resourceManagerMock.isGPUDeviceAvailable(offering, host1, virtualMachineProfile.getId())).thenReturn(false); + + List result = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, offering, null, avoid, inputHosts, 1, true, account); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void allocateToTestHostTypeStorageShouldReturnNull() { + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsEmptyShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(emptyList).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void allocateToTestSuitableHostsNotEmptyShouldCallAllocateToMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(VirtualMachineProfile.class), Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + List suitableHosts = firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).allocateTo(virtualMachineProfile, deploymentPlan, serviceOffering, vmTemplateVO, excludeList, hosts, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity, account); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void allocateToTestProvidedHostsNotNullShouldCallAddHostsToAvoidSetMethod() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(account).when(virtualMachineProfile).getOwner(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(templateTag).when(vmTemplateVO).getTemplateTag(); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).retrieveHosts(Mockito.any(VirtualMachineProfile.class), Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(hosts).when(firstFitAllocatorSpy).allocateTo(Mockito.any(VirtualMachineProfile.class), Mockito.any(DeploymentPlan.class), Mockito.any(ServiceOffering.class), Mockito.any(VMTemplateVO.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyList(), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.any(Account.class)); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + firstFitAllocatorSpy.allocateTo(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).addHostsToAvoidSet(Mockito.any(Host.Type.class), Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndHaTagNotNullShouldReturnOnlyHostsWithHaTag() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNotNullAndHaTagNotNullShouldReturnOnlyHostsToFilterWithHaTag() { + List hostsToFilter = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List hostsWithHaTag = new ArrayList<>(Arrays.asList(host1, host2)); + String hostVmTag = "haVmTag"; + + Mockito.doReturn(hostVmTag).when(virtualMachineProfile).getParameter(Mockito.any(VirtualMachineProfile.Param.class)); + Mockito.doReturn(hostsWithHaTag).when(hostDaoMock).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, hostsToFilter, clusterId, podId, dcId, hostTag, templateTag); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagAndNoHostTagShouldReturnOnlyAllUpAndEnabledNonHaHosts() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List upAndEnabledHostsWithNoHa = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(upAndEnabledHostsWithNoHa).when(resourceManagerMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.nullable(String.class), Mockito.anyList()); + List resultHosts = firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, null, null); + + Assert.assertEquals(2, resultHosts.size()); + Assert.assertEquals(host1, resultHosts.get(0)); + Assert.assertEquals(host2, resultHosts.get(1)); + } + + @Test + public void retrieveHostsTestHostsToFilterIsNullAndNoHaTagWithHostTagShouldCallRetainHostsMatchingServiceOfferingAndTemplateTags() { + List allUpAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(allUpAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doNothing().when(firstFitAllocatorSpy).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doNothing().when(firstFitAllocatorSpy).filterHostsWithUefiEnabled(Mockito.any(Host.Type.class), Mockito.any(VirtualMachineProfile.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyList()); + Mockito.doNothing().when(firstFitAllocatorSpy).addHostsBasedOnTagRules(Mockito.anyString(), Mockito.anyList()); + firstFitAllocatorSpy.retrieveHosts(virtualMachineProfile, type, emptyList, clusterId, podId, dcId, hostTag, templateTag); + + Mockito.verify(firstFitAllocatorSpy, Mockito.times(1)).retainHostsMatchingServiceOfferingAndTemplateTags(Mockito.anyList(), Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + } + + @Test + public void addHostsToAvoidSetTestAllHostsWereConsideredForAllocationShouldNotAddAnyHostToTheAvoidSet() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(suitableHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, suitableHosts); + + Assert.assertTrue(excludeList.getHostsToAvoid().isEmpty()); + } + + @Test + public void addHostsToAvoidSetTestNotAllHostsWereConsideredForAllocationShouldAddHostToTheAvoidSet() { + List allUpAndEnabledNonHAHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List consideredHosts = new ArrayList<>(Arrays.asList(host2, host3)); + + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(allUpAndEnabledNonHAHosts).when(hostDaoMock).listAllUpAndEnabledNonHAHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.nullable(String.class)); + firstFitAllocatorSpy.addHostsToAvoidSet(type, excludeList, clusterId, podId, dcId, consideredHosts); + + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void filterHostsWithUefiEnabledTestNoDetailWithUefiShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + VMInstanceDetailVO userVmDetailVO = null; + + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithInvalidModeShouldNotFilterAnyHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = "Invalid mode"; + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithLegacyModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = ApiConstants.BootMode.LEGACY.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void filterHostsWithUefiEnabledTestDetailWithUefiWithSecureModeShouldFilterHost() { + List suitableHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List uefiHosts = new ArrayList<>(Arrays.asList(host2, host3)); + VMInstanceDetailVO userVmDetailVO = mock(VMInstanceDetailVO.class); + String bootMode = ApiConstants.BootMode.SECURE.toString(); + + Mockito.doReturn(bootMode).when(userVmDetailVO).getValue(); + Mockito.doReturn(userVmDetailVO).when(userVmDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(uefiHosts).when(hostDaoMock).listByHostCapability(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.filterHostsWithUefiEnabled(type, virtualMachineProfile, clusterId, podId, dcId, suitableHosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedButHostDoesNotHaveItShouldReturnTrue() { + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doCallRealMethod().when(excludeList).addHost(Mockito.anyLong()); + Mockito.doCallRealMethod().when(excludeList).getHostsToAvoid(); + Mockito.doReturn(false).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.any(ServiceOffering.class), Mockito.any(Host.class), Mockito.any(Long.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, virtualMachineProfile, excludeList, host1); + + Assert.assertTrue(result); + Assert.assertEquals(1, excludeList.getHostsToAvoid().size()); + Assert.assertTrue(excludeList.getHostsToAvoid().contains(1L)); + } + + @Test + public void offeringRequestedVGpuAndHostDoesNotHaveItTestVGpuRequestedAndHostDoesHaveItShouldReturnFalse() { + Mockito.doReturn(true).when(resourceManagerMock).isGPUDeviceAvailable(Mockito.any(ServiceOffering.class), Mockito.any(Host.class), Mockito.any(Long.class)); + boolean result = firstFitAllocatorSpy.offeringRequestedVGpuAndHostDoesNotHaveIt(serviceOffering, virtualMachineProfile, excludeList, host1); + + Assert.assertFalse(result); + Mockito.verify(excludeList, Mockito.never()).addHost(Mockito.anyLong()); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateDoesNotRequireHvm() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(3, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host2, suitableHosts.get(1)); + Assert.assertEquals(host3, suitableHosts.get(2)); + } + + @Test + public void filterHostWithNoHvmIfTemplateRequestedTestTemplateRequiresHvmShouldReturnOnlyHvmHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + List suitableHosts = firstFitAllocatorSpy.filterHostWithNoHvmIfTemplateRequested(vmTemplateVO, hosts); + + Assert.assertEquals(2, suitableHosts.size()); + Assert.assertEquals(host1, suitableHosts.get(0)); + Assert.assertEquals(host3, suitableHosts.get(1)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingRequestedVGpuViaDetailShouldDoNothing() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + ServiceOfferingDetailsVO requestedVGpuType = mock(ServiceOfferingDetailsVO.class); + + Mockito.doReturn(requestedVGpuType).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, hosts); + + Assert.assertEquals(3, hosts.size()); + Assert.assertEquals(host1, hosts.get(0)); + Assert.assertEquals(host2, hosts.get(1)); + Assert.assertEquals(host3, hosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingRequestedVGpuViaProfileIdShouldDoNothing() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(1L).when(serviceOffering).getVgpuProfileId(); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, hosts); + + Assert.assertEquals(3, hosts.size()); + Assert.assertEquals(host1, hosts.get(0)); + Assert.assertEquals(host2, hosts.get(1)); + Assert.assertEquals(host3, hosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldReorderList() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(null).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(null).when(serviceOffering).getVgpuProfileId(); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(true).when(resourceManagerMock).isHostGpuEnabled(1L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(2L); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(3L); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host2, allHosts.get(0)); + Assert.assertEquals(host3, allHosts.get(1)); + Assert.assertEquals(host1, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByGpuEnabledTestServiceOfferingDidNotRequestVGpuShouldNotReorderListIfThereIsNoHostWithVGpu() { + List allHosts = new ArrayList<>(Arrays.asList(host1, host2, host3)); + + Mockito.doReturn(null).when(serviceOfferingDetailsDao).findDetail(Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(null).when(serviceOffering).getVgpuProfileId(); + Mockito.doReturn(1L).when(host1).getId(); + Mockito.doReturn(2L).when(host2).getId(); + Mockito.doReturn(3L).when(host3).getId(); + Mockito.doReturn(false).when(resourceManagerMock).isHostGpuEnabled(Mockito.anyLong()); + firstFitAllocatorSpy.prioritizeHostsByGpuEnabled(serviceOffering, allHosts); + + Assert.assertEquals(3, allHosts.size()); + Assert.assertEquals(host1, allHosts.get(0)); + Assert.assertEquals(host2, allHosts.get(1)); + Assert.assertEquals(host3, allHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateDidNotRequestedHvmShouldPutHostThatDoesNotSupportHvmInStartOfThePriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(false).when(vmTemplateVO).isRequiresHvm(); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host1); + Mockito.doReturn(false).when(firstFitAllocatorSpy).hostSupportsHVM(host2); + Mockito.doReturn(true).when(firstFitAllocatorSpy).hostSupportsHVM(host3); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host2, prioritizedHosts.get(0)); + Assert.assertEquals(host1, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsByHvmCapabilityTestTemplateRequiresHvmShouldNotReorderList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List prioritizedHosts = new ArrayList<>(); + + Mockito.doReturn(true).when(vmTemplateVO).isRequiresHvm(); + firstFitAllocatorSpy.prioritizeHostsByHvmCapability(vmTemplateVO, hostsToCheck, prioritizedHosts); + + Assert.assertEquals(3, prioritizedHosts.size()); + Assert.assertEquals(host1, prioritizedHosts.get(0)); + Assert.assertEquals(host2, prioritizedHosts.get(1)); + Assert.assertEquals(host3, prioritizedHosts.get(2)); + } + + @Test + public void prioritizeHostsWithMatchingGuestOsTestShouldPutMatchingHostInHighPriorityAndHostsThatDoesNotMatchInLowPriorityList() { + List hostsToCheck = new ArrayList<>(Arrays.asList(host1, host2, host3)); + List highPriorityHosts = new ArrayList<>(); + List lowPriorityHosts = new ArrayList<>(); + String guestOsCategory1 = "guestOsCategory1"; + String guestOsCategory2 = "guestOsCategory2"; + + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getTemplateGuestOSCategory(vmTemplateVO); + Mockito.doReturn(guestOsCategory1).when(firstFitAllocatorSpy).getHostGuestOSCategory(host1); + Mockito.doReturn(guestOsCategory2).when(firstFitAllocatorSpy).getHostGuestOSCategory(host2); + Mockito.doReturn(null).when(firstFitAllocatorSpy).getHostGuestOSCategory(host3); + firstFitAllocatorSpy.prioritizeHostsWithMatchingGuestOs(vmTemplateVO,hostsToCheck, highPriorityHosts, lowPriorityHosts); + + Assert.assertEquals(1, highPriorityHosts.size()); + Assert.assertEquals(host1, highPriorityHosts.get(0)); + Assert.assertEquals(1, lowPriorityHosts.size()); + Assert.assertEquals(host2, lowPriorityHosts.get(0)); + } + + @Test + public void testHostByCombinedCapacityOrder() { + // Test scenario 1: Default capacity usage (0.5 weight) + List mockCapacity = getHostCapacities(); + Map hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.5); + + // Verify host ordering and capacity values + Long firstHostId = hostByCombinedCapacity.keySet().iterator().next(); + Assert.assertEquals("Host with ID 1 should be first in ordering", Long.valueOf(1L), firstHostId); + Assert.assertEquals("Host 1 combined capacity should match expected value", + 0.9609375, hostByCombinedCapacity.get(1L), TOLERANCE); + Assert.assertEquals("Host 2 combined capacity should match expected value", + 0.9296875, hostByCombinedCapacity.get(2L), TOLERANCE); + + // Test scenario 2: Modified capacity usage (0.7 weight) + when(mockCapacity.get(0).getUsedCapacity()).thenReturn(1500L); + hostByCombinedCapacity = FirstFitAllocator.getHostByCombinedCapacities(mockCapacity, 0.7); + + // Verify new ordering after capacity change + firstHostId = hostByCombinedCapacity.keySet().iterator().next(); + Assert.assertEquals("Host with ID 2 should be first after capacity change", Long.valueOf(2L), firstHostId); + Assert.assertEquals("Host 2 combined capacity should match expected value after change", + 0.9515625, hostByCombinedCapacity.get(2L), TOLERANCE); + Assert.assertEquals("Host 1 combined capacity should match expected value after change", + 0.9484375, hostByCombinedCapacity.get(1L), TOLERANCE); + } + + List getHostCapacities() { + CapacityVO cpuCapacity1 = mock(CapacityVO.class); + when(cpuCapacity1.getHostOrPoolId()).thenReturn(1L); + when(cpuCapacity1.getTotalCapacity()).thenReturn(32000L); + when(cpuCapacity1.getReservedCapacity()).thenReturn(0L); + when(cpuCapacity1.getUsedCapacity()).thenReturn(500L); + when(cpuCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + + CapacityVO cpuCapacity2 = mock(CapacityVO.class); + when(cpuCapacity2.getHostOrPoolId()).thenReturn(2L); + when(cpuCapacity2.getTotalCapacity()).thenReturn(32000L); + when(cpuCapacity2.getReservedCapacity()).thenReturn(0L); + when(cpuCapacity2.getUsedCapacity()).thenReturn(500L); + when(cpuCapacity2.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_CPU); + + CapacityVO memCapacity1 = mock(CapacityVO.class); + when(memCapacity1.getHostOrPoolId()).thenReturn(1L); + when(memCapacity1.getTotalCapacity()).thenReturn(8589934592L); + when(memCapacity1.getReservedCapacity()).thenReturn(0L); + when(memCapacity1.getUsedCapacity()).thenReturn(536870912L); + when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + + CapacityVO memCapacity2 = mock(CapacityVO.class); + when(memCapacity2.getHostOrPoolId()).thenReturn(2L); + when(memCapacity2.getTotalCapacity()).thenReturn(8589934592L); + when(memCapacity2.getReservedCapacity()).thenReturn(0L); + when(memCapacity2.getUsedCapacity()).thenReturn(1073741824L); + when(memCapacity1.getCapacityType()).thenReturn(CapacityVO.CAPACITY_TYPE_MEMORY); + return Arrays.asList(cpuCapacity1, memCapacity1, cpuCapacity2, memCapacity2); + } } diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java new file mode 100644 index 00000000000..ab9b322eb84 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java @@ -0,0 +1,332 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.cloud.agent.manager.allocator.HostAllocator; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DeploymentPlanner; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.vm.VirtualMachineProfile; +import org.apache.commons.collections.CollectionUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; + +@RunWith(MockitoJUnitRunner.class) +public class RandomAllocatorTest { + + @Mock + HostDao hostDao; + + @Spy + @InjectMocks + RandomAllocator randomAllocator; + + @Mock + ResourceManager resourceManagerMock; + + private final Host.Type type = Host.Type.Routing; + + private final Long clusterId = 1L; + + private final Long podId = 2L; + + private final Long zoneId = 3L; + + private final List emptyList = new ArrayList<>(); + + private final String hostTag = "hostTag"; + + private final HostVO host1 = Mockito.mock(HostVO.class); + + private final HostVO host2 = Mockito.mock(HostVO.class); + + private final HostVO host3 = Mockito.mock(HostVO.class); + + private final VMTemplateVO vmTemplateVO = Mockito.mock(VMTemplateVO.class); + + private final ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + + private final DeploymentPlanner.ExcludeList excludeList = Mockito.mock(DeploymentPlanner.ExcludeList.class); + + private final VirtualMachineProfile virtualMachineProfile = Mockito.mock(VirtualMachineProfile.class); + + private final DeploymentPlan deploymentPlan = Mockito.mock(DeploymentPlan.class); + + private final boolean considerReservedCapacity = true; + + + @Test + public void testListHostsByTags() { + Host.Type type = Host.Type.Routing; + Long id = 1L; + String templateTag = "tag1"; + String offeringTag = "tag2"; + HostVO host1 = Mockito.mock(HostVO.class); + HostVO host2 = Mockito.mock(HostVO.class); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, offeringTag)).thenReturn(List.of(host1, host2)); + + // No template tagged host + ArrayList noTemplateTaggedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(new ArrayList<>()); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTaggedHosts, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(noTemplateTaggedHosts)); + + // Different template tagged host + ArrayList differentTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); + HostVO host3 = Mockito.mock(HostVO.class); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(List.of(host3)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(differentTemplateTaggedHost, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertTrue(CollectionUtils.isEmpty(differentTemplateTaggedHost)); + + // Matching template tagged host + ArrayList matchingTemplateTaggedHost = new ArrayList<>(Arrays.asList(host1, host2)); + Mockito.when(hostDao.listByHostTag(type, clusterId, podId, zoneId, templateTag)).thenReturn(List.of(host1)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(matchingTemplateTaggedHost, type, zoneId, podId, clusterId, offeringTag, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(matchingTemplateTaggedHost)); + Assert.assertEquals(1, matchingTemplateTaggedHost.size()); + + // No template tag + ArrayList noTemplateTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noTemplateTag, type, zoneId, podId, clusterId, offeringTag, null); + Assert.assertFalse(CollectionUtils.isEmpty(noTemplateTag)); + Assert.assertEquals(2, noTemplateTag.size()); + + // No offering tag + ArrayList noOfferingTag = new ArrayList<>(Arrays.asList(host1, host2)); + randomAllocator.retainHostsMatchingServiceOfferingAndTemplateTags(noOfferingTag, type, zoneId, podId, clusterId, null, templateTag); + Assert.assertFalse(CollectionUtils.isEmpty(noOfferingTag)); + Assert.assertEquals(1, noOfferingTag.size()); + } + + @Test + public void findSuitableHostsTestHostTypeStorageShouldReturnNull() { + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, Host.Type.Storage, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestNoAvailableHostsShouldReturnNull() { + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(emptyList).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Assert.assertNull(suitableHosts); + } + + @Test + public void findSuitableHostsTestAvailableHostsShouldCallFilterAvailableHostsOnce() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + + Mockito.doReturn(serviceOffering).when(virtualMachineProfile).getServiceOffering(); + Mockito.doReturn(vmTemplateVO).when(virtualMachineProfile).getTemplate(); + Mockito.doReturn(hostTag).when(serviceOffering).getHostTag(); + Mockito.doReturn(hosts).when(randomAllocator).retrieveHosts(Mockito.any(Host.Type.class), Mockito.nullable(List.class), Mockito.any(VMTemplateVO.class), Mockito.anyString(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hosts).when(randomAllocator).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + List suitableHosts = randomAllocator.findSuitableHosts(virtualMachineProfile, deploymentPlan, type, excludeList, null, HostAllocator.RETURN_UPTO_ALL, considerReservedCapacity); + + Mockito.verify(randomAllocator, Mockito.times(1)).filterAvailableHosts(Mockito.any(DeploymentPlanner.ExcludeList.class), Mockito.anyInt(), Mockito.anyBoolean(), Mockito.anyList(), Mockito.any(ServiceOffering.class)); + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestAvailableHostsReachedReturnUpToLimitShouldReturnOnlyHostsWithinLimit() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = 1; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestReturnUpToAllShouldReturnAllAvailableHosts() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(2, suitableHosts.size()); + } + + @Test + public void filterAvailableHostsTestHost1InAvoidShouldOnlyReturnHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(true).when(excludeList).shouldAvoid(host1); + Mockito.doReturn(false).when(excludeList).shouldAvoid(host2); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(Mockito.anyBoolean(), Mockito.any(ServiceOffering.class), Mockito.any(Host.class)); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void filterAvailableHostsTestOnlyHost2HasCpuCapacityAndCapabilityShouldReturnOnlyHost2() { + List hosts = new ArrayList<>(Arrays.asList(host1, host2)); + int returnUpTo = HostAllocator.RETURN_UPTO_ALL; + + Mockito.doReturn(false).when(excludeList).shouldAvoid(Mockito.any(Host.class)); + Mockito.doReturn(false).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host1); + Mockito.doReturn(true).when(randomAllocator).hostHasCpuCapabilityAndCapacity(considerReservedCapacity, serviceOffering, host2); + List suitableHosts = randomAllocator.filterAvailableHosts(excludeList, returnUpTo, considerReservedCapacity, hosts, serviceOffering); + + Assert.assertEquals(1, suitableHosts.size()); + Assert.assertEquals(host2, suitableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List upAndEnabledHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(upAndEnabledHosts).when(resourceManagerMock).listAllUpAndEnabledHosts(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, null, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndNoHostTagAndNoTagRuleShouldOnlyReturnHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullAndOnlyHostTagsRulesShouldReturnHostsThatMatchRuleTagsAndHostsWithNoTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithNoRuleTagsAndHostTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host2)); + + Mockito.doReturn(hostsWithNoRuleTagsAndHostTags).when(hostDao).listAllHostsThatHaveNoRuleTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, null, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host2, availableHosts.get(1)); + } + + @Test + public void retrieveHostsTestProvidedHostsNotNullProvidedHostTagsNotNullAndNoHostWithMatchingRuleTagsShouldReturnHostWithMatchingTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(emptyList).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(1, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + } + + @Test + public void retrieveHostsTestProvidedHostsNullNotProvidedHostTagsNotNullAndHostWithMatchingRuleTagsShouldReturnHostWithHostMatchingTagsAndRuleTags() { + List providedHosts = new ArrayList<>(Arrays.asList(host1, host2)); + List hostsWithMatchingTags = new ArrayList<>(Arrays.asList(host1)); + List hostsMatchingRuleTags = new ArrayList<>(Arrays.asList(host3)); + + Mockito.doReturn(hostsWithMatchingTags).when(hostDao).listByHostTag(Mockito.any(Host.Type.class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + Mockito.doReturn(hostsMatchingRuleTags).when(hostDao).findHostsWithTagRuleThatMatchComputeOfferingTags(Mockito.nullable(String.class)); + List availableHosts = randomAllocator.retrieveHosts(type, providedHosts, vmTemplateVO, hostTag, clusterId, podId, zoneId); + + Assert.assertEquals(2, availableHosts.size()); + Assert.assertEquals(host1, availableHosts.get(0)); + Assert.assertEquals(host3, availableHosts.get(1)); + } +} diff --git a/test/integration/smoke/test_vm_strict_host_tags.py b/test/integration/smoke/test_vm_strict_host_tags.py index 2377e9a7618..aac3e1ea65f 100644 --- a/test/integration/smoke/test_vm_strict_host_tags.py +++ b/test/integration/smoke/test_vm_strict_host_tags.py @@ -423,7 +423,7 @@ class TestRestoreVMStrictTags(cloudstackTestCase): vm.restore(self.apiclient, templateid=self.template_t2.id, expunge=True) self.fail("VM should not be restored") except Exception as e: - self.assertTrue("Unable to start VM with specified id" in str(e)) + self.assertTrue("Unable to create a deployment for " in str(e)) class TestMigrateVMStrictTags(cloudstackTestCase): From 1f5dba9bd2d33e95c63cc04b57df51b5c7921630 Mon Sep 17 00:00:00 2001 From: Fabricio Duarte Date: Thu, 30 Apr 2026 12:22:35 -0300 Subject: [PATCH 07/10] Release reserved storage resources on VM deployment failure (#13048) --- .../db/schema-42200to42210-cleanup.sql | 6 +++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 24 ++++++------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 54baf226ac4..2f104568c14 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,3 +18,9 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; + +-- Entries remaining on `cloud`.`resource_reservation` during the upgrade process are stale, so delete them. +-- This script was added to normalize volume/primary storage reservations that got stuck due to a bug on VM deployment, +-- but it is more interesting to introduce a smarter logic to clean these stale reservations in the future without the need +-- for upgrades (for instance, by having a heartbeat_time column for the reservations and automatically cleaning old entries). +DELETE FROM `cloud`.`resource_reservation`; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index fe18263fd3b..b23291586ef 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4301,9 +4301,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return resourceLimitService.getResourceLimitStorageTags(diskOfferingVO); } - private List reserveStorageResourcesForVm(Account owner, Long diskOfferingId, Long diskSize, List dataDiskInfoList, Long rootDiskOfferingId, ServiceOfferingVO offering, Long rootDiskSize) throws ResourceAllocationException { - List checkedReservations = new ArrayList<>(); - + private void reserveStorageResourcesForVm(List checkedReservations, Account owner, Long diskOfferingId, Long diskSize, List dataDiskInfoList, Long rootDiskOfferingId, ServiceOfferingVO offering, Long rootDiskSize) throws ResourceAllocationException { List rootResourceLimitStorageTags = getResourceLimitStorageTags(rootDiskOfferingId != null ? rootDiskOfferingId : offering.getDiskOfferingId()); CheckedReservation rootVolumeReservation = new CheckedReservation(owner, ResourceType.volume, rootResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); checkedReservations.add(rootVolumeReservation); @@ -4311,12 +4309,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir checkedReservations.add(rootPrimaryStorageReservation); if (diskOfferingId != null) { - List additionalResourceLimitStorageTags = diskOfferingId != null ? getResourceLimitStorageTags(diskOfferingId) : null; + List additionalResourceLimitStorageTags = getResourceLimitStorageTags(diskOfferingId); DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); Long size = verifyAndGetDiskSize(diskOffering, diskSize); - CheckedReservation additionalVolumeReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService) : null; + CheckedReservation additionalVolumeReservation = new CheckedReservation(owner, ResourceType.volume, additionalResourceLimitStorageTags, 1L, reservationDao, resourceLimitService); checkedReservations.add(additionalVolumeReservation); - CheckedReservation additionalPrimaryStorageReservation = diskOfferingId != null ? new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, size, reservationDao, resourceLimitService) : null; + CheckedReservation additionalPrimaryStorageReservation = new CheckedReservation(owner, ResourceType.primary_storage, additionalResourceLimitStorageTags, size, reservationDao, resourceLimitService); checkedReservations.add(additionalPrimaryStorageReservation); } @@ -4332,7 +4330,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir checkedReservations.add(additionalPrimaryStorageReservation); } } - return checkedReservations; } private UserVm getUncheckedUserVmResource(DataCenter zone, String hostName, String displayName, Account owner, @@ -4344,10 +4341,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Map userVmOVFPropertiesMap, boolean dynamicScalingEnabled, String vmType, VMTemplateVO template, HypervisorType hypervisorType, long accountId, ServiceOfferingVO offering, boolean isIso, Long rootDiskOfferingId, long volumesSize, Volume volume, Snapshot snapshot) throws ResourceAllocationException { - List checkedReservations = new ArrayList<>(); + List checkedReservations = new ArrayList<>(); try { - checkedReservations = reserveStorageResourcesForVm(owner, diskOfferingId, diskSize, dataDiskInfoList, rootDiskOfferingId, offering, volumesSize); + reserveStorageResourcesForVm(checkedReservations, owner, diskOfferingId, diskSize, dataDiskInfoList, rootDiskOfferingId, offering, volumesSize); // verify security group ids if (securityGroupIdList != null) { @@ -4638,14 +4635,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir logger.error("error during resource reservation and allocation", e); throw new CloudRuntimeException(e); } finally { - for (CheckedReservation checkedReservation : checkedReservations) { - try { - checkedReservation.close(); - } catch (Exception e) { - logger.error("error during resource reservation and allocation", e); - throw new CloudRuntimeException(e); - } - } + ReservationHelper.closeAll(checkedReservations); } } From c07f1fd5d29ab2c6ba3a419dbf6d21b0a4f3900b Mon Sep 17 00:00:00 2001 From: Henrique Sato Date: Fri, 1 May 2026 06:54:40 -0300 Subject: [PATCH 08/10] Number of running and stopped VMs as preset variables for `Network` type Quota tariffs (#11689) Co-authored-by: Fabricio Duarte --- .../presetvariables/PresetVariableHelper.java | 14 ++++- .../presetvariables/ResourceCounting.java | 52 +++++++++++++++++++ .../activationrule/presetvariables/Value.java | 11 ++++ .../PresetVariableHelperTest.java | 37 +++++++++++-- .../response/QuotaResponseBuilderImpl.java | 3 +- 5 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index cd87052e878..23020292027 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -35,6 +35,7 @@ import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -783,10 +784,21 @@ public class PresetVariableHelper { value.setId(network.getUuid()); value.setName(network.getName()); value.setState(usageRecord.getState()); - + value.setResourceCounting(getPresetVariableValueNetworkResourceCounting(networkId)); value.setNetworkOffering(getPresetVariableValueNetworkOffering(network.getNetworkOfferingId())); } + protected ResourceCounting getPresetVariableValueNetworkResourceCounting(Long networkId) { + ResourceCounting resourceCounting = new ResourceCounting(); + List vmInstancesVO = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(networkId, VirtualMachine.Type.User); + int runningVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Running)).count(); + int stoppedVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Stopped)).count(); + + resourceCounting.setRunningVms(runningVms); + resourceCounting.setStoppedVms(stoppedVms); + return resourceCounting; + } + protected GenericPresetVariable getPresetVariableValueNetworkOffering(Long networkOfferingId) { NetworkOfferingVO networkOfferingVo = networkOfferingDao.findByIdIncludingRemoved(networkOfferingId); validateIfObjectIsNull(networkOfferingVo, networkOfferingId, "network offering"); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java new file mode 100644 index 00000000000..75049c3486a --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + + +import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class ResourceCounting { + + @PresetVariableDefinition(description = "The number of running user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int runningVms; + @PresetVariableDefinition(description = "The number of stopped user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int stoppedVms; + + public int getRunningVms() { + return runningVms; + } + + public void setRunningVms(int runningVms) { + this.runningVms = runningVms; + } + + public int getStoppedVms() { + return stoppedVms; + } + + public void setStoppedVms(int stoppedVms) { + this.stoppedVms = stoppedVms; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index aff2c040e74..286fe5c60fd 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -84,6 +84,9 @@ public class Value extends GenericPresetVariable { @PresetVariableDefinition(description = "Backup offering of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private BackupOffering backupOffering; + @PresetVariableDefinition(description = "The amount of resources of the usage type.") + private ResourceCounting resourceCounting; + @PresetVariableDefinition(description = "The hypervisor where the resource was deployed. Values can be: XenServer, KVM, VMware, Hyperv, BareMetal, Ovm, Ovm3 and LXC.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.VM_SNAPSHOT, QuotaTypes.SNAPSHOT}) private String hypervisorType; @@ -262,6 +265,14 @@ public class Value extends GenericPresetVariable { this.state = state; } + public ResourceCounting getResourceCounting() { + return resourceCounting; + } + + public void setResourceCounting(ResourceCounting resourceCounting) { + this.resourceCounting = resourceCounting; + } + public GenericPresetVariable getNetworkOffering() { return networkOffering; } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index fb093a45679..bcdbc3b46ce 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -31,11 +31,13 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -238,6 +240,8 @@ public class PresetVariableHelperTest { value.setVmSnapshotType(VMSnapshot.Type.Disk.toString()); value.setComputingResources(getComputingResourcesForTests()); value.setVolumeType(Volume.Type.DATADISK.toString()); + value.setState(Network.State.Implemented.toString()); + value.setResourceCounting(getResourceCountingForTests()); value.setNetworkOffering(getNetworkOfferingForTests()); value.setVpcOffering(getVpcOfferingForTests()); return value; @@ -276,6 +280,13 @@ public class PresetVariableHelperTest { return configuration; } + private ResourceCounting getResourceCountingForTests() { + ResourceCounting resourceCounting = new ResourceCounting(); + resourceCounting.setRunningVms(1); + resourceCounting.setStoppedVms(1); + return resourceCounting; + } + private List getHostTagsForTests() { return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); } @@ -1343,8 +1354,8 @@ public class PresetVariableHelperTest { Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); Mockito.doReturn(expected.getState()).when(usageVoMock).getState(); + Mockito.doReturn(expected.getResourceCounting()).when(presetVariableHelperSpy).getPresetVariableValueNetworkResourceCounting(Mockito.anyLong()); Mockito.doReturn(expected.getNetworkOffering()).when(presetVariableHelperSpy).getPresetVariableValueNetworkOffering(Mockito.anyLong()); - Mockito.doReturn(UsageTypes.NETWORK).when(usageVoMock).getUsageType(); Value result = new Value(); @@ -1352,7 +1363,27 @@ public class PresetVariableHelperTest { assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getState(), result.getState()); - Assert.assertEquals(expected.getNetworkOffering(), result.getNetworkOffering()); + Assert.assertEquals(expected.getResourceCounting(), result.getResourceCounting()); + } + + @Test + public void getPresetVariableValueNetworkResourceCountingTestSetValueAndReturnObject() { + VMInstanceVO vmInstanceVoMock1 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock1.setState(VirtualMachine.State.Stopped); + + VMInstanceVO vmInstanceVoMock2 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock2.setState(VirtualMachine.State.Running); + + Mockito.doReturn(List.of(vmInstanceVoMock1, vmInstanceVoMock2)).when(vmInstanceDaoMock).listNonRemovedVmsByTypeAndNetwork(Mockito.anyLong(), Mockito.any()); + + mockMethodValidateIfObjectIsNull(); + + ResourceCounting expected = getResourceCountingForTests(); + + ResourceCounting result = presetVariableHelperSpy.getPresetVariableValueNetworkResourceCounting(1L); + + Assert.assertEquals(expected.getRunningVms(), result.getRunningVms()); + Assert.assertEquals(expected.getStoppedVms(), result.getStoppedVms()); } @Test @@ -1362,7 +1393,7 @@ public class PresetVariableHelperTest { presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, null); }); - Mockito.verifyNoInteractions(networkDaoMock); + Mockito.verifyNoInteractions(vpcDaoMock); } @Test diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 173c0723731..c919bb5887c 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -98,6 +98,7 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.ComputingResou import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.ResourceCounting; import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; @@ -185,7 +186,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private VolumeDao volumeDao; - private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class, ResourceCounting.class}; private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); From a17bff9ba8770349c3b8344cb45778e89d4379f4 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 3 May 2026 22:09:41 +0530 Subject: [PATCH 09/10] ui: fix webhook filters listing (#13068) --- ui/public/locales/en.json | 1 + ui/src/components/view/ListView.vue | 2 +- ui/src/components/view/WebhookFiltersTab.vue | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 4a23b454252..e2045eba8df 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -4167,6 +4167,7 @@ "message.warn.select.template": "Please select a Template for Registration.", "message.warn.zone.mtu.update": "Please note that this limit won't affect pre-existing Network's MTU settings", "message.webhook.deliveries.time.filter": "Webhook deliveries list can be filtered based on date-time. Select 'Custom' for specifying start and end date range.", +"message.webhook.filter.add": "Webhook deliveries can be controlled using filters (currently by Event type). Please select the parameters to add to the applied filters list.", "message.zone.creation.complete": "Zone creation complete.", "message.zone.detail.description": "Populate Zone details.", "message.zone.detail.hint": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 66dd6b3db9e..b03293efaca 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -949,7 +949,7 @@ style="margin-left: 5px" :actions="actions" :resource="record" - :enabled="quickViewEnabled() && actions.length > 0" + :enabled="quickViewEnabled(actions, columns, column.key)" @exec-action="$parent.execAction" /> diff --git a/ui/src/components/view/WebhookFiltersTab.vue b/ui/src/components/view/WebhookFiltersTab.vue index 2d276e2a3cf..513720a8625 100644 --- a/ui/src/components/view/WebhookFiltersTab.vue +++ b/ui/src/components/view/WebhookFiltersTab.vue @@ -18,6 +18,7 @@