From 35e809e7ce4cc7504ae8438bbde4592f9dc7ba2b Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Mon, 17 Mar 2025 22:16:05 +0530 Subject: [PATCH 01/42] Set external Id to null after backupProvider.removeVMFromBackup (#10562) --- .../org/apache/cloudstack/backup/VeeamBackupProvider.java | 3 +++ .../org/apache/cloudstack/backup/BackupManagerImpl.java | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index e20f67995b9..e8b65e19e0b 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -199,6 +199,9 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, public boolean removeVMFromBackupOffering(final VirtualMachine vm) { final VeeamClient client = getClient(vm.getDataCenterId()); final VmwareDatacenter vmwareDC = findVmwareDatacenterForVM(vm); + if (vm.getBackupExternalId() == null) { + throw new CloudRuntimeException("The VM does not have a backup job assigned."); + } try { if (!client.removeVMFromVeeamJob(vm.getBackupExternalId(), vm.getInstanceName(), vmwareDC.getVcenterHost())) { LOG.warn("Failed to remove VM from Veeam Job id: " + vm.getBackupExternalId()); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 36978ab2f87..2fc98973ad1 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -368,10 +368,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { boolean result = false; try { - vm.setBackupOfferingId(null); - vm.setBackupExternalId(null); - vm.setBackupVolumes(null); result = backupProvider.removeVMFromBackupOffering(vm); + vm.setBackupOfferingId(null); + vm.setBackupVolumes(null); + vm.setBackupExternalId(null); if (result && backupProvider.willDeleteBackupsOnOfferingRemoval()) { final List backups = backupDao.listByVmId(null, vm.getId()); for (final Backup backup : backups) { From 02d0dca24b2cac976a6deef181a4774368b4005f Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Mon, 17 Mar 2025 17:14:57 -0400 Subject: [PATCH 02/42] List only VMs associated to a userdata (#10569) * List only VMs associated to a userdata * add since param --- .../cloudstack/api/command/user/vm/ListVMsCmd.java | 8 ++++++++ .../main/java/com/cloud/api/query/QueryManagerImpl.java | 9 +++++++++ ui/src/config/section/compute.js | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java index 37b702e166a..a6c5bafbd51 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java @@ -41,6 +41,7 @@ import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VpcResponse; @@ -151,6 +152,9 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements @Parameter(name = ApiConstants.USER_DATA, type = CommandType.BOOLEAN, description = "Whether to return the VMs' user data or not. By default, user data will not be returned.", since = "4.18.0.0") private Boolean showUserData; + @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, required = false, description = "the instances by userdata", since = "4.20.1") + private Long userdataId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -245,6 +249,10 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements return CollectionUtils.isEmpty(viewDetails); } + public Long getUserdataId() { + return userdataId; + } + public EnumSet getDetails() throws InvalidParameterValueException { if (isViewDetailsEmpty()) { if (_queryService.ReturnVmStatsOnVmList.value()) { diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 16cbd5ebe1b..0e99af41338 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -1267,6 +1267,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long storageId = null; StoragePoolVO pool = null; Long userId = cmd.getUserId(); + Long userdataId = cmd.getUserdataId(); Map tags = cmd.getTags(); boolean isAdmin = false; @@ -1339,6 +1340,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q userVmSearchBuilder.and("templateId", userVmSearchBuilder.entity().getTemplateId(), Op.EQ); } + if (userdataId != null) { + userVmSearchBuilder.and("userdataId", userVmSearchBuilder.entity().getUserDataId(), Op.EQ); + } + if (hypervisor != null) { userVmSearchBuilder.and("hypervisorType", userVmSearchBuilder.entity().getHypervisorType(), Op.EQ); } @@ -1531,6 +1536,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q userVmSearchCriteria.setParameters("templateId", templateId); } + if (userdataId != null) { + userVmSearchCriteria.setParameters("userdataId", userdataId); + } + if (display != null) { userVmSearchCriteria.setParameters("display", display); } diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index ba7b3d75eb8..0d5c8de6628 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -919,7 +919,7 @@ export default { related: [{ name: 'vm', title: 'label.instances', - param: 'userdata' + param: 'userdataid' }], tabs: [ { From 89e6b1f8ea00cc38aa384fd6383a614daf1663b6 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 18 Mar 2025 08:52:11 +0530 Subject: [PATCH 03/42] server: fix npe during start vr edge case (#10366) DeploymentPlanner.addPod takes long value while VmInstanceVO.getPodIdToDeployIn returns a Long value which can be null when the VM is never started. Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/network/router/NetworkHelperImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index 38286b5d4d9..4a1c5b64d21 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -445,7 +445,9 @@ public class NetworkHelperImpl implements NetworkHelper { final int retryIndex = 5; final ExcludeList[] avoids = new ExcludeList[5]; avoids[0] = new ExcludeList(); - avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); + if (routerToBeAvoid.getPodIdToDeployIn() != null) { + avoids[0].addPod(routerToBeAvoid.getPodIdToDeployIn()); + } avoids[1] = new ExcludeList(); avoids[1].addCluster(_hostDao.findById(routerToBeAvoid.getHostId()).getClusterId()); avoids[2] = new ExcludeList(); From 7978141464c60ac7f7a2fc7f29ed10b1108b6547 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 18 Mar 2025 09:20:50 +0100 Subject: [PATCH 04/42] api: fix EntityReference in NetworkResponse.java (#10563) --- .../org/apache/cloudstack/api/response/NetworkResponse.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java index 24f76215d09..d18c0e1266a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java @@ -27,12 +27,11 @@ import org.apache.cloudstack.api.BaseResponseWithAssociatedNetwork; import org.apache.cloudstack.api.EntityReference; import com.cloud.network.Network; -import com.cloud.projects.ProjectAccount; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") -@EntityReference(value = {Network.class, ProjectAccount.class}) +@EntityReference(value = {Network.class}) public class NetworkResponse extends BaseResponseWithAssociatedNetwork implements ControlledEntityResponse, SetResourceIconResponse { @SerializedName(ApiConstants.ID) From f4a7c8ab89ac04aa70350f643f26c3280ff37c9e Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Tue, 18 Mar 2025 13:50:19 +0100 Subject: [PATCH 05/42] linstor: implement missing deleteDatastore (#10561) Somehow deleteDatastore was never implemented, that meant: templates haven't been cleaned up on datastore delete and also agents have never been informed about storage pool removal. --- .../BasePrimaryDataStoreLifeCycleImpl.java | 47 +++++++++++++++++++ plugins/storage/volume/linstor/CHANGELOG.md | 6 +++ .../LinstorPrimaryDataStoreLifeCycleImpl.java | 5 +- .../ScaleIOPrimaryDataStoreLifeCycle.java | 39 ++------------- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java index adc74a77d43..021e49155d3 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/BasePrimaryDataStoreLifeCycleImpl.java @@ -22,8 +22,13 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.template.TemplateManager; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; import org.apache.log4j.Logger; @@ -54,6 +59,10 @@ public class BasePrimaryDataStoreLifeCycleImpl { protected HostDao hostDao; @Inject protected StoragePoolHostDao storagePoolHostDao; + @Inject + private PrimaryDataStoreDao primaryDataStoreDao; + @Inject + private TemplateManager templateMgr; private List getPoolHostsList(ClusterScope clusterScope, HypervisorType hypervisorType) { List hosts; @@ -103,4 +112,42 @@ public class BasePrimaryDataStoreLifeCycleImpl { } dataStoreHelper.switchToCluster(store, clusterScope); } + + private void evictTemplates(StoragePoolVO storagePoolVO) { + List unusedTemplatesInPool = templateMgr.getUnusedTemplatesInPool(storagePoolVO); + for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) { + if (templatePoolVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { + templateMgr.evictTemplateFromStoragePool(templatePoolVO); + } + } + } + + private void deleteAgentStoragePools(StoragePool storagePool) { + List poolHostVOs = storagePoolHostDao.listByPoolId(storagePool.getId()); + for (StoragePoolHostVO poolHostVO : poolHostVOs) { + DeleteStoragePoolCommand deleteStoragePoolCommand = new DeleteStoragePoolCommand(storagePool); + final Answer answer = agentMgr.easySend(poolHostVO.getHostId(), deleteStoragePoolCommand); + if (answer != null && answer.getResult()) { + s_logger.info("Successfully deleted storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId()); + } else { + if (answer != null) { + s_logger.error("Failed to delete storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId() + " , result: " + answer.getResult()); + } else { + s_logger.error("Failed to delete storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId()); + } + } + } + } + + protected boolean cleanupDatastore(DataStore store) { + StoragePool storagePool = (StoragePool)store; + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(storagePool.getId()); + if (storagePoolVO == null) { + return false; + } + + evictTemplates(storagePoolVO); + deleteAgentStoragePools(storagePool); + return true; + } } diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index e27e521bcd8..7e9d754b9f6 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2025-03-13] + +### Fixed + +- Implemented missing delete datastore, to correctly cleanup on datastore removal + ## [2025-02-21] ### Fixed diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java index 0ef97ae4796..cbe7b3f7d81 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java @@ -289,7 +289,10 @@ public class LinstorPrimaryDataStoreLifeCycleImpl extends BasePrimaryDataStoreLi @Override public boolean deleteDataStore(DataStore store) { - return dataStoreHelper.deletePrimaryDataStore(store); + if (cleanupDatastore(store)) { + return dataStoreHelper.deletePrimaryDataStore(store); + } + return false; } /* (non-Javadoc) diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java index c1a7411d29f..7ddd41fb7b6 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java @@ -50,8 +50,6 @@ import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.DeleteStoragePoolCommand; import com.cloud.agent.api.StoragePoolInfo; import com.cloud.capacity.CapacityManager; import com.cloud.dc.ClusterVO; @@ -65,9 +63,6 @@ import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolAutomation; -import com.cloud.storage.StoragePoolHostVO; -import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.template.TemplateManager; import com.cloud.utils.UriUtils; @@ -345,37 +340,11 @@ public class ScaleIOPrimaryDataStoreLifeCycle extends BasePrimaryDataStoreLifeCy @Override public boolean deleteDataStore(DataStore dataStore) { - StoragePool storagePool = (StoragePool)dataStore; - StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(storagePool.getId()); - if (storagePoolVO == null) { - return false; + if (cleanupDatastore(dataStore)) { + ScaleIOGatewayClientConnectionPool.getInstance().removeClient(dataStore.getId()); + return dataStoreHelper.deletePrimaryDataStore(dataStore); } - - List unusedTemplatesInPool = templateMgr.getUnusedTemplatesInPool(storagePoolVO); - for (VMTemplateStoragePoolVO templatePoolVO : unusedTemplatesInPool) { - if (templatePoolVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - templateMgr.evictTemplateFromStoragePool(templatePoolVO); - } - } - - List poolHostVOs = storagePoolHostDao.listByPoolId(dataStore.getId()); - for (StoragePoolHostVO poolHostVO : poolHostVOs) { - DeleteStoragePoolCommand deleteStoragePoolCommand = new DeleteStoragePoolCommand(storagePool); - final Answer answer = agentMgr.easySend(poolHostVO.getHostId(), deleteStoragePoolCommand); - if (answer != null && answer.getResult()) { - LOGGER.info("Successfully deleted storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId()); - } else { - if (answer != null) { - LOGGER.error("Failed to delete storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId() + " , result: " + answer.getResult()); - } else { - LOGGER.error("Failed to delete storage pool: " + storagePool.getId() + " from host: " + poolHostVO.getHostId()); - } - } - } - - ScaleIOGatewayClientConnectionPool.getInstance().removeClient(dataStore.getId()); - - return dataStoreHelper.deletePrimaryDataStore(dataStore); + return false; } @Override From 6c40a7bebbc4c9d4dd9c96769834ad4b9c6cb6c7 Mon Sep 17 00:00:00 2001 From: dahn Date: Tue, 18 Mar 2025 14:53:39 +0100 Subject: [PATCH 06/42] deal with null return for create deployment plan for maintenance (#10518) * deal with null return for create deployment plan for maintenance * deal with ide warnings --- .../cloud/resource/ResourceManagerImpl.java | 219 ++++++++---------- 1 file changed, 92 insertions(+), 127 deletions(-) diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 5ba79c2d7bd..c05776ee6e1 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -67,9 +67,9 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; + import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -99,7 +99,6 @@ import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.cluster.ClusterManager; import com.cloud.configuration.Config; -import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.dc.ClusterVO; @@ -131,7 +130,6 @@ import com.cloud.exception.DiscoveryException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.gpu.GPU; import com.cloud.gpu.HostGpuGroupsVO; @@ -169,7 +167,6 @@ import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StorageService; import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; @@ -201,6 +198,7 @@ import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SSHCmdHelper; import com.cloud.utils.ssh.SshException; import com.cloud.vm.UserVmManager; +import com.cloud.utils.StringUtils; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; @@ -235,8 +233,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Inject private CapacityDao _capacityDao; @Inject - private DiskOfferingDao diskOfferingDao; - @Inject private ServiceOfferingDao serviceOfferingDao; @Inject private HostDao _hostDao; @@ -295,8 +291,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Inject private VMTemplateDao _templateDao; @Inject - private ConfigurationManager _configMgr; - @Inject private ClusterVSMMapDao _clusterVSMMapDao; @Inject private UserVmDetailsDao userVmDetailsDao; @@ -311,9 +305,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, private final long _nodeId = ManagementServerNode.getManagementServerId(); - private final HashMap _resourceStateAdapters = new HashMap(); + private final HashMap _resourceStateAdapters = new HashMap<>(); - private final HashMap> _lifeCycleListeners = new HashMap>(); + private final HashMap> _lifeCycleListeners = new HashMap<>(); private HypervisorType _defaultSystemVMHypervisor; private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 30; // seconds @@ -323,11 +317,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, private SearchBuilder _gpuAvailability; private void insertListener(final Integer event, final ResourceListener listener) { - List lst = _lifeCycleListeners.get(event); - if (lst == null) { - lst = new ArrayList(); - _lifeCycleListeners.put(event, lst); - } + List lst = _lifeCycleListeners.computeIfAbsent(event, k -> new ArrayList<>()); if (lst.contains(listener)) { throw new CloudRuntimeException("Duplicate resource lisener:" + listener.getClass().getSimpleName()); @@ -369,9 +359,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public void unregisterResourceEvent(final ResourceListener listener) { synchronized (_lifeCycleListeners) { - final Iterator it = _lifeCycleListeners.entrySet().iterator(); - while (it.hasNext()) { - final Map.Entry> items = (Map.Entry>)it.next(); + for (Map.Entry> items : _lifeCycleListeners.entrySet()) { final List lst = items.getValue(); lst.remove(listener); } @@ -380,7 +368,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, protected void processResourceEvent(final Integer event, final Object... params) { final List lst = _lifeCycleListeners.get(event); - if (lst == null || lst.size() == 0) { + if (lst == null || lst.isEmpty()) { return; } @@ -421,7 +409,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @DB @Override - public List discoverCluster(final AddClusterCmd cmd) throws IllegalArgumentException, DiscoveryException, ResourceInUseException { + public List discoverCluster(final AddClusterCmd cmd) throws IllegalArgumentException, DiscoveryException { final long dcId = cmd.getZoneId(); final long podId = cmd.getPodId(); final String clusterName = cmd.getClusterName(); @@ -430,10 +418,10 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, final String password = cmd.getPassword(); if (url != null) { - url = URLDecoder.decode(url); + url = URLDecoder.decode(url, com.cloud.utils.StringUtils.getPreferredCharset()); } - URI uri = null; + URI uri; // Check if the zone exists in the system final DataCenterVO zone = _dcDao.findById(dcId); @@ -520,7 +508,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, discoverer.putParam(allParams); } - final List result = new ArrayList(); + final List result = new ArrayList<>(); ClusterVO cluster = new ClusterVO(dcId, podId, clusterName); cluster.setHypervisorType(hypervisorType.toString()); @@ -540,7 +528,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, result.add(cluster); if (clusterType == Cluster.ClusterType.CloudManaged) { - final Map details = new HashMap(); + final Map details = new HashMap<>(); // should do this nicer perhaps ? if (hypervisorType == HypervisorType.Ovm3) { final Map allParams = cmd.getFullUrlParams(); @@ -578,8 +566,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new InvalidParameterValueException(url + " is not a valid uri"); } - final List hosts = new ArrayList(); - Map> resources = null; + final List hosts = new ArrayList<>(); + Map> resources; resources = discoverer.find(dcId, podId, cluster.getId(), uri, username, password, null); if (resources != null) { @@ -670,7 +658,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, private List discoverHostsFull(final Long dcId, final Long podId, Long clusterId, final String clusterName, String url, String username, String password, final String hypervisorType, final List hostTags, final Map params, final boolean deferAgentCreation) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException { - URI uri = null; + URI uri; // Check if the zone exists in the system final DataCenterVO zone = _dcDao.findById(dcId); @@ -808,7 +796,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new InvalidParameterValueException(url + " is not a valid uri"); } - final List hosts = new ArrayList(); + final List hosts = new ArrayList<>(); s_logger.info("Trying to add a new host at " + url + " in data center " + dcId); boolean isHypervisorTypeSupported = false; for (final Discoverer discoverer : _discoverers) { @@ -870,7 +858,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, return null; } - HostVO host = null; + HostVO host; if (deferAgentCreation) { host = (HostVO)createHostAndAgentDeferred(resource, entry.getValue(), true, hostTags, false); } else { @@ -964,7 +952,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, _hostDao.remove(hostId); if (clusterId != null) { final List hosts = listAllHostsInCluster(clusterId); - if (hosts.size() == 0) { + if (hosts.isEmpty()) { final ClusterVO cluster = _clusterDao.findById(clusterId); cluster.setGuid(null); _clusterDao.update(clusterId, cluster); @@ -1089,7 +1077,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, final Hypervisor.HypervisorType hypervisorType = cluster.getHypervisorType(); final List hosts = listAllHostsInCluster(cmd.getId()); - if (hosts.size() > 0) { + if (!hosts.isEmpty()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Cluster: " + cmd.getId() + " still has hosts, can't remove"); } @@ -1099,7 +1087,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, // don't allow to remove the cluster if it has non-removed storage // pools final List storagePools = _storagePoolDao.listPoolsByCluster(cmd.getId()); - if (storagePools.size() > 0) { + if (!storagePools.isEmpty()) { if (s_logger.isDebugEnabled()) { s_logger.debug("Cluster: " + cmd.getId() + " still has storage pools, can't remove"); } @@ -1167,36 +1155,26 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } - Cluster.ClusterType newClusterType = null; + Cluster.ClusterType newClusterType; if (clusterType != null && !clusterType.isEmpty()) { try { newClusterType = Cluster.ClusterType.valueOf(clusterType); } catch (final IllegalArgumentException ex) { throw new InvalidParameterValueException("Unable to resolve " + clusterType + " to a supported type"); } - if (newClusterType == null) { - s_logger.error("Unable to resolve " + clusterType + " to a valid supported cluster type"); - throw new InvalidParameterValueException("Unable to resolve " + clusterType + " to a supported type"); - } else { - cluster.setClusterType(newClusterType); - doUpdate = true; - } + cluster.setClusterType(newClusterType); + doUpdate = true; } - Grouping.AllocationState newAllocationState = null; + Grouping.AllocationState newAllocationState; if (allocationState != null && !allocationState.isEmpty()) { try { newAllocationState = Grouping.AllocationState.valueOf(allocationState); } catch (final IllegalArgumentException ex) { throw new InvalidParameterValueException("Unable to resolve Allocation State '" + allocationState + "' to a supported state"); } - if (newAllocationState == null) { - s_logger.error("Unable to resolve " + allocationState + " to a valid supported allocation State"); - throw new InvalidParameterValueException("Unable to resolve " + allocationState + " to a supported state"); - } else { - cluster.setAllocationState(newAllocationState); - doUpdate = true; - } + cluster.setAllocationState(newAllocationState); + doUpdate = true; } Managed.ManagedState newManagedState = null; @@ -1207,12 +1185,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } catch (final IllegalArgumentException ex) { throw new InvalidParameterValueException("Unable to resolve Managed State '" + managedstate + "' to a supported state"); } - if (newManagedState == null) { - s_logger.error("Unable to resolve Managed State '" + managedstate + "' to a supported state"); - throw new InvalidParameterValueException("Unable to resolve Managed State '" + managedstate + "' to a supported state"); - } else { - doUpdate = true; - } + doUpdate = true; } if (doUpdate) { @@ -1240,12 +1213,13 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } final int retry = 40; - boolean lsuccess = true; + boolean lsuccess; for (int i = 0; i < retry; i++) { lsuccess = true; try { - Thread.sleep(5 * 1000); - } catch (final Exception e) { + Thread.currentThread().wait(5 * 1000); + } catch (final InterruptedException e) { + s_logger.debug("thread unexpectedly interupted during wait, while updating cluster"); } hosts = listAllUpAndEnabledHosts(Host.Type.Routing, cluster.getId(), cluster.getPodId(), cluster.getDataCenterId()); for (final HostVO host : hosts) { @@ -1254,12 +1228,12 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, break; } } - if (lsuccess == true) { + if (lsuccess) { success = true; break; } } - if (success == false) { + if (!success) { throw new CloudRuntimeException("PrepareUnmanaged Failed due to some hosts are still in UP status after 5 Minutes, please try later "); } } finally { @@ -1377,7 +1351,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (host.getType() == Host.Type.Routing) { final List vms = _vmDao.listByHostId(hostId); - if (vms.size() == 0) { + if (vms.isEmpty()) { return true; } @@ -1405,7 +1379,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, String logMessage = String.format( "Unsupported host.maintenance.local.storage.strategy: %s. Please set a strategy according to the global settings description: " + "'Error', 'Migration', or 'ForceStop'.", - HOST_MAINTENANCE_LOCAL_STRATEGY.value().toString()); + HOST_MAINTENANCE_LOCAL_STRATEGY.value()); s_logger.error(logMessage); throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } @@ -1461,13 +1435,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getServiceOfferingId()); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm, null, offeringVO, null, null); plan.setMigrationPlan(true); - DeployDestination dest = null; - try { - dest = deploymentManager.planDeployment(profile, plan, new DeploymentPlanner.ExcludeList(), null); - } catch (InsufficientServerCapacityException e) { - throw new CloudRuntimeException(String.format("Maintenance failed, could not find deployment destination for VM [id=%s, name=%s].", vm.getId(), vm.getInstanceName()), - e); - } + DeployDestination dest = getDeployDestination(vm, profile, plan); Host destHost = dest.getHost(); try { @@ -1479,6 +1447,20 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } + private DeployDestination getDeployDestination(VMInstanceVO vm, VirtualMachineProfile profile, DataCenterDeployment plan) { + DeployDestination dest; + try { + dest = deploymentManager.planDeployment(profile, plan, new DeploymentPlanner.ExcludeList(), null); + } catch (InsufficientServerCapacityException e) { + throw new CloudRuntimeException(String.format("Maintenance failed, could not find deployment destination for VM [id=%s, name=%s].", vm.getId(), vm.getInstanceName()), + e); + } + if (dest == null) { + throw new CloudRuntimeException(String.format("Maintenance failed, could not find deployment destination for VM [id=%s, name=%s], using plan: %s.", vm.getId(), vm.getInstanceName(), plan)); + } + return dest; + } + @Override public boolean maintain(final long hostId) throws AgentUnavailableException { final Boolean result = propagateResourceEvent(hostId, ResourceState.Event.AdminAskMaintenance); @@ -1526,15 +1508,15 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, List migratingInVMs = _vmDao.findByHostInStates(hostId, State.Migrating); - if (migratingInVMs.size() > 0) { + if (!migratingInVMs.isEmpty()) { throw new CloudRuntimeException("Host contains incoming VMs migrating. Please wait for them to complete before putting to maintenance."); } - if (_vmDao.findByHostInStates(hostId, State.Starting, State.Stopping).size() > 0) { + if (!_vmDao.findByHostInStates(hostId, State.Starting, State.Stopping).isEmpty()) { throw new CloudRuntimeException("Host contains VMs in starting/stopping state. Please wait for them to complete before putting to maintenance."); } - if (_vmDao.findByHostInStates(hostId, State.Error, State.Unknown).size() > 0) { + if (!_vmDao.findByHostInStates(hostId, State.Error, State.Unknown).isEmpty()) { throw new CloudRuntimeException("Host contains VMs in error/unknown/shutdown state. Please fix errors to proceed."); } @@ -1555,25 +1537,22 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if(StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { return false; } - return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.Migration.toString().toLowerCase()); + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().equalsIgnoreCase(WorkType.Migration.toString()); } protected boolean isMaintenanceLocalStrategyForceStop() { if(StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value())) { return false; } - return HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(WorkType.ForceStop.toString().toLowerCase()); + return HOST_MAINTENANCE_LOCAL_STRATEGY.value().equalsIgnoreCase(WorkType.ForceStop.toString()); } /** * Returns true if the host.maintenance.local.storage.strategy is the Default: "Error", blank, empty, or null. */ protected boolean isMaintenanceLocalStrategyDefault() { - if (StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value().toString()) - || HOST_MAINTENANCE_LOCAL_STRATEGY.value().toLowerCase().equals(State.Error.toString().toLowerCase())) { - return true; - } - return false; + return StringUtils.isBlank(HOST_MAINTENANCE_LOCAL_STRATEGY.value()) + || HOST_MAINTENANCE_LOCAL_STRATEGY.value().equalsIgnoreCase(State.Error.toString()); } /** @@ -1640,7 +1619,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, HostVO host = _hostDao.findById(hostId); if (host == null || host.getRemoved() != null) { - throw new InvalidParameterValueException(String.format("Host [id=%s] does not exist", host.getId())); + throw new InvalidParameterValueException(String.format("Host [id=%s] does not exist", host == null ? "" : host.getId())); } if (host.getResourceState() != ResourceState.Degraded) { @@ -1725,7 +1704,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, * Return true if host goes into Maintenance mode. There are various possibilities for VMs' states * on a host. We need to track the various VM states on each run and accordingly transit to the * appropriate state. - * * We change states as follows - * 1. If there are no VMs in running, migrating, starting, stopping, error, unknown states we can move * to maintenance state. Note that there cannot be incoming migrations as the API Call prepare for @@ -1899,7 +1877,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, guestOSDetail.setValue(String.valueOf(guestOSCategory.getId())); _hostDetailsDao.update(guestOSDetail.getId(), guestOSDetail); } else { - final Map detail = new HashMap(); + final Map detail = new HashMap<>(); detail.put("guest.os.category.id", String.valueOf(guestOSCategory.getId())); _hostDetailsDao.persist(hostId, detail); } @@ -2049,9 +2027,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public List getSupportedHypervisorTypes(final long zoneId, final boolean forVirtualRouter, final Long podId) { - final List hypervisorTypes = new ArrayList(); + final List hypervisorTypes = new ArrayList<>(); - List clustersForZone = new ArrayList(); + List clustersForZone; if (podId != null) { clustersForZone = _clusterDao.listByPodId(podId); } else { @@ -2060,7 +2038,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, for (final ClusterVO cluster : clustersForZone) { final HypervisorType hType = cluster.getHypervisorType(); - if (!forVirtualRouter || forVirtualRouter && hType != HypervisorType.BareMetal && hType != HypervisorType.Ovm) { + if (!forVirtualRouter || (hType != HypervisorType.BareMetal && hType != HypervisorType.Ovm)) { hypervisorTypes.add(hType); } } @@ -2096,7 +2074,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (isValid) { final List clusters = _clusterDao.listByDcHyType(zoneId, defaultHyper.toString()); - if (clusters.size() <= 0) { + if (clusters.isEmpty()) { isValid = false; } } @@ -2113,7 +2091,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, HypervisorType defaultHype = getDefaultHypervisor(zoneId); if (defaultHype == HypervisorType.None) { final List supportedHypes = getSupportedHypervisorTypes(zoneId, false, null); - if (supportedHypes.size() > 0) { + if (!supportedHypes.isEmpty()) { Collections.shuffle(supportedHypes); defaultHype = supportedHypes.get(0); } @@ -2237,10 +2215,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, final String cidrNetmask = NetUtils.getCidrSubNet("255.255.255.255", cidrSize); final long cidrNetmaskNumeric = NetUtils.ip2Long(cidrNetmask); final long serverNetmaskNumeric = NetUtils.ip2Long(serverPrivateNetmask); - if (serverNetmaskNumeric > cidrNetmaskNumeric) { - return false; - } - return true; + return serverNetmaskNumeric <= cidrNetmaskNumeric; } private HostVO getNewHost(StartupCommand[] startupCommands) { @@ -2254,11 +2229,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, host = findHostByGuid(startupCommand.getGuidWithoutResource()); - if (host != null) { - return host; - } - - return null; + return host; // even when host == null! } protected HostVO createHostVO(final StartupCommand[] cmds, final ServerResource resource, final Map details, List hostTags, @@ -2289,11 +2260,11 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } - long dcId = -1; + long dcId; DataCenterVO dc = _dcDao.findByName(dataCenter); if (dc == null) { try { - dcId = Long.parseLong(dataCenter); + dcId = Long.parseLong(dataCenter != null ? dataCenter : "-1"); dc = _dcDao.findById(dcId); } catch (final NumberFormatException e) { s_logger.debug("Cannot parse " + dataCenter + " into Long."); @@ -2307,7 +2278,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, HostPodVO p = _podDao.findByName(pod, dcId); if (p == null) { try { - final long podId = Long.parseLong(pod); + final long podId = Long.parseLong(pod != null ? pod : "-1"); p = _podDao.findById(podId); } catch (final NumberFormatException e) { s_logger.debug("Cannot parse " + pod + " into Long."); @@ -2326,9 +2297,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, clusterId = Long.valueOf(cluster); } catch (final NumberFormatException e) { if (podId != null) { - ClusterVO c = _clusterDao.findBy(cluster, podId.longValue()); + ClusterVO c = _clusterDao.findBy(cluster, podId); if (c == null) { - c = new ClusterVO(dcId, podId.longValue(), cluster); + c = new ClusterVO(dcId, podId, cluster); c = _clusterDao.persist(c); } clusterId = c.getId(); @@ -2445,7 +2416,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, for (HostVO hostVO : hostVOs) { DetailVO hostDetailVO = _hostDetailsDao.findDetail(hostVO.getId(), name); - if (hostDetailVO == null || Boolean.parseBoolean(hostDetailVO.getValue()) == false) { + if (hostDetailVO == null || !Boolean.parseBoolean(hostDetailVO.getValue())) { clusterSupportsResigning = false; break; @@ -2533,7 +2504,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } if (s_logger.isDebugEnabled()) { - new Request(-1l, -1l, cmds, true, false).logD("Startup request from directly connected host: ", true); + new Request(-1L, -1L, cmds, true, false).logD("Startup request from directly connected host: ", true); } if (old) { @@ -2603,7 +2574,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } if (s_logger.isDebugEnabled()) { - new Request(-1l, -1l, cmds, true, false).logD("Startup request from directly connected host: ", true); + new Request(-1L, -1L, cmds, true, false).logD("Startup request from directly connected host: ", true); } if (old) { @@ -2688,8 +2659,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public Host createHostAndAgent(final Long hostId, final ServerResource resource, final Map details, final boolean old, final List hostTags, final boolean forRebalance) { - final Host host = createHostAndAgent(resource, details, old, hostTags, forRebalance); - return host; + return createHostAndAgent(resource, details, old, hostTags, forRebalance); } @Override @@ -2699,8 +2669,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new InvalidParameterValueException("Can't find zone with id " + zoneId); } - final Map details = hostDetails; - final String guid = details.get("guid"); + final String guid = hostDetails.get("guid"); final List currentHosts = listAllUpAndEnabledHostsInOneZoneByType(hostType, zoneId); for (final HostVO currentHost : currentHosts) { if (currentHost.getGuid().equals(guid)) { @@ -2716,7 +2685,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, return createHostVO(cmds, null, null, null, ResourceStateAdapter.Event.CREATE_HOST_VO_FOR_CONNECTED); } - private void checkIPConflicts(final HostPodVO pod, final DataCenterVO dc, final String serverPrivateIP, final String serverPrivateNetmask, final String serverPublicIP, final String serverPublicNetmask) { + private void checkIPConflicts(final HostPodVO pod, final DataCenterVO dc, final String serverPrivateIP, final String serverPublicIP) { // If the server's private IP is the same as is public IP, this host has // a host-only private network. Don't check for conflicts with the // private IP address table. @@ -2745,7 +2714,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, // If the server's public IP address is already in the database, // return false final List existingPublicIPs = _publicIPAddressDao.listByDcIdIpAddress(dc.getId(), serverPublicIP); - if (existingPublicIPs.size() > 0) { + if (!existingPublicIPs.isEmpty()) { throw new IllegalArgumentException("The public ip address of the server (" + serverPublicIP + ") is already in use in zone: " + dc.getName()); } } @@ -2775,7 +2744,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, final HostPodVO pod = _podDao.findById(host.getPodId()); final DataCenterVO dc = _dcDao.findById(host.getDataCenterId()); - checkIPConflicts(pod, dc, ssCmd.getPrivateIpAddress(), ssCmd.getPublicIpAddress(), ssCmd.getPublicIpAddress(), ssCmd.getPublicNetmask()); + checkIPConflicts(pod, dc, ssCmd.getPrivateIpAddress(), ssCmd.getPublicIpAddress()); host.setType(com.cloud.host.Host.Type.Routing); host.setDetails(details); host.setCaps(ssCmd.getCapabilities()); @@ -2812,8 +2781,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new UnableDeleteHostException("Failed to set primary storage into maintenance mode"); } } catch (final Exception e) { - s_logger.debug("Failed to set primary storage into maintenance mode, due to: " + e.toString()); - throw new UnableDeleteHostException("Failed to set primary storage into maintenance mode, due to: " + e.toString()); + s_logger.debug("Failed to set primary storage into maintenance mode", e); + throw new UnableDeleteHostException("Failed to set primary storage into maintenance mode, due to: " + e); } } @@ -2957,7 +2926,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (result.getReturnCode() != 0) { throw new CloudRuntimeException(String.format("Could not restart agent on %s due to: %s", host, result.getStdErr())); } - s_logger.debug("cloudstack-agent restart result: " + result.toString()); + s_logger.debug("cloudstack-agent restart result: " + result); } catch (final SshException e) { throw new CloudRuntimeException("SSH to agent is enabled, but agent restart failed", e); } @@ -2978,7 +2947,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } @Override - public boolean executeUserRequest(final long hostId, final ResourceState.Event event) throws AgentUnavailableException { + public boolean executeUserRequest(final long hostId, final ResourceState.Event event) { if (event == ResourceState.Event.AdminAskMaintenance) { return doMaintain(hostId); } else if (event == ResourceState.Event.AdminCancelMaintenance) { @@ -3292,7 +3261,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, // The search is not able to return list of enums, so getting // list of hypervisors as strings and then converting them to enum final List hvs = _hostDao.customSearch(sc, null); - final List hypervisors = new ArrayList(); + final List hypervisors = new ArrayList<>(); for (final String hv : hvs) { hypervisors.add(HypervisorType.getType(hv)); } @@ -3317,7 +3286,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, public HostStats getHostStatistics(final long hostId) { final Answer answer = _agentMgr.easySend(hostId, new GetHostStatsCommand(_hostDao.findById(hostId).getGuid(), _hostDao.findById(hostId).getName(), hostId)); - if (answer != null && answer instanceof UnsupportedAnswer) { + if (answer instanceof UnsupportedAnswer) { return null; } @@ -3354,20 +3323,16 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public String getHostTags(final long hostId) { final List hostTags = _hostTagsDao.getHostTags(hostId).parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); - if (hostTags == null) { - return null; - } else { - return com.cloud.utils.StringUtils.listToCsvTags(hostTags); - } + return StringUtils.listToCsvTags(hostTags); } @Override public List listByDataCenter(final long dcId) { final List pods = _podDao.listByDataCenterId(dcId); - final ArrayList pcs = new ArrayList(); + final ArrayList pcs = new ArrayList<>(); for (final HostPodVO pod : pods) { final List clusters = _clusterDao.listByPodId(pod.getId()); - if (clusters.size() == 0) { + if (clusters.isEmpty()) { pcs.add(new PodCluster(pod, null)); } else { for (final ClusterVO cluster : clusters) { @@ -3412,7 +3377,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, public boolean isHostGpuEnabled(final long hostId) { final SearchCriteria sc = _gpuAvailability.create(); sc.setParameters("hostId", hostId); - return _hostGpuGroupsDao.customSearch(sc, null).size() > 0 ? true : false; + return !_hostGpuGroupsDao.customSearch(sc, null).isEmpty(); } @Override @@ -3477,7 +3442,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, // Update GPU group capacity final TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); - _hostGpuGroupsDao.persist(hostId, new ArrayList(groupDetails.keySet())); + _hostGpuGroupsDao.persist(hostId, new ArrayList<>(groupDetails.keySet())); _vgpuTypesDao.persist(hostId, groupDetails); txn.commit(); } @@ -3485,7 +3450,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public HashMap> getGPUStatistics(final HostVO host) { final Answer answer = _agentMgr.easySend(host.getId(), new GetGPUStatsCommand(host.getGuid(), host.getName())); - if (answer != null && answer instanceof UnsupportedAnswer) { + if (answer instanceof UnsupportedAnswer) { return null; } if (answer == null || !answer.getResult()) { @@ -3526,7 +3491,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @ActionEvent(eventType = EventTypes.EVENT_HOST_RESERVATION_RELEASE, eventDescription = "releasing host reservation", async = true) public boolean releaseHostReservation(final Long hostId) { try { - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public Boolean doInTransaction(final TransactionStatus status) { final PlannerHostReservationVO reservationEntry = _plannerHostReserveDao.findByHostId(hostId); From 33cdddfcd1bc4b630cfde041a36ab5baa20c812d Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 19 Mar 2025 00:28:04 +0530 Subject: [PATCH 07/42] Fix to propagate updated management servers list and lb algorithm in host and indirect.agent.lb.algorithm settings resp, to systemvm agents (#10524) * propagate updated management servers list and lb algorithm in host and indirect.agent.lb.algorithm settings, to systemvm agents * addressed comments * addressed comments --- .../agent/lb/IndirectAgentLBServiceImpl.java | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java index 84c3081bfc1..d51b42ab17f 100644 --- a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java @@ -74,6 +74,8 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement ResourceState.ErrorInMaintenance, ResourceState.PrepareForMaintenance); private static final List agentValidHostTypes = List.of(Host.Type.Routing, Host.Type.ConsoleProxy, Host.Type.SecondaryStorage, Host.Type.SecondaryStorageVM); + private static final List agentNonRoutingHostTypes = List.of(Host.Type.ConsoleProxy, + Host.Type.SecondaryStorage, Host.Type.SecondaryStorageVM); private static final List agentValidHypervisorTypes = List.of( Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.LXC); @@ -136,6 +138,16 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement return hostIdList; } + private List getAllAgentBasedNonRoutingHostsFromDB(final Long zoneId) { + return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, null, + agentValidResourceStates, agentNonRoutingHostTypes, agentValidHypervisorTypes); + } + + private List getAllAgentBasedRoutingHostsFromDB(final Long zoneId, final Long clusterId) { + return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, clusterId, + agentValidResourceStates, List.of(Host.Type.Routing), agentValidHypervisorTypes); + } + private List getAllAgentBasedHostsFromDB(final Long zoneId, final Long clusterId) { return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, clusterId, agentValidResourceStates, agentValidHostTypes, agentValidHypervisorTypes); @@ -158,32 +170,42 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement public void propagateMSListToAgents() { logger.debug("Propagating management server list update to agents"); final String lbAlgorithm = getLBAlgorithmName(); + final Long globalLbCheckInterval = getLBPreferredHostCheckInterval(null); List zones = dataCenterDao.listAll(); for (DataCenterVO zone : zones) { List zoneHostIds = new ArrayList<>(); + List nonRoutingHostIds = getAllAgentBasedNonRoutingHostsFromDB(zone.getId()); + zoneHostIds.addAll(nonRoutingHostIds); Map> clusterHostIdsMap = new HashMap<>(); List clusterIds = clusterDao.listAllClusterIds(zone.getId()); for (Long clusterId : clusterIds) { - List hostIds = getAllAgentBasedHostsFromDB(zone.getId(), clusterId); + List hostIds = getAllAgentBasedRoutingHostsFromDB(zone.getId(), clusterId); clusterHostIdsMap.put(clusterId, hostIds); zoneHostIds.addAll(hostIds); } zoneHostIds.sort(Comparator.comparingLong(x -> x)); + for (Long nonRoutingHostId : nonRoutingHostIds) { + setupMSList(nonRoutingHostId, zone.getId(), zoneHostIds, lbAlgorithm, globalLbCheckInterval); + } for (Long clusterId : clusterIds) { - final Long lbCheckInterval = getLBPreferredHostCheckInterval(clusterId); + final Long clusterLbCheckInterval = getLBPreferredHostCheckInterval(clusterId); List hostIds = clusterHostIdsMap.get(clusterId); for (Long hostId : hostIds) { - final List msList = getManagementServerList(hostId, zone.getId(), zoneHostIds); - final SetupMSListCommand cmd = new SetupMSListCommand(msList, lbAlgorithm, lbCheckInterval); - final Answer answer = agentManager.easySend(hostId, cmd); - if (answer == null || !answer.getResult()) { - logger.warn("Failed to setup management servers list to the agent of ID: {}", hostId); - } + setupMSList(hostId, zone.getId(), zoneHostIds, lbAlgorithm, clusterLbCheckInterval); } } } } + private void setupMSList(final Long hostId, final Long dcId, final List orderedHostIdList, final String lbAlgorithm, final Long lbCheckInterval) { + final List msList = getManagementServerList(hostId, dcId, orderedHostIdList); + final SetupMSListCommand cmd = new SetupMSListCommand(msList, lbAlgorithm, lbCheckInterval); + final Answer answer = agentManager.easySend(hostId, cmd); + if (answer == null || !answer.getResult()) { + logger.warn(String.format("Failed to setup management servers list to the agent of ID: %d", hostId)); + } + } + private void configureAlgorithmMap() { final List algorithms = new ArrayList<>(); algorithms.add(new IndirectAgentLBStaticAlgorithm()); From 9dceae46148f916f4094f90797567652667ccadb Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 19 Mar 2025 14:18:05 +0530 Subject: [PATCH 08/42] MS maintenance improvements (#10417) * Update last agents during ms maintenance, and some code improvements * Send 503 (Service Unavailable) response status when maintenance or shutdown is initiated [Any load balancer in the clustered environment can avoid routing requests to this MS node] * Migrate systemvm agents before routing host agents, and some code improvements * Added events for ms maintenance and shutdown operations * Added the following ms maintenance and shutdown improvements - block new agent connections during prepare for maintenance of ms - maintain avoids ms list - propagate updated management servers list and lb algorithm in host and indirect.agent.lb.algorithm settings respectively, to systemvm (non-routing) agents - updated setup ms list and migrate agent connections to executor service - migrate agent connection through executor, and send the answer to the ms host that initiated the migration - re-initialize ssl handshake executor if it is shutdown - don't allow prepare for maintenance or shutdown when other management server nodes are in preparing states - don't allow trigger shutdown when management server is up and other management server nodes are in preparing states - stop agent connections monitor on ms maintenance - update avoid ms list in ready command - updated connected host from the client connection - update last agents in ms metrics from the database - updated some agent config descriptions - update last management server in the hosts during shutdown - added agents and lastagents in management server response - updated management server maintenance & shutdown unit tests - some code improvements * refactored code / addressed comments * removed shutdown testcase (maybe, calling System.exit) * Revert "removed shutdown testcase (maybe, calling System.exit)" This reverts commit e14b0717152ef6c8be102d61c80f42803a53172e. * avoid system.exit during shutdown test * code improvements * testcase fix * Fix cutoff time in agent connections monitor thread --- .../src/main/java/com/cloud/agent/Agent.java | 86 +-- .../main/java/com/cloud/agent/AgentShell.java | 19 +- .../java/com/cloud/agent/IAgentShell.java | 7 +- .../agent/properties/AgentProperties.java | 2 +- .../java/com/cloud/agent/AgentShellTest.java | 2 +- .../main/java/com/cloud/event/EventTypes.java | 13 + .../apache/cloudstack/api/ApiConstants.java | 1 + .../apache/cloudstack/api/ApiErrorCode.java | 1 + .../response/ManagementServerResponse.java | 24 + .../java/com/cloud/agent/api/PingAnswer.java | 10 +- .../com/cloud/agent/api/ReadyCommand.java | 9 + .../agent/lb/SetupMSListCommand.java | 8 +- .../java/com/cloud/agent/AgentManager.java | 5 - .../cloud/agent/manager/AgentManagerImpl.java | 114 ++-- .../manager/ClusteredAgentManagerImpl.java | 24 +- .../ManagementServiceConfiguration.java | 2 +- .../main/java/com/cloud/host/dao/HostDao.java | 9 +- .../java/com/cloud/host/dao/HostDaoImpl.java | 20 +- .../com/cloud/host/dao/HostDaoImplTest.java | 4 +- .../jobs/impl/AsyncJobManagerImpl.java | 2 +- .../ManagementServerMaintenanceListener.java | 4 + .../ManagementServerMaintenanceManager.java | 4 + ...anagementServerMaintenanceManagerImpl.java | 132 +++-- ...ementServerMaintenanceManagerImplTest.java | 502 +++++++++++++++++- .../ManagementServerMetricsResponse.java | 4 +- .../java/com/cloud/api/ApiDispatcher.java | 2 +- .../main/java/com/cloud/api/ApiServer.java | 5 + .../com/cloud/api/query/QueryManagerImpl.java | 6 +- .../java/com/cloud/server/StatsCollector.java | 16 +- .../agent/lb/IndirectAgentLBServiceImpl.java | 264 +++++++-- .../lb/IndirectAgentLBServiceImplTest.java | 6 +- .../com/cloud/utils/nio/HandlerFactory.java | 2 +- .../main/java/com/cloud/utils/nio/Link.java | 4 +- .../java/com/cloud/utils/nio/NioClient.java | 4 + .../com/cloud/utils/nio/NioConnection.java | 59 +- 35 files changed, 1146 insertions(+), 230 deletions(-) diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index 0a76bfbb4f8..ad480fef4e5 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -342,7 +342,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater logger.info("Attempted to connect to the server, but received an unexpected exception, trying again...", e); } } - shell.updateConnectedHost(); + shell.updateConnectedHost(((NioClient)connection).getHost()); scavengeOldAgentObjects(); } @@ -617,15 +617,11 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } protected void reconnect(final Link link) { - reconnect(link, null, null, false); + reconnect(link, null, false); } - protected void reconnect(final Link link, String preferredHost, List avoidHostList, boolean forTransfer) { + protected void reconnect(final Link link, String preferredMSHost, boolean forTransfer) { if (!(forTransfer || reconnectAllowed)) { - return; - } - - if (!reconnectAllowed) { logger.debug("Reconnect requested but it is not allowed {}", () -> getLinkLog(link)); return; } @@ -637,19 +633,26 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater serverResource.disconnected(); logger.info("Lost connection to host: {}. Attempting reconnection while we still have {} commands in progress.", shell.getConnectedHost(), commandsInProgress.get()); stopAndCleanupConnection(true); + String host = preferredMSHost; + if (org.apache.commons.lang3.StringUtils.isBlank(host)) { + host = shell.getNextHost(); + } + List avoidMSHostList = shell.getAvoidHosts(); do { - final String host = shell.getNextHost(); - connection = new NioClient(getAgentName(), host, shell.getPort(), shell.getWorkers(), shell.getSslHandshakeTimeout(), this); - logger.info("Reconnecting to host: {}", host); - try { - connection.start(); - } catch (final NioConnectionException e) { - logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e); - stopAndCleanupConnection(false); + if (CollectionUtils.isEmpty(avoidMSHostList) || !avoidMSHostList.contains(host)) { + connection = new NioClient(getAgentName(), host, shell.getPort(), shell.getWorkers(), shell.getSslHandshakeTimeout(), this); + logger.info("Reconnecting to host: {}", host); + try { + connection.start(); + } catch (final NioConnectionException e) { + logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e); + stopAndCleanupConnection(false); + } } shell.getBackoffAlgorithm().waitBeforeRetry(); + host = shell.getNextHost(); } while (!connection.isStartup()); - shell.updateConnectedHost(); + shell.updateConnectedHost(((NioClient)connection).getHost()); logger.info("Connected to the host: {}", shell.getConnectedHost()); } @@ -922,7 +925,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater return new SetupCertificateAnswer(true); } - private void processManagementServerList(final List msList, final String lbAlgorithm, final Long lbCheckInterval) { + private void processManagementServerList(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) { if (CollectionUtils.isNotEmpty(msList) && StringUtils.isNotEmpty(lbAlgorithm)) { try { final String newMSHosts = String.format("%s%s%s", com.cloud.utils.StringUtils.toCSVList(msList), IAgentShell.hostLbAlgorithmSeparator, lbAlgorithm); @@ -934,6 +937,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater throw new CloudRuntimeException("Could not persist received management servers list", e); } } + shell.setAvoidHosts(avoidMsList); if ("shuffle".equals(lbAlgorithm)) { scheduleHostLBCheckerTask(0); } else { @@ -942,16 +946,18 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } private Answer setupManagementServerList(final SetupMSListCommand cmd) { - processManagementServerList(cmd.getMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); + processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); return new SetupMSListAnswer(true); } private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) { try { if (CollectionUtils.isNotEmpty(cmd.getMsList())) { - processManagementServerList(cmd.getMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); + processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); } - migrateAgentConnection(cmd.getAvoidMsList()); + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")).schedule(() -> { + migrateAgentConnection(cmd.getAvoidMsList()); + }, 3, TimeUnit.SECONDS); } catch (Exception e) { String errMsg = "Migrate agent connection failed, due to " + e.getMessage(); logger.debug(errMsg, e); @@ -972,25 +978,26 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater throw new CloudRuntimeException("No other Management Server hosts to migrate"); } - String preferredHost = null; + String preferredMSHost = null; for (String msHost : msHostsList) { try (final Socket socket = new Socket()) { socket.connect(new InetSocketAddress(msHost, shell.getPort()), 5000); - preferredHost = msHost; + preferredMSHost = msHost; break; } catch (final IOException e) { throw new CloudRuntimeException("Management server host: " + msHost + " is not reachable, to migrate connection"); } } - if (preferredHost == null) { + if (preferredMSHost == null) { throw new CloudRuntimeException("Management server host(s) are not reachable, to migrate connection"); } - logger.debug("Management server host " + preferredHost + " is found to be reachable, trying to reconnect"); + logger.debug("Management server host " + preferredMSHost + " is found to be reachable, trying to reconnect"); shell.resetHostCounter(); + shell.setAvoidHosts(avoidMsList); shell.setConnectionTransfer(true); - reconnect(link, preferredHost, avoidMsList, true); + reconnect(link, preferredMSHost, true); } public void processResponse(final Response response, final Link link) { @@ -1003,14 +1010,21 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater for (final IAgentControlListener listener : controlListeners) { listener.processControlResponse(response, (AgentControlAnswer)answer); } - } else if (answer instanceof PingAnswer && (((PingAnswer) answer).isSendStartup()) && reconnectAllowed) { - logger.info("Management server requested startup command to reinitialize the agent"); - sendStartup(link); + } else if (answer instanceof PingAnswer) { + processPingAnswer((PingAnswer) answer); } else { updateLastPingResponseTime(); } } + private void processPingAnswer(final PingAnswer answer) { + if ((answer.isSendStartup()) && reconnectAllowed) { + logger.info("Management server requested startup command to reinitialize the agent"); + sendStartup(link); + } + shell.setAvoidHosts(answer.getAvoidMsList()); + } + public void processReadyCommand(final Command cmd) { final ReadyCommand ready = (ReadyCommand)cmd; // Set human readable sizes; @@ -1027,7 +1041,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } verifyAgentArch(ready.getArch()); - processManagementServerList(ready.getMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval()); + processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval()); logger.info("Ready command is processed for agent [id: {}, uuid: {}, name: {}]", getId(), getUuid(), getName()); } @@ -1374,26 +1388,26 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater if (msList == null || msList.length < 1) { return; } - final String preferredHost = msList[0]; + final String preferredMSHost = msList[0]; final String connectedHost = shell.getConnectedHost(); logger.debug("Running preferred host checker task, connected host={}, preferred host={}", - connectedHost, preferredHost); - if (preferredHost == null || preferredHost.equals(connectedHost) || link == null) { + connectedHost, preferredMSHost); + if (preferredMSHost == null || preferredMSHost.equals(connectedHost) || link == null) { return; } boolean isHostUp = false; try (final Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(preferredHost, shell.getPort()), 5000); + socket.connect(new InetSocketAddress(preferredMSHost, shell.getPort()), 5000); isHostUp = true; } catch (final IOException e) { - logger.debug("Host: {} is not reachable", preferredHost); + logger.debug("Host: {} is not reachable", preferredMSHost); } if (isHostUp && link != null && commandsInProgress.get() == 0) { if (logger.isDebugEnabled()) { - logger.debug("Preferred host {} is found to be reachable, trying to reconnect", preferredHost); + logger.debug("Preferred host {} is found to be reachable, trying to reconnect", preferredMSHost); } shell.resetHostCounter(); - reconnect(link); + reconnect(link, preferredMSHost, false); } } catch (Throwable t) { logger.error("Error caught while attempting to connect to preferred host", t); diff --git a/agent/src/main/java/com/cloud/agent/AgentShell.java b/agent/src/main/java/com/cloud/agent/AgentShell.java index aea7fd3a8de..4862e7e001e 100644 --- a/agent/src/main/java/com/cloud/agent/AgentShell.java +++ b/agent/src/main/java/com/cloud/agent/AgentShell.java @@ -66,6 +66,7 @@ public class AgentShell implements IAgentShell, Daemon { private String _zone; private String _pod; private String _host; + private List _avoidHosts; private String _privateIp; private int _port; private int _proxyPort; @@ -76,7 +77,6 @@ public class AgentShell implements IAgentShell, Daemon { private volatile boolean _exit = false; private int _pingRetries; private final List _agents = new ArrayList(); - private String hostToConnect; private String connectedHost; private Long preferredHostCheckInterval; private boolean connectionTransfer = false; @@ -121,7 +121,7 @@ public class AgentShell implements IAgentShell, Daemon { if (_hostCounter >= hosts.length) { _hostCounter = 0; } - hostToConnect = hosts[_hostCounter % hosts.length]; + String hostToConnect = hosts[_hostCounter % hosts.length]; _hostCounter++; return hostToConnect; } @@ -143,11 +143,10 @@ public class AgentShell implements IAgentShell, Daemon { } @Override - public void updateConnectedHost() { - connectedHost = hostToConnect; + public void updateConnectedHost(String connectedHost) { + this.connectedHost = connectedHost; } - @Override public void resetHostCounter() { _hostCounter = 0; @@ -166,6 +165,16 @@ public class AgentShell implements IAgentShell, Daemon { } } + @Override + public void setAvoidHosts(List avoidHosts) { + _avoidHosts = avoidHosts; + } + + @Override + public List getAvoidHosts() { + return _avoidHosts; + } + @Override public String getPrivateIp() { return _privateIp; diff --git a/agent/src/main/java/com/cloud/agent/IAgentShell.java b/agent/src/main/java/com/cloud/agent/IAgentShell.java index c0ecd90ae69..9eefa6d2eee 100644 --- a/agent/src/main/java/com/cloud/agent/IAgentShell.java +++ b/agent/src/main/java/com/cloud/agent/IAgentShell.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.agent; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -63,9 +64,13 @@ public interface IAgentShell { String[] getHosts(); + void setAvoidHosts(List hosts); + + List getAvoidHosts(); + long getLbCheckerInterval(Long receivedLbInterval); - void updateConnectedHost(); + void updateConnectedHost(String connectedHost); String getConnectedHost(); diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 61cd27fff77..feb1845d84b 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -816,7 +816,7 @@ public class AgentProperties{ * Data type: Integer.
* Default value: null */ - public static final Property SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", null, Integer.class); + public static final Property SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", 30, Integer.class); public static class Property { private String name; diff --git a/agent/src/test/java/com/cloud/agent/AgentShellTest.java b/agent/src/test/java/com/cloud/agent/AgentShellTest.java index 6d9758cc3dc..d8def24a603 100644 --- a/agent/src/test/java/com/cloud/agent/AgentShellTest.java +++ b/agent/src/test/java/com/cloud/agent/AgentShellTest.java @@ -358,7 +358,7 @@ public class AgentShellTest { AgentShell shell = new AgentShell(); shell.setHosts("test"); shell.getNextHost(); - shell.updateConnectedHost(); + shell.updateConnectedHost("test"); Assert.assertEquals(expected, shell.getConnectedHost()); } diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 862a6e21fa8..815bd2363d5 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -739,6 +739,13 @@ public class EventTypes { //Purge resources public static final String EVENT_PURGE_EXPUNGED_RESOURCES = "PURGE.EXPUNGED.RESOURCES"; + // Management Server + public static final String EVENT_MS_MAINTENANCE_PREPARE = "MS.MAINTENANCE.PREPARE"; + public static final String EVENT_MS_MAINTENANCE_CANCEL = "MS.MAINTENANCE.CANCEL"; + public static final String EVENT_MS_SHUTDOWN_PREPARE = "MS.SHUTDOWN.PREPARE"; + public static final String EVENT_MS_SHUTDOWN_CANCEL = "MS.SHUTDOWN.CANCEL"; + public static final String EVENT_MS_SHUTDOWN = "MS.SHUTDOWN"; + // OBJECT STORE public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE"; public static final String EVENT_OBJECT_STORE_DELETE = "OBJECT.STORE.DELETE"; @@ -1233,6 +1240,12 @@ public class EventTypes { entityEventDetails.put(EVENT_UPDATE_IMAGE_STORE_ACCESS_STATE, ImageStore.class); entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs"); + entityEventDetails.put(EVENT_MS_MAINTENANCE_PREPARE, "ManagementServer"); + entityEventDetails.put(EVENT_MS_MAINTENANCE_CANCEL, "ManagementServer"); + entityEventDetails.put(EVENT_MS_SHUTDOWN_PREPARE, "ManagementServer"); + entityEventDetails.put(EVENT_MS_SHUTDOWN_CANCEL, "ManagementServer"); + entityEventDetails.put(EVENT_MS_SHUTDOWN, "ManagementServer"); + //Object Store entityEventDetails.put(EVENT_OBJECT_STORE_CREATE, ObjectStore.class); entityEventDetails.put(EVENT_OBJECT_STORE_UPDATE, ObjectStore.class); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3e8b329cac7..627e7395e1e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1150,6 +1150,7 @@ public class ApiConstants { public static final String PENDING_JOBS_COUNT = "pendingjobscount"; public static final String AGENTS_COUNT = "agentscount"; public static final String AGENTS = "agents"; + public static final String LAST_AGENTS = "lastagents"; public static final String PUBLIC_MTU = "publicmtu"; public static final String PRIVATE_MTU = "privatemtu"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java b/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java index d4fdeddc9a9..03dc37325d4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java @@ -30,6 +30,7 @@ public enum ApiErrorCode { UNSUPPORTED_ACTION_ERROR(432), API_LIMIT_EXCEED(429), + SERVICE_UNAVAILABLE(503), INTERNAL_ERROR(530), ACCOUNT_ERROR(531), ACCOUNT_RESOURCE_LIMIT_ERROR(532), diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java index 729fb5ff3bc..e6cad482fe5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java @@ -82,6 +82,14 @@ public class ManagementServerResponse extends BaseResponse { @Param(description = "the Management Server Peers") private List peers; + @SerializedName(ApiConstants.LAST_AGENTS) + @Param(description = "the last agents this Management Server is responsible for, before shutdown or preparing for maintenance", since = "4.21.0.0") + private List lastAgents; + + @SerializedName(ApiConstants.AGENTS) + @Param(description = "the agents this Management Server is responsible for", since = "4.21.0.0") + private List agents; + @SerializedName(ApiConstants.AGENTS_COUNT) @Param(description = "the number of host agents this Management Server is responsible for", since = "4.21.0.0") private Long agentsCount; @@ -134,6 +142,14 @@ public class ManagementServerResponse extends BaseResponse { return ipAddress; } + public List getLastAgents() { + return lastAgents; + } + + public List getAgents() { + return agents; + } + public Long getAgentsCount() { return this.agentsCount; } @@ -190,6 +206,14 @@ public class ManagementServerResponse extends BaseResponse { this.ipAddress = ipAddress; } + public void setLastAgents(List lastAgents) { + this.lastAgents = lastAgents; + } + + public void setAgents(List agents) { + this.agents = agents; + } + public void setAgentsCount(Long agentsCount) { this.agentsCount = agentsCount; } diff --git a/core/src/main/java/com/cloud/agent/api/PingAnswer.java b/core/src/main/java/com/cloud/agent/api/PingAnswer.java index 6353b121583..3a40ad3925f 100644 --- a/core/src/main/java/com/cloud/agent/api/PingAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/PingAnswer.java @@ -19,18 +19,22 @@ package com.cloud.agent.api; +import java.util.List; + public class PingAnswer extends Answer { private PingCommand _command = null; private boolean sendStartup = false; + private List avoidMsList; protected PingAnswer() { } - public PingAnswer(PingCommand cmd, boolean sendStartup) { + public PingAnswer(PingCommand cmd, List avoidMsList, boolean sendStartup) { super(cmd); _command = cmd; this.sendStartup = sendStartup; + this.avoidMsList = avoidMsList; } public PingCommand getCommand() { @@ -44,4 +48,8 @@ public class PingAnswer extends Answer { public void setSendStartup(boolean sendStartup) { this.sendStartup = sendStartup; } + + public List getAvoidMsList() { + return avoidMsList; + } } diff --git a/core/src/main/java/com/cloud/agent/api/ReadyCommand.java b/core/src/main/java/com/cloud/agent/api/ReadyCommand.java index e2d974e3878..49768297ad5 100644 --- a/core/src/main/java/com/cloud/agent/api/ReadyCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ReadyCommand.java @@ -35,6 +35,7 @@ public class ReadyCommand extends Command { private String hostUuid; private String hostName; private List msHostList; + private List avoidMsHostList; private String lbAlgorithm; private Long lbCheckInterval; private Boolean enableHumanReadableSizes; @@ -90,6 +91,14 @@ public class ReadyCommand extends Command { this.msHostList = msHostList; } + public List getAvoidMsHostList() { + return avoidMsHostList; + } + + public void setAvoidMsHostList(List msHostList) { + this.avoidMsHostList = avoidMsHostList; + } + public String getLbAlgorithm() { return lbAlgorithm; } diff --git a/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java b/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java index 50cf956c9e7..32f436434c1 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java @@ -26,12 +26,14 @@ import com.cloud.agent.api.Command; public class SetupMSListCommand extends Command { private List msList; + private List avoidMsList; private String lbAlgorithm; private Long lbCheckInterval; - public SetupMSListCommand(final List msList, final String lbAlgorithm, final Long lbCheckInterval) { + public SetupMSListCommand(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) { super(); this.msList = msList; + this.avoidMsList = avoidMsList; this.lbAlgorithm = lbAlgorithm; this.lbCheckInterval = lbCheckInterval; } @@ -40,6 +42,10 @@ public class SetupMSListCommand extends Command { return msList; } + public List getAvoidMsList() { + return avoidMsList; + } + public String getLbAlgorithm() { return lbAlgorithm; } diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index 82e2d29f407..c01345ca21b 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -16,7 +16,6 @@ // under the License. package com.cloud.agent; -import java.util.List; import java.util.Map; import org.apache.cloudstack.framework.config.ConfigKey; @@ -173,8 +172,4 @@ public interface AgentManager { void propagateChangeToAgents(Map params); boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs); - - List getLastAgents(); - - void setLastAgents(List lastAgents); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index ca56446631c..6d4bcb7b0d9 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -214,13 +214,13 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl protected final ConfigKey Workers = new ConfigKey<>("Advanced", Integer.class, "workers", "5", "Number of worker threads handling remote agent connections.", false); - protected final ConfigKey Port = new ConfigKey<>("Advanced", Integer.class, "port", "8250", "Port to listen on for remote agent connections.", false); + protected final ConfigKey Port = new ConfigKey<>("Advanced", Integer.class, "port", "8250", "Port to listen on for remote (indirect) agent connections.", false); protected final ConfigKey RemoteAgentSslHandshakeTimeout = new ConfigKey<>("Advanced", Integer.class, "agent.ssl.handshake.timeout", "30", - "Seconds after which SSL handshake times out during remote agent connections.", false); + "Seconds after which SSL handshake times out during remote (indirect) agent connections.", false); protected final ConfigKey RemoteAgentMaxConcurrentNewConnections = new ConfigKey<>("Advanced", Integer.class, "agent.max.concurrent.new.connections", "0", - "Number of maximum concurrent new connections server allows for remote agents. " + + "Number of maximum concurrent new connections server allows for remote (indirect) agents. " + "If set to zero (default value) then no limit will be enforced on concurrent new connections", false); protected final ConfigKey AlertWait = new ConfigKey<>("Advanced", Integer.class, "alert.wait", "1800", @@ -255,9 +255,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl _executor = new ThreadPoolExecutor(agentTaskThreads, agentTaskThreads, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentTaskPool")); - _connectExecutor = new ThreadPoolExecutor(100, 500, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentConnectTaskPool")); - // allow core threads to time out even when there are no items in the queue - _connectExecutor.allowCoreThreadTimeOut(true); + initConnectExecutor(); maxConcurrentNewAgentConnections = RemoteAgentMaxConcurrentNewConnections.value(); @@ -273,10 +271,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl logger.debug("Created DirectAgentAttache pool with size: {}.", directAgentPoolSize); _directAgentThreadCap = Math.round(directAgentPoolSize * DirectAgentThreadCap.value()) + 1; // add 1 to always make the value > 0 - _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor")); - - newAgentConnectionsMonitor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NewAgentConnectionsMonitor")); - initializeCommandTimeouts(); return true; @@ -351,10 +345,27 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl _hostMonitors.remove(id); } + @Override + public void onManagementServerPreparingForMaintenance() { + logger.debug("Management server preparing for maintenance"); + if (_connection != null) { + _connection.block(); + } + } + + @Override + public void onManagementServerCancelPreparingForMaintenance() { + logger.debug("Management server cancel preparing for maintenance"); + if (_connection != null) { + _connection.unblock(); + } + } + @Override public void onManagementServerMaintenance() { logger.debug("Management server maintenance enabled"); _monitorExecutor.shutdownNow(); + newAgentConnectionsMonitor.shutdownNow(); if (_connection != null) { _connection.stop(); @@ -371,10 +382,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl public void onManagementServerCancelMaintenance() { logger.debug("Management server maintenance disabled"); if (_connectExecutor.isShutdown()) { - _connectExecutor = new ThreadPoolExecutor(100, 500, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentConnectTaskPool")); - _connectExecutor.allowCoreThreadTimeOut(true); + initConnectExecutor(); } - startDirectlyConnectedHosts(true); if (_connection != null) { try { @@ -385,9 +394,28 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl } if (_monitorExecutor.isShutdown()) { - _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor")); - _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS); + initAndScheduleMonitorExecutor(); } + if (newAgentConnectionsMonitor.isShutdown()) { + initAndScheduleAgentConnectionsMonitor(); + } + } + + private void initConnectExecutor() { + _connectExecutor = new ThreadPoolExecutor(100, 500, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentConnectTaskPool")); + // allow core threads to time out even when there are no items in the queue + _connectExecutor.allowCoreThreadTimeOut(true); + } + + private void initAndScheduleMonitorExecutor() { + _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor")); + _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS); + } + + private void initAndScheduleAgentConnectionsMonitor() { + final int cleanupTimeInSecs = Wait.value(); + newAgentConnectionsMonitor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NewAgentConnectionsMonitor")); + newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTimeInSecs, cleanupTimeInSecs, TimeUnit.SECONDS); } private AgentControlAnswer handleControlCommand(final AgentAttache attache, final AgentControlCommand cmd) { @@ -426,16 +454,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl return attache; } - @Override - public List getLastAgents() { - return lastAgents; - } - - @Override - public void setLastAgents(List lastAgents) { - this.lastAgents = lastAgents; - } - @Override public Answer sendTo(final Long dcId, final HypervisorType type, final Command cmd) { final List clusters = _clusterDao.listByDcHyType(dcId, type.toString()); @@ -779,6 +797,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl ManagementServerHostVO msHost = _mshostDao.findByMsid(_nodeId); if (msHost != null && (ManagementServerHost.State.Maintenance.equals(msHost.getState()) || ManagementServerHost.State.PreparingForMaintenance.equals(msHost.getState()))) { _monitorExecutor.shutdownNow(); + newAgentConnectionsMonitor.shutdownNow(); return true; } @@ -792,12 +811,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl } } - _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS); - - final int cleanupTime = Wait.value(); - newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTime, - cleanupTime, TimeUnit.MINUTES); - + initAndScheduleMonitorExecutor(); + initAndScheduleAgentConnectionsMonitor(); return true; } @@ -1304,6 +1319,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl if (!indirectAgentLB.compareManagementServerList(host.getId(), host.getDataCenterId(), agentMSHostList, lbAlgorithm)) { final List newMSList = indirectAgentLB.getManagementServerList(host.getId(), host.getDataCenterId(), null); ready.setMsHostList(newMSList); + final List avoidMsList = _mshostDao.listNonUpStateMsIPs(); + ready.setAvoidMsHostList(avoidMsList); ready.setLbAlgorithm(indirectAgentLB.getLBAlgorithmName()); ready.setLbCheckInterval(indirectAgentLB.getLBPreferredHostCheckInterval(host.getClusterId())); logger.debug("Agent's management server host list is not up to date, sending list update: {}", newMSList); @@ -1608,7 +1625,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl if (host!= null && host.getStatus() != Status.Up && gatewayAccessible) { requestStartupCommand = true; } - answer = new PingAnswer((PingCommand)cmd, requestStartupCommand); + final List avoidMsList = _mshostDao.listNonUpStateMsIPs(); + answer = new PingAnswer((PingCommand)cmd, avoidMsList, requestStartupCommand); } else if (cmd instanceof ReadyAnswer) { final HostVO host = _hostDao.findById(attache.getId()); if (host == null) { @@ -1929,25 +1947,19 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl logger.trace("Agent New Connections Monitor is started."); final int cleanupTime = Wait.value(); Set> entrySet = newAgentConnections.entrySet(); - long cutOff = System.currentTimeMillis() - (cleanupTime * 60 * 1000L); - if (logger.isDebugEnabled()) { - List expiredConnections = newAgentConnections.entrySet() - .stream() - .filter(e -> e.getValue() <= cutOff) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - logger.debug("Currently {} active new connections, of which {} have expired - {}", - entrySet.size(), - expiredConnections.size(), - StringUtils.join(expiredConnections)); - } - for (Map.Entry entry : entrySet) { - if (entry.getValue() <= cutOff) { - if (logger.isTraceEnabled()) { - logger.trace("Cleaning up new agent connection for {}", entry.getKey()); - } - newAgentConnections.remove(entry.getKey()); - } + long cutOff = System.currentTimeMillis() - (cleanupTime * 1000L); + List expiredConnections = newAgentConnections.entrySet() + .stream() + .filter(e -> e.getValue() <= cutOff) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + logger.debug("Currently {} active new connections, of which {} have expired - {}", + entrySet.size(), + expiredConnections.size(), + StringUtils.join(expiredConnections)); + for (String connection : expiredConnections) { + logger.trace("Cleaning up new agent connection for {}", connection); + newAgentConnections.remove(connection); } } } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index c667df5412e..dad7d401b94 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -151,11 +151,11 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust super(); } - protected final ConfigKey EnableLB = new ConfigKey<>(Boolean.class, "agent.lb.enabled", "Advanced", "false", "Enable agent load balancing between management server nodes", true); + protected final ConfigKey EnableLB = new ConfigKey<>(Boolean.class, "agent.lb.enabled", "Advanced", "false", "Enable direct agents load balancing between management server nodes", true); protected final ConfigKey ConnectedAgentThreshold = new ConfigKey<>(Double.class, "agent.load.threshold", "Advanced", "0.7", - "What percentage of the agents can be held by one management server before load balancing happens", true, EnableLB.key()); - protected final ConfigKey LoadSize = new ConfigKey<>(Integer.class, "direct.agent.load.size", "Advanced", "16", "How many agents to connect to in each round", true); - protected final ConfigKey ScanInterval = new ConfigKey<>(Integer.class, "direct.agent.scan.interval", "Advanced", "90", "Interval between scans to load agents", false, + "What percentage of the direct agents can be held by one management server before load balancing happens", true, EnableLB.key()); + protected final ConfigKey LoadSize = new ConfigKey<>(Integer.class, "direct.agent.load.size", "Advanced", "16", "How many direct agents to connect to in each round", true); + protected final ConfigKey ScanInterval = new ConfigKey<>(Integer.class, "direct.agent.scan.interval", "Advanced", "90", "Interval between scans to load direct agents", false, ConfigKey.Scope.Global, 1000); @Override @@ -1395,7 +1395,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust return false; } - long transferStartTime = System.currentTimeMillis(); + long transferStartTimeInMs = System.currentTimeMillis(); if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId))) { logger.info("No direct agent hosts available on management server node {} (id: {}), to transfer", fromMsId, fromMsUuid); return true; @@ -1417,7 +1417,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust } logger.debug("Transferring {} direct agents from management server node {} (id: {}) of zone {}", directAgentHostsInDc.size(), fromMsId, fromMsUuid, dc); for (HostVO host : directAgentHostsInDc) { - long transferElapsedTimeInMs = System.currentTimeMillis() - transferStartTime; + long transferElapsedTimeInMs = System.currentTimeMillis() - transferStartTimeInMs; if (transferElapsedTimeInMs >= timeoutDurationInMs) { logger.debug("Stop transferring remaining direct agents from management server node {} (id: {}), timed out", fromMsId, fromMsUuid); return false; @@ -1486,6 +1486,18 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust } } + @Override + public void onManagementServerPreparingForMaintenance() { + logger.debug("Management server preparing for maintenance"); + super.onManagementServerPreparingForMaintenance(); + } + + @Override + public void onManagementServerCancelPreparingForMaintenance() { + logger.debug("Management server cancel preparing for maintenance"); + super.onManagementServerPreparingForMaintenance(); + } + @Override public void onManagementServerMaintenance() { logger.debug("Management server maintenance enabled"); diff --git a/engine/schema/src/main/java/com/cloud/configuration/ManagementServiceConfiguration.java b/engine/schema/src/main/java/com/cloud/configuration/ManagementServiceConfiguration.java index 51b7f62f56d..841447de5fd 100644 --- a/engine/schema/src/main/java/com/cloud/configuration/ManagementServiceConfiguration.java +++ b/engine/schema/src/main/java/com/cloud/configuration/ManagementServiceConfiguration.java @@ -21,7 +21,7 @@ import org.apache.cloudstack.framework.config.Configurable; public interface ManagementServiceConfiguration extends Configurable { ConfigKey PingInterval = new ConfigKey("Advanced", Integer.class, "ping.interval", "60", - "Interval to send application level pings to make sure the connection is still working", false); + "Interval in seconds to send application level pings to make sure the connection is still working", false); ConfigKey PingTimeout = new ConfigKey("Advanced", Float.class, "ping.timeout", "2.5", "Multiplier to ping.interval before announcing an agent has timed out", true); public int getPingInterval(); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index cfd75b1a94b..d44e842db8b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -183,6 +183,13 @@ public interface HostDao extends GenericDao, StateDao listByMs(long msId); + /** + * Retrieves the last host ids/agents this {@see ManagementServer} has responsibility over. + * @param msId the id of the {@see ManagementServer} + * @return the last host ids/agents this {@see ManagementServer} has responsibility over + */ + List listByLastMs(long msId); + /** * Retrieves the hypervisor versions of the hosts in the datacenter which are in Up state in ascending order * @param datacenterId data center id @@ -200,7 +207,7 @@ public interface HostDao extends GenericDao, StateDao findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long zoneId, final Long clusterId, - final List resourceStates, final List types, + final Long msId, final List resourceStates, final List types, final List hypervisorTypes); List listDistinctHypervisorTypes(final Long zoneId); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 54146e55049..fac895400f3 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -129,6 +129,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected SearchBuilder ResponsibleMsSearch; protected SearchBuilder ResponsibleMsDcSearch; protected GenericSearchBuilder ResponsibleMsIdSearch; + protected GenericSearchBuilder LastMsIdSearch; protected SearchBuilder HostTypeClusterCountSearch; protected SearchBuilder HostTypeZoneCountSearch; protected SearchBuilder ClusterStatusSearch; @@ -209,6 +210,11 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao ResponsibleMsIdSearch.and("managementServerId", ResponsibleMsIdSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ); ResponsibleMsIdSearch.done(); + LastMsIdSearch = createSearchBuilder(String.class); + LastMsIdSearch.selectFields(LastMsIdSearch.entity().getUuid()); + LastMsIdSearch.and("lastManagementServerId", LastMsIdSearch.entity().getLastManagementServerId(), SearchCriteria.Op.EQ); + LastMsIdSearch.done(); + HostTypeClusterCountSearch = createSearchBuilder(); HostTypeClusterCountSearch.and("cluster", HostTypeClusterCountSearch.entity().getClusterId(), SearchCriteria.Op.EQ); HostTypeClusterCountSearch.and("type", HostTypeClusterCountSearch.entity().getType(), SearchCriteria.Op.EQ); @@ -1569,6 +1575,13 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return customSearch(sc, null); } + @Override + public List listByLastMs(long msId) { + SearchCriteria sc = LastMsIdSearch.create(); + sc.addAnd("lastManagementServerId", SearchCriteria.Op.EQ, msId); + return customSearch(sc, null); + } + @Override public List listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType) { PreparedStatement pstmt; @@ -1745,13 +1758,15 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao } @Override - public List findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long zoneId, final Long clusterId, + public List findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long zoneId, + final Long clusterId, final Long managementServerId, final List resourceStates, final List types, final List hypervisorTypes) { GenericSearchBuilder sb = createSearchBuilder(Long.class); sb.selectFields(sb.entity().getId()); sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); + sb.and("msId", sb.entity().getManagementServerId(), SearchCriteria.Op.EQ); sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.IN); sb.and("type", sb.entity().getType(), SearchCriteria.Op.IN); if (CollectionUtils.isNotEmpty(hypervisorTypes)) { @@ -1767,6 +1782,9 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao if (clusterId != null) { sc.setParameters("clusterId", clusterId); } + if (managementServerId != null) { + sc.setParameters("msId", managementServerId); + } if (CollectionUtils.isNotEmpty(hypervisorTypes)) { sc.setParameters("hypervisorTypes", hypervisorTypes.toArray()); } diff --git a/engine/schema/src/test/java/com/cloud/host/dao/HostDaoImplTest.java b/engine/schema/src/test/java/com/cloud/host/dao/HostDaoImplTest.java index 81163321c6b..8f41162f242 100644 --- a/engine/schema/src/test/java/com/cloud/host/dao/HostDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/host/dao/HostDaoImplTest.java @@ -104,6 +104,7 @@ public class HostDaoImplTest { public void testFindHostIdsByZoneClusterResourceStateTypeAndHypervisorType() { Long zoneId = 1L; Long clusterId = 2L; + Long msId = 1L; List resourceStates = List.of(ResourceState.Enabled); List types = List.of(Host.Type.Routing); List hypervisorTypes = List.of(Hypervisor.HypervisorType.KVM); @@ -117,10 +118,11 @@ public class HostDaoImplTest { Mockito.doReturn(sb).when(hostDao).createSearchBuilder(Long.class); Mockito.doReturn(mockResults).when(hostDao).customSearch(Mockito.any(SearchCriteria.class), Mockito.any()); List hostIds = hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType( - zoneId, clusterId, resourceStates, types, hypervisorTypes); + zoneId, clusterId, msId, resourceStates, types, hypervisorTypes); Assert.assertEquals(mockResults, hostIds); Mockito.verify(sc).setParameters("zoneId", zoneId); Mockito.verify(sc).setParameters("clusterId", clusterId); + Mockito.verify(sc).setParameters("msId", msId); Mockito.verify(sc).setParameters("resourceState", resourceStates.toArray()); Mockito.verify(sc).setParameters("type", types.toArray()); Mockito.verify(sc).setParameters("hypervisorTypes", hypervisorTypes.toArray()); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 448a4eb219c..41af291bd69 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -237,7 +237,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, } } - throw new CloudRuntimeException("Maintenance or Shutdown has been initiated on this management server. Can not accept new jobs"); + throw new CloudRuntimeException("Maintenance or Shutdown has been initiated on this management server. Can not accept new async jobs"); } private boolean checkSyncQueueItemAllowed(SyncQueueItemVO item) { diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceListener.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceListener.java index bd82d1b257d..e0fe49a19ac 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceListener.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceListener.java @@ -18,6 +18,10 @@ package org.apache.cloudstack.maintenance; public interface ManagementServerMaintenanceListener { + void onManagementServerPreparingForMaintenance(); + + void onManagementServerCancelPreparingForMaintenance(); + void onManagementServerMaintenance(); void onManagementServerCancelMaintenance(); diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java index d474f718826..3af19164cc9 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java @@ -44,6 +44,10 @@ public interface ManagementServerMaintenanceManager { void unregisterListener(ManagementServerMaintenanceListener listener); + void onPreparingForMaintenance(); + + void onCancelPreparingForMaintenance(); + void onMaintenance(); void onCancelMaintenance(); diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java index 0af8a7c114d..fcfa32d6ce8 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java @@ -53,6 +53,9 @@ import com.cloud.agent.api.Command; import com.cloud.cluster.ClusterManager; import com.cloud.cluster.ManagementServerHostVO; import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.serializer.GsonHelper; import com.cloud.utils.StringUtils; @@ -108,6 +111,25 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen return true; } + @Override + public boolean stop() { + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost != null) { + updateLastManagementServerForHosts(msHost.getMsid()); + } + return true; + } + + private void updateLastManagementServerForHosts(long msId) { + List hosts = hostDao.listHostsByMs(msId); + for (HostVO host : hosts) { + if (host != null) { + host.setLastManagementServerId(msId); + hostDao.update(host.getId(), host); + } + } + } + @Override public void registerListener(ManagementServerMaintenanceListener listener) { synchronized (_listeners) { @@ -124,6 +146,26 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } } + @Override + public void onPreparingForMaintenance() { + synchronized (_listeners) { + for (final ManagementServerMaintenanceListener listener : _listeners) { + logger.info("Invoke, on preparing for maintenance for listener " + listener.getClass()); + listener.onManagementServerPreparingForMaintenance(); + } + } + } + + @Override + public void onCancelPreparingForMaintenance() { + synchronized (_listeners) { + for (final ManagementServerMaintenanceListener listener : _listeners) { + logger.info("Invoke, on cancel preparing for maintenance for listener " + listener.getClass()); + listener.onManagementServerCancelPreparingForMaintenance(); + } + } + } + @Override public void onMaintenance() { synchronized (_listeners) { @@ -243,6 +285,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen this.maintenanceStartTime = System.currentTimeMillis(); this.lbAlgorithm = lbAlorithm; jobManager.disableAsyncJobs(); + onPreparingForMaintenance(); waitForPendingJobs(); } @@ -257,8 +300,13 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen jobManager.enableAsyncJobs(); cancelWaitForPendingJobs(); ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); - if (msHost != null && State.Maintenance.equals(msHost.getState())) { - onCancelMaintenance(); + if (msHost != null) { + if (State.PreparingForMaintenance.equals(msHost.getState())) { + onCancelPreparingForMaintenance(); + } + if (State.Maintenance.equals(msHost.getState())) { + onCancelMaintenance(); + } } } @@ -284,6 +332,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } @Override + @ActionEvent(eventType = EventTypes.EVENT_MS_SHUTDOWN_PREPARE, eventDescription = "preparing for shutdown") public ManagementServerMaintenanceResponse prepareForShutdown(PrepareForShutdownCmd cmd) { ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); if (msHost == null) { @@ -294,19 +343,18 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen throw new CloudRuntimeException("Management server is not in the right state to prepare for shutdown"); } + checkAnyMsInPreparingStates("prepare for shutdown"); + final Command[] cmds = new Command[1]; cmds[0] = new PrepareForShutdownManagementServerHostCommand(msHost.getMsid()); - String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); - logger.info("PrepareForShutdownCmd result : " + result); - if (!result.startsWith("Success")) { - throw new CloudRuntimeException(result); - } + executeCmd(msHost, cmds); msHostDao.updateState(msHost.getId(), State.PreparingForShutDown); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @Override + @ActionEvent(eventType = EventTypes.EVENT_MS_SHUTDOWN, eventDescription = "triggering shutdown") public ManagementServerMaintenanceResponse triggerShutdown(TriggerShutdownCmd cmd) { ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); if (msHost == null) { @@ -319,22 +367,20 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } if (State.Up.equals(msHost.getState())) { + checkAnyMsInPreparingStates("trigger shutdown"); msHostDao.updateState(msHost.getId(), State.PreparingForShutDown); } final Command[] cmds = new Command[1]; cmds[0] = new TriggerShutdownManagementServerHostCommand(msHost.getMsid()); - String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); - logger.info("TriggerShutdownCmd result : " + result); - if (!result.startsWith("Success")) { - throw new CloudRuntimeException(result); - } + executeCmd(msHost, cmds); msHostDao.updateState(msHost.getId(), State.ShuttingDown); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @Override + @ActionEvent(eventType = EventTypes.EVENT_MS_SHUTDOWN_CANCEL, eventDescription = "cancelling shutdown") public ManagementServerMaintenanceResponse cancelShutdown(CancelShutdownCmd cmd) { ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); if (msHost == null) { @@ -347,17 +393,14 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen final Command[] cmds = new Command[1]; cmds[0] = new CancelShutdownManagementServerHostCommand(msHost.getMsid()); - String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); - logger.info("CancelShutdownCmd result : " + result); - if (!result.startsWith("Success")) { - throw new CloudRuntimeException(result); - } + executeCmd(msHost, cmds); msHostDao.updateState(msHost.getId(), State.Up); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @Override + @ActionEvent(eventType = EventTypes.EVENT_MS_MAINTENANCE_PREPARE, eventDescription = "preparing for maintenance") public ManagementServerMaintenanceResponse prepareForMaintenance(PrepareForMaintenanceCmd cmd) { if (StringUtils.isNotBlank(cmd.getAlgorithm())) { indirectAgentLB.checkLBAlgorithmName(cmd.getAlgorithm()); @@ -381,10 +424,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen throw new CloudRuntimeException("Management server is not in the right state to prepare for maintenance"); } - final List preparingForMaintenanceMsList = msHostDao.listBy(State.PreparingForMaintenance); - if (CollectionUtils.isNotEmpty(preparingForMaintenanceMsList)) { - throw new CloudRuntimeException("Cannot prepare for maintenance, there are other management servers preparing for maintenance"); - } + checkAnyMsInPreparingStates("prepare for maintenance"); if (indirectAgentLB.haveAgentBasedHosts(msHost.getMsid())) { List indirectAgentMsList = indirectAgentLB.getManagementServerList(); @@ -396,23 +436,16 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } } - List lastAgents = hostDao.listByMs(cmd.getManagementServerId()); - agentMgr.setLastAgents(lastAgents); - final Command[] cmds = new Command[1]; cmds[0] = new PrepareForMaintenanceManagementServerHostCommand(msHost.getMsid(), cmd.getAlgorithm()); - String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); - logger.info("PrepareForMaintenanceCmd result : " + result); - if (!result.startsWith("Success")) { - agentMgr.setLastAgents(null); - throw new CloudRuntimeException(result); - } + executeCmd(msHost, cmds); msHostDao.updateState(msHost.getId(), State.PreparingForMaintenance); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @Override + @ActionEvent(eventType = EventTypes.EVENT_MS_MAINTENANCE_CANCEL, eventDescription = "cancelling maintenance") public ManagementServerMaintenanceResponse cancelMaintenance(CancelMaintenanceCmd cmd) { ManagementServerHostVO msHost = msHostDao.findById(cmd.getManagementServerId()); if (msHost == null) { @@ -425,15 +458,29 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen final Command[] cmds = new Command[1]; cmds[0] = new CancelMaintenanceManagementServerHostCommand(msHost.getMsid()); - String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), true); - logger.info("CancelMaintenanceCmd result : " + result); + executeCmd(msHost, cmds); + + msHostDao.updateState(msHost.getId(), State.Up); + return prepareMaintenanceResponse(cmd.getManagementServerId()); + } + + private void executeCmd(ManagementServerHostVO msHost, Command[] cmds) { + if (msHost == null) { + throw new CloudRuntimeException("Management server node not specified, to execute the cmd"); + } + if (cmds == null || cmds.length <= 0) { + throw new CloudRuntimeException(String.format("Cmd not specified, to execute on the management server node %s", msHost)); + } + String result = clusterManager.execute(String.valueOf(msHost.getMsid()), 0, gson.toJson(cmds), false); + if (result == null) { + String msg = String.format("Unable to reach or execute %s on the management server node: %s", cmds[0], msHost); + logger.warn(msg); + throw new CloudRuntimeException(msg); + } + logger.info(String.format("Cmd %s - result: %s", cmds[0], result)); if (!result.startsWith("Success")) { throw new CloudRuntimeException(result); } - - msHostDao.updateState(msHost.getId(), State.Up); - agentMgr.setLastAgents(null); - return prepareMaintenanceResponse(cmd.getManagementServerId()); } @Override @@ -445,9 +492,17 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen if (msHost == null) { msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); } + onCancelPreparingForMaintenance(); msHostDao.updateState(msHost.getId(), State.Up); } + private void checkAnyMsInPreparingStates(String operation) { + final List preparingForMaintenanceOrShutDownMsList = msHostDao.listBy(State.PreparingForMaintenance, State.PreparingForShutDown); + if (CollectionUtils.isNotEmpty(preparingForMaintenanceOrShutDownMsList)) { + throw new CloudRuntimeException(String.format("Cannot %s, there are other management servers preparing for maintenance/shutdown", operation)); + } + } + private ManagementServerMaintenanceResponse prepareMaintenanceResponse(Long managementServerId) { ManagementServerHostVO msHost; Long[] msIds; @@ -465,8 +520,8 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen boolean maintenanceInitiatedForMS = Arrays.asList(maintenanceStates).contains(msHost.getState()); boolean shutdownTriggeredForMS = Arrays.asList(shutdownStates).contains(msHost.getState()); msIds = new Long[]{msHost.getMsid()}; - List agents = hostDao.listByMs(managementServerId); - long agentsCount = hostDao.countByMs(managementServerId); + List agents = hostDao.listByMs(msHost.getMsid()); + long agentsCount = agents.size(); long pendingJobCount = countPendingJobs(msIds); return new ManagementServerMaintenanceResponse(msHost.getUuid(), msHost.getState(), maintenanceInitiatedForMS, shutdownTriggeredForMS, pendingJobCount == 0, pendingJobCount, agentsCount, agents); } @@ -535,7 +590,6 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen // No more pending jobs. Good to terminate if (managementServerMaintenanceManager.isShutdownTriggered()) { logger.info("MS is Shutting Down Now"); - // update state to down ? System.exit(0); } if (managementServerMaintenanceManager.isPreparingForMaintenance()) { diff --git a/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java b/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java index 8e1c09bf995..dc14124d018 100644 --- a/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java +++ b/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java @@ -17,7 +17,23 @@ package org.apache.cloudstack.maintenance; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.cloudstack.agent.lb.IndirectAgentLB; +import org.apache.cloudstack.api.command.CancelMaintenanceCmd; +import org.apache.cloudstack.api.command.CancelShutdownCmd; +import org.apache.cloudstack.api.command.PrepareForMaintenanceCmd; +import org.apache.cloudstack.api.command.PrepareForShutdownCmd; +import org.apache.cloudstack.api.command.TriggerShutdownCmd; import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.management.ManagementServerHost; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,6 +43,11 @@ import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import com.cloud.agent.AgentManager; +import com.cloud.cluster.ClusterManager; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.host.dao.HostDao; import com.cloud.utils.exception.CloudRuntimeException; @@ -40,6 +61,21 @@ public class ManagementServerMaintenanceManagerImplTest { @Mock AsyncJobManager jobManagerMock; + @Mock + IndirectAgentLB indirectAgentLBMock; + + @Mock + AgentManager agentManagerMock; + + @Mock + ClusterManager clusterManagerMock; + + @Mock + HostDao hostDao; + + @Mock + ManagementServerHostDao msHostDao; + private long prepareCountPendingJobs() { long expectedCount = 1L; Mockito.doReturn(expectedCount).when(jobManagerMock).countPendingNonPseudoJobs(1L); @@ -53,13 +89,6 @@ public class ManagementServerMaintenanceManagerImplTest { Assert.assertEquals(expectedCount, count); } - @Test - public void cancelShutdown() { - Assert.assertThrows(CloudRuntimeException.class, () -> { - spy.cancelShutdown(); - }); - } - @Test public void prepareForShutdown() { Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); @@ -74,4 +103,463 @@ public class ManagementServerMaintenanceManagerImplTest { spy.cancelShutdown(); Mockito.verify(jobManagerMock).enableAsyncJobs(); } + + @Test + public void cancelShutdown() { + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.cancelShutdown(); + }); + } + + @Test + public void triggerShutdown() { + Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); + Mockito.lenient().when(spy.isShutdownTriggered()).thenReturn(false); + spy.triggerShutdown(); + Mockito.verify(jobManagerMock).disableAsyncJobs(); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.triggerShutdown(); + }); + } + + @Test + public void prepareForShutdownCmdNoMsHost() { + Mockito.when(msHostDao.findById(1L)).thenReturn(null); + PrepareForShutdownCmd cmd = mock(PrepareForShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForShutdown(cmd); + }); + } + + @Test + public void prepareForShutdownCmdMsHostWithNonUpState() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Maintenance); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + PrepareForShutdownCmd cmd = mock(PrepareForShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForShutdown(cmd); + }); + } + + @Test + public void prepareForShutdownCmdOtherMsHostsInPreparingState() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Up); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(any())).thenReturn(msHostList); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + PrepareForShutdownCmd cmd = mock(PrepareForShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForShutdown(cmd); + }); + } + + @Test + public void prepareForShutdownCmdNullResponseFromClusterManager() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Up); + List msHostList = new ArrayList<>(); + Mockito.when(msHostDao.listBy(any())).thenReturn(msHostList); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + PrepareForShutdownCmd cmd = mock(PrepareForShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn(null); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForShutdown(cmd); + }); + } + + @Test + public void prepareForShutdownCmdFailedResponseFromClusterManager() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Up); + List msHostList = new ArrayList<>(); + Mockito.when(msHostDao.listBy(any())).thenReturn(msHostList); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + PrepareForShutdownCmd cmd = mock(PrepareForShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Failed"); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForShutdown(cmd); + }); + } + + @Test + public void prepareForShutdownCmdSuccessResponseFromClusterManager() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Up); + Mockito.when(msHostDao.listBy(any())).thenReturn(new ArrayList<>()); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + Mockito.when(hostDao.listByMs(anyLong())).thenReturn(new ArrayList<>()); + PrepareForShutdownCmd cmd = mock(PrepareForShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Success"); + + spy.prepareForShutdown(cmd); + Mockito.verify(clusterManagerMock, Mockito.times(1)).execute(anyString(), anyLong(), anyString(), anyBoolean()); + } + + @Test + public void cancelShutdownCmdNoMsHost() { + Mockito.when(msHostDao.findById(1L)).thenReturn(null); + CancelShutdownCmd cmd = mock(CancelShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.cancelShutdown(cmd); + }); + } + + @Test + public void cancelShutdownCmdMsHostNotInShutdownState() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Up); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + CancelShutdownCmd cmd = mock(CancelShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.cancelShutdown(cmd); + }); + } + + @Test + public void cancelShutdownCmd() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.ReadyToShutDown); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + CancelShutdownCmd cmd = mock(CancelShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Success"); + + spy.cancelShutdown(cmd); + Mockito.verify(clusterManagerMock, Mockito.times(1)).execute(anyString(), anyLong(), anyString(), anyBoolean()); + } + + @Test + public void triggerShutdownCmdNoMsHost() { + Mockito.when(msHostDao.findById(1L)).thenReturn(null); + TriggerShutdownCmd cmd = mock(TriggerShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.triggerShutdown(cmd); + }); + } + + @Test + public void triggerShutdownCmdMsHostWithNotRightState() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.PreparingForMaintenance); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + TriggerShutdownCmd cmd = mock(TriggerShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.triggerShutdown(cmd); + }); + } + + @Test + public void triggerShutdownCmdMsInUpStateAndOtherMsHostsInPreparingState() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Up); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(any())).thenReturn(msHostList); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + TriggerShutdownCmd cmd = mock(TriggerShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.triggerShutdown(cmd); + }); + } + + @Test + public void triggerShutdownCmd() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.ReadyToShutDown); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + TriggerShutdownCmd cmd = mock(TriggerShutdownCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Success"); + + spy.triggerShutdown(cmd); + Mockito.verify(clusterManagerMock, Mockito.times(1)).execute(anyString(), anyLong(), anyString(), anyBoolean()); + } + + @Test + public void prepareForMaintenanceAndCancelFromMaintenanceState() { + Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); + spy.prepareForMaintenance("static"); + Mockito.verify(jobManagerMock).disableAsyncJobs(); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance("static"); + }); + + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Maintenance); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); + Mockito.doNothing().when(jobManagerMock).enableAsyncJobs(); + spy.cancelMaintenance(); + Mockito.verify(jobManagerMock).enableAsyncJobs(); + Mockito.verify(spy, Mockito.times(1)).onCancelMaintenance(); + } + + @Test + public void prepareForMaintenanceAndCancelFromPreparingForMaintenanceState() { + Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); + spy.prepareForMaintenance("static"); + Mockito.verify(jobManagerMock).disableAsyncJobs(); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance("static"); + }); + + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.PreparingForMaintenance); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); + Mockito.doNothing().when(jobManagerMock).enableAsyncJobs(); + spy.cancelMaintenance(); + Mockito.verify(jobManagerMock).enableAsyncJobs(); + Mockito.verify(spy, Mockito.times(1)).onCancelPreparingForMaintenance(); + } + + @Test + public void cancelMaintenance() { + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.cancelMaintenance(); + }); + } + + @Test + public void cancelPreparingForMaintenance() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); + + spy.cancelPreparingForMaintenance(null); + Mockito.verify(jobManagerMock).enableAsyncJobs(); + Mockito.verify(spy, Mockito.times(1)).onCancelPreparingForMaintenance(); + } + + @Test + public void prepareForMaintenanceCmdNoOtherMsHostsWithUpState() { + Mockito.when(msHostDao.listBy(any())).thenReturn(new ArrayList<>()); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getAlgorithm()).thenReturn("test algorithm"); + Mockito.doNothing().when(indirectAgentLBMock).checkLBAlgorithmName(anyString()); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdOnlyOneMsHostsWithUpState() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getAlgorithm()).thenReturn("test algorithm"); + Mockito.doNothing().when(indirectAgentLBMock).checkLBAlgorithmName(anyString()); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdNoMsHost() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost1); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList); + Mockito.when(msHostDao.findById(1L)).thenReturn(null); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdMsHostWithNonUpState() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Maintenance); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost1); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdOtherMsHostsInPreparingState() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Up); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList1 = new ArrayList<>(); + msHostList1.add(msHost1); + msHostList1.add(msHost2); + ManagementServerHostVO msHost3 = mock(ManagementServerHostVO.class); + List msHostList2 = new ArrayList<>(); + msHostList2.add(msHost3); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList1); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.PreparingForMaintenance, ManagementServerHost.State.PreparingForShutDown)).thenReturn(msHostList2); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdNoIndirectMsHosts() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Up); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost1); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.PreparingForMaintenance, ManagementServerHost.State.PreparingForShutDown)).thenReturn(new ArrayList<>()); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + Mockito.when(msHostDao.listNonUpStateMsIPs()).thenReturn(new ArrayList<>()); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(true); + Mockito.when(indirectAgentLBMock.getManagementServerList()).thenReturn(new ArrayList<>()); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdNullResponseFromClusterManager() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Up); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost1); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.PreparingForMaintenance, ManagementServerHost.State.PreparingForShutDown)).thenReturn(new ArrayList<>()); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn(null); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdFailedResponseFromClusterManager() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Up); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost1); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.PreparingForMaintenance, ManagementServerHost.State.PreparingForShutDown)).thenReturn(new ArrayList<>()); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Failed"); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.prepareForMaintenance(cmd); + }); + } + + @Test + public void prepareForMaintenanceCmdSuccessResponseFromClusterManager() { + ManagementServerHostVO msHost1 = mock(ManagementServerHostVO.class); + Mockito.when(msHost1.getState()).thenReturn(ManagementServerHost.State.Up); + ManagementServerHostVO msHost2 = mock(ManagementServerHostVO.class); + List msHostList = new ArrayList<>(); + msHostList.add(msHost1); + msHostList.add(msHost2); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.Up)).thenReturn(msHostList); + Mockito.when(msHostDao.listBy(ManagementServerHost.State.PreparingForMaintenance, ManagementServerHost.State.PreparingForShutDown)).thenReturn(new ArrayList<>()); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); + PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false); + Mockito.when(hostDao.listByMs(anyLong())).thenReturn(new ArrayList<>()); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Success"); + + spy.prepareForMaintenance(cmd); + Mockito.verify(clusterManagerMock, Mockito.times(1)).execute(anyString(), anyLong(), anyString(), anyBoolean()); + } + + @Test + public void cancelMaintenanceCmdNoMsHost() { + Mockito.when(msHostDao.findById(1L)).thenReturn(null); + CancelMaintenanceCmd cmd = mock(CancelMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.cancelMaintenance(cmd); + }); + } + + @Test + public void cancelMaintenanceCmdMsHostNotInMaintenanceState() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Up); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + CancelMaintenanceCmd cmd = mock(CancelMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + + Assert.assertThrows(CloudRuntimeException.class, () -> { + spy.cancelMaintenance(cmd); + }); + } + + @Test + public void cancelMaintenanceCmd() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Maintenance); + Mockito.when(msHostDao.findById(1L)).thenReturn(msHost); + CancelMaintenanceCmd cmd = mock(CancelMaintenanceCmd.class); + Mockito.when(cmd.getManagementServerId()).thenReturn(1L); + Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Success"); + + spy.cancelMaintenance(cmd); + Mockito.verify(clusterManagerMock, Mockito.times(1)).execute(anyString(), anyLong(), anyString(), anyBoolean()); + } } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/response/ManagementServerMetricsResponse.java b/plugins/metrics/src/main/java/org/apache/cloudstack/response/ManagementServerMetricsResponse.java index d96f5b14f0d..83c6f3dc7d4 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/response/ManagementServerMetricsResponse.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/response/ManagementServerMetricsResponse.java @@ -31,11 +31,11 @@ public class ManagementServerMetricsResponse extends ManagementServerResponse { private Integer availableProcessors; @SerializedName(MetricConstants.LAST_AGENTS) - @Param(description = "the last agents this Management Server is responsible for, before preparing for maintenance", since = "4.18.1") + @Param(description = "the last agents this Management Server is responsible for, before shutdown or preparing for maintenance", since = "4.21.0.0") private List lastAgents; @SerializedName(MetricConstants.AGENTS) - @Param(description = "the agents this Management Server is responsible for", since = "4.18.1") + @Param(description = "the agents this Management Server is responsible for", since = "4.21.0.0") private List agents; @SerializedName(MetricConstants.AGENT_COUNT) diff --git a/server/src/main/java/com/cloud/api/ApiDispatcher.java b/server/src/main/java/com/cloud/api/ApiDispatcher.java index 6a43ff10f31..90cbb6afc8e 100644 --- a/server/src/main/java/com/cloud/api/ApiDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiDispatcher.java @@ -94,7 +94,7 @@ public class ApiDispatcher { if (asyncJobManager.isAsyncJobsEnabled()) { asyncCreationDispatchChain.dispatch(new DispatchTask(cmd, params)); } else { - throw new CloudRuntimeException("Maintenance or Shutdown has been initiated on this management server. Can not accept new jobs"); + throw new CloudRuntimeException("Maintenance or Shutdown has been initiated on this management server. Can not accept new async creation jobs"); } } diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index 8964001a5d0..df72a2719c2 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -750,6 +750,11 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer // BaseAsyncCreateCmd: cmd params are processed and create() is called, then same workflow as BaseAsyncCmd. // BaseAsyncCmd: cmd is processed and submitted as an AsyncJob, job related info is serialized and returned. if (cmdObj instanceof BaseAsyncCmd) { + if (!asyncMgr.isAsyncJobsEnabled()) { + String msg = "Maintenance or Shutdown has been initiated on this management server. Can not accept new jobs"; + logger.warn(msg); + throw new ServerApiException(ApiErrorCode.SERVICE_UNAVAILABLE, msg); + } Long objectId = null; String objectUuid = null; if (cmdObj instanceof BaseAsyncCreateCmd) { diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 8bf84fcac15..cbbf664014d 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -5445,7 +5445,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q mgmtResponse.addPeer(createPeerManagementServerNodeResponse(peer)); } } - mgmtResponse.setAgentsCount((long) hostDao.countByMs(mgmt.getMsid())); + List lastAgents = hostDao.listByLastMs(mgmt.getMsid()); + mgmtResponse.setLastAgents(lastAgents); + List agents = hostDao.listByMs(mgmt.getMsid()); + mgmtResponse.setAgents(agents); + mgmtResponse.setAgentsCount((long) agents.size()); mgmtResponse.setPendingJobsCount(jobManager.countPendingNonPseudoJobs(mgmt.getMsid())); mgmtResponse.setIpAddress(mgmt.getServiceIP()); mgmtResponse.setObjectName("managementserver"); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index cbd175eece2..e82d99028d7 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -752,21 +752,21 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc logger.debug(String.format("%s is running...", this.getClass().getSimpleName())); long msid = ManagementServerNode.getManagementServerId(); ManagementServerHostVO mshost = null; - ManagementServerHostStatsEntry hostStatsEntry = null; + ManagementServerHostStatsEntry msHostStatsEntry = null; try { mshost = managementServerHostDao.findByMsid(msid); // get local data - hostStatsEntry = getDataFrom(mshost); - managementServerHostStats.put(mshost.getUuid(), hostStatsEntry); + msHostStatsEntry = getDataFrom(mshost); + managementServerHostStats.put(mshost.getUuid(), msHostStatsEntry); // send to other hosts - clusterManager.publishStatus(gson.toJson(hostStatsEntry)); + clusterManager.publishStatus(gson.toJson(msHostStatsEntry)); } catch (Throwable t) { // pokemon catch to make sure the thread stays running logger.error("Error trying to retrieve management server host statistics", t); } try { // send to DB - storeStatus(hostStatsEntry, mshost); + storeStatus(msHostStatsEntry, mshost); } catch (Throwable t) { // pokemon catch to make sure the thread stays running logger.error("Error trying to store management server host statistics", t); @@ -834,11 +834,11 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc } private void getDataBaseStatistics(ManagementServerHostStatsEntry newEntry, long msid) { - newEntry.setLastAgents(_agentMgr.getLastAgents()); + List lastAgents = _hostDao.listByLastMs(msid); + newEntry.setLastAgents(lastAgents); List agents = _hostDao.listByMs(msid); newEntry.setAgents(agents); - int count = _hostDao.countByMs(msid); - newEntry.setAgentCount(count); + newEntry.setAgentCount(agents.size()); } private void getMemoryData(@NotNull ManagementServerHostStatsEntry newEntry) { diff --git a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java index 1f0f439d819..3336d44dba8 100644 --- a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java @@ -19,10 +19,13 @@ package org.apache.cloudstack.agent.lb; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.EnumSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -33,6 +36,8 @@ import org.apache.cloudstack.agent.lb.algorithm.IndirectAgentLBStaticAlgorithm; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.agent.AgentManager; @@ -40,6 +45,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.MigrateAgentConnectionCommand; import com.cloud.cluster.ManagementServerHostVO; import com.cloud.cluster.dao.ManagementServerHostDao; +import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; @@ -49,20 +55,20 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ResourceState; import com.cloud.utils.component.ComponentLifecycleBase; +import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.collections.CollectionUtils; - public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implements IndirectAgentLB, Configurable { public static final ConfigKey IndirectAgentLBAlgorithm = new ConfigKey<>(String.class, "indirect.agent.lb.algorithm", "Advanced", "static", - "The algorithm to be applied on the provided 'host' management server list that is sent to indirect agents. Allowed values are: static, roundrobin and shuffle.", + "The algorithm to be applied on the provided management server list in the 'host' config that that is sent to indirect agents. Allowed values are: static, roundrobin and shuffle.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "static,roundrobin,shuffle"); public static final ConfigKey IndirectAgentLBCheckInterval = new ConfigKey<>("Advanced", Long.class, "indirect.agent.lb.check.interval", "0", - "The interval in seconds after which agent should check and try to connect to its preferred host. Set 0 to disable it.", + "The interval in seconds after which indirect agent should check and try to connect to its preferred host (the first management server from the propagated list provided in the 'host' config)." + + " Set 0 to disable it.", true, ConfigKey.Scope.Cluster); private static Map algorithmMap = new HashMap<>(); @@ -85,6 +91,8 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement ResourceState.ErrorInMaintenance, ResourceState.PrepareForMaintenance); private static final List agentValidHostTypes = List.of(Host.Type.Routing, Host.Type.ConsoleProxy, Host.Type.SecondaryStorage, Host.Type.SecondaryStorageVM); + private static final List agentNonRoutingHostTypes = List.of(Host.Type.ConsoleProxy, + Host.Type.SecondaryStorage, Host.Type.SecondaryStorageVM); private static final List agentValidHypervisorTypes = List.of( Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.LXC); @@ -246,8 +254,18 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement agentBasedHosts.add(host); } + private List getAllAgentBasedNonRoutingHostsFromDB(final Long zoneId, final Long msId) { + return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, null, msId, + agentValidResourceStates, agentNonRoutingHostTypes, agentValidHypervisorTypes); + } + + private List getAllAgentBasedRoutingHostsFromDB(final Long zoneId, final Long clusterId, final Long msId) { + return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, clusterId, msId, + agentValidResourceStates, List.of(Host.Type.Routing), agentValidHypervisorTypes); + } + private List getAllAgentBasedHostsFromDB(final Long zoneId, final Long clusterId) { - return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, clusterId, + return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, clusterId, null, agentValidResourceStates, agentValidHostTypes, agentValidHypervisorTypes); } @@ -287,31 +305,159 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement @Override public void propagateMSListToAgents() { logger.debug("Propagating management server list update to agents"); + ExecutorService setupMSListExecutorService = Executors.newFixedThreadPool(10, new NamedThreadFactory("SetupMSList-Worker")); final String lbAlgorithm = getLBAlgorithmName(); + final Long globalLbCheckInterval = getLBPreferredHostCheckInterval(null); List zones = dataCenterDao.listAll(); for (DataCenterVO zone : zones) { List zoneHostIds = new ArrayList<>(); + List nonRoutingHostIds = getAllAgentBasedNonRoutingHostsFromDB(zone.getId(), null); + zoneHostIds.addAll(nonRoutingHostIds); Map> clusterHostIdsMap = new HashMap<>(); List clusterIds = clusterDao.listAllClusterIds(zone.getId()); for (Long clusterId : clusterIds) { - List hostIds = getAllAgentBasedHostsFromDB(zone.getId(), clusterId); + List hostIds = getAllAgentBasedRoutingHostsFromDB(zone.getId(), clusterId, null); clusterHostIdsMap.put(clusterId, hostIds); zoneHostIds.addAll(hostIds); } zoneHostIds.sort(Comparator.comparingLong(x -> x)); + final List avoidMsList = mshostDao.listNonUpStateMsIPs(); + for (Long nonRoutingHostId : nonRoutingHostIds) { + setupMSListExecutorService.submit(new SetupMSListTask(nonRoutingHostId, zone.getId(), zoneHostIds, avoidMsList, lbAlgorithm, globalLbCheckInterval)); + } for (Long clusterId : clusterIds) { - final Long lbCheckInterval = getLBPreferredHostCheckInterval(clusterId); + final Long clusterLbCheckInterval = getLBPreferredHostCheckInterval(clusterId); List hostIds = clusterHostIdsMap.get(clusterId); for (Long hostId : hostIds) { - final List msList = getManagementServerList(hostId, zone.getId(), zoneHostIds); - final SetupMSListCommand cmd = new SetupMSListCommand(msList, lbAlgorithm, lbCheckInterval); - final Answer answer = agentManager.easySend(hostId, cmd); - if (answer == null || !answer.getResult()) { - logger.warn("Failed to setup management servers list to the agent of ID: {}", hostId); - } + setupMSListExecutorService.submit(new SetupMSListTask(hostId, zone.getId(), zoneHostIds, avoidMsList, lbAlgorithm, clusterLbCheckInterval)); } } } + + setupMSListExecutorService.shutdown(); + try { + if (!setupMSListExecutorService.awaitTermination(300, TimeUnit.SECONDS)) { + setupMSListExecutorService.shutdownNow(); + } + } catch (InterruptedException e) { + setupMSListExecutorService.shutdownNow(); + logger.debug(String.format("Force shutdown setup ms list service as it did not shutdown in the desired time due to: %s", e.getMessage())); + } + } + + private final class SetupMSListTask extends ManagedContextRunnable { + private Long hostId; + private Long dcId; + private List orderedHostIdList; + private List avoidMsList; + private String lbAlgorithm; + private Long lbCheckInterval; + + public SetupMSListTask(Long hostId, Long dcId, List orderedHostIdList, List avoidMsList, + String lbAlgorithm, Long lbCheckInterval) { + this.hostId = hostId; + this.dcId = dcId; + this.orderedHostIdList = orderedHostIdList; + this.avoidMsList = avoidMsList; + this.lbAlgorithm = lbAlgorithm; + this.lbCheckInterval = lbCheckInterval; + } + + @Override + protected void runInContext() { + final List msList = getManagementServerList(hostId, dcId, orderedHostIdList); + final SetupMSListCommand cmd = new SetupMSListCommand(msList, avoidMsList, lbAlgorithm, lbCheckInterval); + cmd.setWait(60); + final Answer answer = agentManager.easySend(hostId, cmd); + if (answer == null || !answer.getResult()) { + logger.warn(String.format("Failed to setup management servers list to the agent of ID: %d", hostId)); + } + } + } + + protected boolean migrateNonRoutingHostAgentsInZone(String fromMsUuid, long fromMsId, DataCenter dc, + long migrationStartTimeInMs, long timeoutDurationInMs, final List avoidMsList, String lbAlgorithm, + boolean lbAlgorithmChanged, List orderedHostIdList) { + List systemVmAgentsInDc = getAllAgentBasedNonRoutingHostsFromDB(dc.getId(), fromMsId); + if (CollectionUtils.isEmpty(systemVmAgentsInDc)) { + return true; + } + logger.debug(String.format("Migrating %d non-routing host agents from management server node %d (id: %s) of zone %s", + systemVmAgentsInDc.size(), fromMsId, fromMsUuid, dc)); + ExecutorService migrateAgentsExecutorService = Executors.newFixedThreadPool(5, new NamedThreadFactory("MigrateNonRoutingHostAgent-Worker")); + Long lbCheckInterval = getLBPreferredHostCheckInterval(null); + boolean stopMigration = false; + for (final Long hostId : systemVmAgentsInDc) { + long migrationElapsedTimeInMs = System.currentTimeMillis() - migrationStartTimeInMs; + if (migrationElapsedTimeInMs >= timeoutDurationInMs) { + logger.debug(String.format("Stop migrating remaining non-routing host agents from management server node %d (id: %s), timed out", fromMsId, fromMsUuid)); + stopMigration = true; + break; + } + + migrateAgentsExecutorService.submit(new MigrateAgentConnectionTask(fromMsId, hostId, dc.getId(), orderedHostIdList, avoidMsList, lbCheckInterval, lbAlgorithm, lbAlgorithmChanged)); + } + + if (stopMigration) { + migrateAgentsExecutorService.shutdownNow(); + return false; + } + + migrateAgentsExecutorService.shutdown(); + long pendingTimeoutDurationInMs = timeoutDurationInMs - (System.currentTimeMillis() - migrationStartTimeInMs); + try { + if (pendingTimeoutDurationInMs <= 0 || !migrateAgentsExecutorService.awaitTermination(pendingTimeoutDurationInMs, TimeUnit.MILLISECONDS)) { + migrateAgentsExecutorService.shutdownNow(); + } + } catch (InterruptedException e) { + migrateAgentsExecutorService.shutdownNow(); + logger.debug(String.format("Force shutdown migrate non-routing agents service as it did not shutdown in the desired time due to: %s", e.getMessage())); + } + + return true; + } + + protected boolean migrateRoutingHostAgentsInCluster(long clusterId, String fromMsUuid, long fromMsId, DataCenter dc, + long migrationStartTimeInMs, long timeoutDurationInMs, final List avoidMsList, String lbAlgorithm, + boolean lbAlgorithmChanged, List orderedHostIdList) { + + List agentBasedHostsOfMsInDcAndCluster = getAllAgentBasedRoutingHostsFromDB(dc.getId(), clusterId, fromMsId); + if (CollectionUtils.isEmpty(agentBasedHostsOfMsInDcAndCluster)) { + return true; + } + logger.debug(String.format("Migrating %d indirect routing host agents from management server node %d (id: %s) of zone %s, " + + "cluster ID: %d", agentBasedHostsOfMsInDcAndCluster.size(), fromMsId, fromMsUuid, dc, clusterId)); + ExecutorService migrateAgentsExecutorService = Executors.newFixedThreadPool(10, new NamedThreadFactory("MigrateRoutingHostAgent-Worker")); + Long lbCheckInterval = getLBPreferredHostCheckInterval(clusterId); + boolean stopMigration = false; + for (final Long hostId : agentBasedHostsOfMsInDcAndCluster) { + long migrationElapsedTimeInMs = System.currentTimeMillis() - migrationStartTimeInMs; + if (migrationElapsedTimeInMs >= timeoutDurationInMs) { + logger.debug(String.format("Stop migrating remaining indirect routing host agents from management server node %d (id: %s), timed out", fromMsId, fromMsUuid)); + stopMigration = true; + break; + } + + migrateAgentsExecutorService.submit(new MigrateAgentConnectionTask(fromMsId, hostId, dc.getId(), orderedHostIdList, avoidMsList, lbCheckInterval, lbAlgorithm, lbAlgorithmChanged)); + } + + if (stopMigration) { + migrateAgentsExecutorService.shutdownNow(); + return false; + } + + migrateAgentsExecutorService.shutdown(); + long pendingTimeoutDurationInMs = timeoutDurationInMs - (System.currentTimeMillis() - migrationStartTimeInMs); + try { + if (pendingTimeoutDurationInMs <= 0 || !migrateAgentsExecutorService.awaitTermination(pendingTimeoutDurationInMs, TimeUnit.MILLISECONDS)) { + migrateAgentsExecutorService.shutdownNow(); + } + } catch (InterruptedException e) { + migrateAgentsExecutorService.shutdownNow(); + logger.debug(String.format("Force shutdown migrate routing agents service as it did not shutdown in the desired time due to: %s", e.getMessage())); + } + + return true; } @Override @@ -322,7 +468,7 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement } logger.debug(String.format("Migrating indirect agents from management server node %d (id: %s) to other nodes", fromMsId, fromMsUuid)); - long migrationStartTime = System.currentTimeMillis(); + long migrationStartTimeInMs = System.currentTimeMillis(); if (!haveAgentBasedHosts(fromMsId)) { logger.info(String.format("No indirect agents available on management server node %d (id: %s), to migrate", fromMsId, fromMsUuid)); return true; @@ -342,37 +488,75 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement List dataCenterList = dcDao.listAll(); for (DataCenterVO dc : dataCenterList) { - Long dcId = dc.getId(); - List orderedHostIdList = getOrderedHostIdList(dcId); - List agentBasedHostsOfMsInDc = getAllAgentBasedHostsInDc(fromMsId, dcId); - if (CollectionUtils.isEmpty(agentBasedHostsOfMsInDc)) { - continue; - } - logger.debug(String.format("Migrating %d indirect agents from management server node %d (id: %s) of zone %s", agentBasedHostsOfMsInDc.size(), fromMsId, fromMsUuid, dc)); - for (final Host host : agentBasedHostsOfMsInDc) { - long migrationElapsedTimeInMs = System.currentTimeMillis() - migrationStartTime; - if (migrationElapsedTimeInMs >= timeoutDurationInMs) { - logger.debug(String.format("Stop migrating remaining indirect agents from management server node %d (id: %s), timed out", fromMsId, fromMsUuid)); - return false; - } - - List msList = null; - Long lbCheckInterval = 0L; - if (lbAlgorithmChanged) { - // send new MS list when there is change in lb algorithm - msList = getManagementServerList(host.getId(), dcId, orderedHostIdList, lbAlgorithm); - lbCheckInterval = getLBPreferredHostCheckInterval(host.getClusterId()); - } - - final MigrateAgentConnectionCommand cmd = new MigrateAgentConnectionCommand(msList, avoidMsList, lbAlgorithm, lbCheckInterval); - agentManager.easySend(host.getId(), cmd); //answer not received as the agent disconnects and reconnects to other ms - updateLastManagementServer(host.getId(), fromMsId); + if (!migrateAgentsInZone(dc, fromMsUuid, fromMsId, avoidMsList, lbAlgorithm, lbAlgorithmChanged, + migrationStartTimeInMs, timeoutDurationInMs)) { + return false; } } return true; } + private boolean migrateAgentsInZone(DataCenterVO dc, String fromMsUuid, long fromMsId, List avoidMsList, + String lbAlgorithm, boolean lbAlgorithmChanged, long migrationStartTimeInMs, long timeoutDurationInMs) { + List orderedHostIdList = getOrderedHostIdList(dc.getId()); + if (!migrateNonRoutingHostAgentsInZone(fromMsUuid, fromMsId, dc, migrationStartTimeInMs, + timeoutDurationInMs, avoidMsList, lbAlgorithm, lbAlgorithmChanged, orderedHostIdList)) { + return false; + } + List clusterIds = clusterDao.listAllClusterIds(dc.getId()); + for (Long clusterId : clusterIds) { + if (!migrateRoutingHostAgentsInCluster(clusterId, fromMsUuid, fromMsId, dc, migrationStartTimeInMs, + timeoutDurationInMs, avoidMsList, lbAlgorithm, lbAlgorithmChanged, orderedHostIdList)) { + return false; + } + } + return true; + } + + private final class MigrateAgentConnectionTask extends ManagedContextRunnable { + private long fromMsId; + Long hostId; + Long dcId; + List orderedHostIdList; + List avoidMsList; + Long lbCheckInterval; + String lbAlgorithm; + boolean lbAlgorithmChanged; + + public MigrateAgentConnectionTask(long fromMsId, Long hostId, Long dcId, List orderedHostIdList, + List avoidMsList, Long lbCheckInterval, String lbAlgorithm, boolean lbAlgorithmChanged) { + this.fromMsId = fromMsId; + this.hostId = hostId; + this.orderedHostIdList = orderedHostIdList; + this.avoidMsList = avoidMsList; + this.lbCheckInterval = lbCheckInterval; + this.lbAlgorithm = lbAlgorithm; + this.lbAlgorithmChanged = lbAlgorithmChanged; + } + + @Override + protected void runInContext() { + try { + List msList = null; + if (lbAlgorithmChanged) { + // send new MS list when there is change in lb algorithm + msList = getManagementServerList(hostId, dcId, orderedHostIdList, lbAlgorithm); + } + + final MigrateAgentConnectionCommand cmd = new MigrateAgentConnectionCommand(msList, avoidMsList, lbAlgorithm, lbCheckInterval); + cmd.setWait(60); + final Answer answer = agentManager.easySend(hostId, cmd); //may not receive answer when the agent disconnects immediately and try reconnecting to other ms host + if (answer != null && !answer.getResult()) { + logger.warn(String.format("Error while initiating migration of agent connection for host agent ID: %d - %s", hostId, answer.getDetails())); + } + updateLastManagementServer(hostId, fromMsId); + } catch (final Exception e) { + logger.error(String.format("Error migrating agent connection for host %d", hostId), e); + } + } + } + private void updateLastManagementServer(long hostId, long msId) { HostVO hostVO = hostDao.findById(hostId); if (hostVO != null) { diff --git a/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java index 0c0097393ca..1b9923ad3ea 100644 --- a/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java @@ -106,7 +106,7 @@ public class IndirectAgentLBServiceImplTest { List hostIds = hosts.stream().map(HostVO::getId).collect(Collectors.toList()); doReturn(hostIds).when(hostDao).findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(Mockito.anyLong(), - Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); + Mockito.eq(null), Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); } @Before @@ -203,14 +203,14 @@ public class IndirectAgentLBServiceImplTest { @Test public void testGetOrderedRunningHostIdsEmptyList() { doReturn(Collections.emptyList()).when(hostDao).findHostIdsByZoneClusterResourceStateTypeAndHypervisorType( - Mockito.eq(DC_1_ID), Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); + Mockito.eq(DC_1_ID), Mockito.eq(null), Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_1_ID).isEmpty()); } @Test public void testGetOrderedRunningHostIdsOrderList() { doReturn(Arrays.asList(host4.getId(), host2.getId(), host1.getId(), host3.getId())).when(hostDao) - .findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(Mockito.eq(DC_1_ID), Mockito.eq(null), + .findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(Mockito.eq(DC_1_ID), Mockito.eq(null), Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); Assert.assertEquals(Arrays.asList(host1.getId(), host2.getId(), host3.getId(), host4.getId()), agentMSLB.getOrderedHostIdList(DC_1_ID)); diff --git a/utils/src/main/java/com/cloud/utils/nio/HandlerFactory.java b/utils/src/main/java/com/cloud/utils/nio/HandlerFactory.java index 6f0f1945e01..9493f24b92b 100644 --- a/utils/src/main/java/com/cloud/utils/nio/HandlerFactory.java +++ b/utils/src/main/java/com/cloud/utils/nio/HandlerFactory.java @@ -25,7 +25,7 @@ import java.net.SocketAddress; * WorkerFactory creates and selects workers. */ public interface HandlerFactory { - public Task create(Task.Type type, Link link, byte[] data); + Task create(Task.Type type, Link link, byte[] data); default int getMaxConcurrentNewConnectionsCount() { return 0; } diff --git a/utils/src/main/java/com/cloud/utils/nio/Link.java b/utils/src/main/java/com/cloud/utils/nio/Link.java index 5404cd15343..4e68554eb49 100644 --- a/utils/src/main/java/com/cloud/utils/nio/Link.java +++ b/utils/src/main/java/com/cloud/utils/nio/Link.java @@ -617,8 +617,8 @@ public class Link { final long timeTaken = System.currentTimeMillis() - startTimeMills; if (timeTaken > timeoutMillis) { - LOGGER.warn("SSL Handshake has taken more than {}ms to connect to: {}" + - " while status: {}. Please investigate this connection.", socketChannel.getRemoteAddress(), + LOGGER.warn("SSL Handshake has taken more than {} ms to connect to: {}" + + " while status: {}. Please investigate this connection.", timeoutMillis, socketChannel.getRemoteAddress(), handshakeStatus); return false; } diff --git a/utils/src/main/java/com/cloud/utils/nio/NioClient.java b/utils/src/main/java/com/cloud/utils/nio/NioClient.java index 46d67feaaf3..d274973a658 100644 --- a/utils/src/main/java/com/cloud/utils/nio/NioClient.java +++ b/utils/src/main/java/com/cloud/utils/nio/NioClient.java @@ -115,4 +115,8 @@ public class NioClient extends NioConnection { } logger.info("NioClient connection closed"); } + + public String getHost() { + return host; + } } diff --git a/utils/src/main/java/com/cloud/utils/nio/NioConnection.java b/utils/src/main/java/com/cloud/utils/nio/NioConnection.java index 98fa69716cd..ed6b5748289 100644 --- a/utils/src/main/java/com/cloud/utils/nio/NioConnection.java +++ b/utils/src/main/java/com/cloud/utils/nio/NioConnection.java @@ -83,26 +83,19 @@ public abstract class NioConnection implements Callable { protected Set socketChannels = new HashSet<>(); protected Integer sslHandshakeTimeout = null; private final int factoryMaxNewConnectionsCount; + protected boolean blockNewConnections; public NioConnection(final String name, final int port, final int workers, final HandlerFactory factory) { _name = name; _isRunning = false; + blockNewConnections = false; _selector = null; _port = port; _workers = workers; _factory = factory; this.factoryMaxNewConnectionsCount = factory.getMaxConcurrentNewConnectionsCount(); - _executor = new ThreadPoolExecutor(workers, 5 * workers, 1, TimeUnit.DAYS, - new LinkedBlockingQueue<>(5 * workers), new NamedThreadFactory(name + "-Handler"), - new ThreadPoolExecutor.AbortPolicy()); - String sslHandshakeHandlerName = name + "-SSLHandshakeHandler"; - if (factoryMaxNewConnectionsCount > 0) { - _sslHandshakeExecutor = new ThreadPoolExecutor(0, this.factoryMaxNewConnectionsCount, 30, - TimeUnit.MINUTES, new SynchronousQueue<>(), new NamedThreadFactory(sslHandshakeHandlerName), - new ThreadPoolExecutor.AbortPolicy()); - } else { - _sslHandshakeExecutor = Executors.newCachedThreadPool(new NamedThreadFactory(sslHandshakeHandlerName)); - } + initWorkersExecutor(); + initSSLHandshakeExecutor(); } public void setCAService(final CAService caService) { @@ -127,10 +120,14 @@ public abstract class NioConnection implements Callable { _isStartup = true; if (_executor.isShutdown()) { - _executor = new ThreadPoolExecutor(_workers, 5 * _workers, 1, TimeUnit.DAYS, new LinkedBlockingQueue<>(), new NamedThreadFactory(_name + "-Handler")); + initWorkersExecutor(); + } + if (_sslHandshakeExecutor.isShutdown()) { + initSSLHandshakeExecutor(); } _threadExecutor = Executors.newSingleThreadExecutor(new NamedThreadFactory(this._name + "-NioConnectionHandler")); _isRunning = true; + blockNewConnections = false; _futureTask = _threadExecutor.submit(this); } @@ -138,12 +135,30 @@ public abstract class NioConnection implements Callable { _executor.shutdown(); _sslHandshakeExecutor.shutdown(); _isRunning = false; + blockNewConnections = true; if (_threadExecutor != null) { _futureTask.cancel(false); _threadExecutor.shutdown(); } } + private void initWorkersExecutor() { + _executor = new ThreadPoolExecutor(_workers, 5 * _workers, 1, TimeUnit.DAYS, + new LinkedBlockingQueue<>(5 * _workers), new NamedThreadFactory(_name + "-Handler"), + new ThreadPoolExecutor.AbortPolicy()); + } + + private void initSSLHandshakeExecutor() { + String sslHandshakeHandlerName = _name + "-SSLHandshakeHandler"; + if (factoryMaxNewConnectionsCount > 0) { + _sslHandshakeExecutor = new ThreadPoolExecutor(0, this.factoryMaxNewConnectionsCount, 30, + TimeUnit.MINUTES, new SynchronousQueue<>(), new NamedThreadFactory(sslHandshakeHandlerName), + new ThreadPoolExecutor.AbortPolicy()); + } else { + _sslHandshakeExecutor = Executors.newCachedThreadPool(new NamedThreadFactory(sslHandshakeHandlerName)); + } + } + public boolean isRunning() { return !_futureTask.isDone(); } @@ -210,6 +225,16 @@ public abstract class NioConnection implements Callable { abstract void unregisterLink(InetSocketAddress saddr); + protected boolean rejectConnectionIfBlocked(final SocketChannel socketChannel) throws IOException { + if (!blockNewConnections) { + return false; + } + logger.warn("Rejecting new connection as the server is blocked from accepting new connections"); + socketChannel.close(); + _selector.wakeup(); + return true; + } + protected boolean rejectConnectionIfBusy(final SocketChannel socketChannel) throws IOException { if (factoryMaxNewConnectionsCount <= 0 || _factory.getNewConnectionsCount() < factoryMaxNewConnectionsCount) { return false; @@ -226,7 +251,7 @@ public abstract class NioConnection implements Callable { protected void accept(final SelectionKey key) throws IOException { final ServerSocketChannel serverSocketChannel = (ServerSocketChannel)key.channel(); final SocketChannel socketChannel = serverSocketChannel.accept(); - if (rejectConnectionIfBusy(socketChannel)) { + if (rejectConnectionIfBlocked(socketChannel) || rejectConnectionIfBusy(socketChannel)) { return; } socketChannel.configureBlocking(false); @@ -520,6 +545,14 @@ public abstract class NioConnection implements Callable { } } + public void block() { + blockNewConnections = true; + } + + public void unblock() { + blockNewConnections = false; + } + public class ChangeRequest { public static final int REGISTER = 1; public static final int CHANGEOPS = 2; From 653b9738400d6b39fc4009ceb77b90d2609073a3 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Wed, 19 Mar 2025 08:33:44 -0400 Subject: [PATCH 09/42] Update ubuntu image link for template download (#10559) --- tools/marvin/marvin/config/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 3485eeb8b18..bde48c87616 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1069,7 +1069,7 @@ test_data = { "format": "raw", "hypervisor": "kvm", "ostype": "Other Linux (64-bit)", - "url": "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img", + "url": "https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64.img", "requireshvm": "True", "ispublic": "True", "isextractable": "False" From e6b2df2e0ff490850add8b4a5273eee2d1b6b11e Mon Sep 17 00:00:00 2001 From: Daniel Gruno Date: Thu, 20 Mar 2025 15:17:49 +0100 Subject: [PATCH 10/42] [Infra] reset collaborators --- .asf.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 43f0351bc7c..ee188624155 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,15 +49,7 @@ github: squash: true rebase: false - collaborators: - - acs-robot - - gpordeus - - hsato03 - - bernardodemarco - - abh1sar - - FelipeM525 - - lucas-a-martins - - nicoschmdt + collaborators: [] protected_branches: ~ From a521985662ab2a1558f32a2be680f266d63e4b0f Mon Sep 17 00:00:00 2001 From: Daniel Gruno Date: Thu, 20 Mar 2025 15:18:27 +0100 Subject: [PATCH 11/42] [Infra] reset collaborators list --- .asf.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index ee188624155..17429a27519 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,7 +49,15 @@ github: squash: true rebase: false - collaborators: [] + collaborators: + - acs-robot + - gpordeus + - hsato03 + - bernardodemarco + - abh1sar + - FelipeM525 + - lucas-a-martins + - nicoschmdt protected_branches: ~ From 8df1161f14e41e9b0a0b71422de510b3510500e1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 21 Mar 2025 21:27:24 +0530 Subject: [PATCH 12/42] framework-config: improve configkey caching (#10513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using a simple hyphen as a delimiter for config cache key can lead to ambiguity if the “name” field itself contains hyphens. To address this, a Ternary object of configkey name, scope and scope ID is used as the config cache keys. Signed-off-by: Abhishek Kumar --- .../config/impl/ConfigDepotImpl.java | 19 +++++++------------ .../config/impl/ConfigDepotImplTest.java | 6 ++++++ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java index 911a4ad3707..8b8b6368527 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java @@ -85,7 +85,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { List _scopedStorages; Set _configured = Collections.synchronizedSet(new HashSet()); Set newConfigs = Collections.synchronizedSet(new HashSet<>()); - LazyCache configCache; + LazyCache, String> configCache; private HashMap>> _allKeys = new HashMap>>(1007); @@ -273,15 +273,10 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { return _configDao; } - protected String getConfigStringValueInternal(String cacheKey) { - String[] parts = cacheKey.split("-"); - String key = parts[0]; - ConfigKey.Scope scope = ConfigKey.Scope.Global; - Long scopeId = null; - try { - scope = ConfigKey.Scope.valueOf(parts[1]); - scopeId = Long.valueOf(parts[2]); - } catch (IllegalArgumentException ignored) {} + protected String getConfigStringValueInternal(Ternary cacheKey) { + String key = cacheKey.first(); + ConfigKey.Scope scope = cacheKey.second(); + Long scopeId = cacheKey.third(); if (!ConfigKey.Scope.Global.equals(scope) && scopeId != null) { ScopedConfigStorage scopedConfigStorage = null; for (ScopedConfigStorage storage : _scopedStorages) { @@ -301,8 +296,8 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { return null; } - private String getConfigCacheKey(String key, ConfigKey.Scope scope, Long scopeId) { - return String.format("%s-%s-%d", key, scope, (scopeId == null ? 0 : scopeId)); + protected Ternary getConfigCacheKey(String key, ConfigKey.Scope scope, Long scopeId) { + return new Ternary<>(key, scope, scopeId); } @Override diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java index 8a7da795345..ca2f54f1442 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java @@ -81,6 +81,12 @@ public class ConfigDepotImplTest { runTestGetConfigStringValue("test", "value"); } + @Test + public void testGetConfigStringValue_nameWithCharacters() { + runTestGetConfigStringValue("test.1-1", "value"); + runTestGetConfigStringValue("test_1#2", "value"); + } + private void runTestGetConfigStringValueExpiry(long wait, int configDBRetrieval) { String key = "test1"; String value = "expiry"; From 16b7b71e36e3287a72ffa56dbdbcb1ca16c1e9df Mon Sep 17 00:00:00 2001 From: Fabricio Duarte Date: Mon, 24 Mar 2025 08:07:06 -0300 Subject: [PATCH 13/42] Fix secondary storage selectors feature (#10546) --- .../GenericPresetVariable.java | 6 ++- .../presetvariables/Resource.java | 6 ++- .../heuristics/presetvariables/Domain.java | 2 +- .../GenericHeuristicPresetVariable.java | 10 ++-- .../presetvariables/AccountTest.java | 46 +++++++++++++++++++ .../presetvariables/DomainTest.java | 41 +++++++++++++++++ .../GenericHeuristicPresetVariableTest.java | 40 ++++++++++++++++ .../presetvariables/SecondaryStorageTest.java | 45 ++++++++++++++++++ .../presetvariables/SnapshotTest.java | 44 ++++++++++++++++++ .../presetvariables/TemplateTest.java | 46 +++++++++++++++++++ .../presetvariables/VolumeTest.java | 44 ++++++++++++++++++ 11 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/AccountTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/DomainTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariableTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorageTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SnapshotTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/TemplateTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/VolumeTest.java diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java index f59f23abdc1..7073d2760d7 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/GenericPresetVariable.java @@ -49,8 +49,12 @@ public class GenericPresetVariable { fieldNamesToIncludeInToString.add("name"); } + /*** + * Converts the preset variable into a valid JSON object that will be injected into the JS interpreter. + * This method should not be overridden or changed. + */ @Override - public String toString() { + public final String toString() { return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, fieldNamesToIncludeInToString.toArray(new String[0])); } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Resource.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Resource.java index b1dfbacbfcd..4f8a971b869 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Resource.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Resource.java @@ -40,8 +40,12 @@ public class Resource { this.domainId = domainId; } + /*** + * Converts the preset variable into a valid JSON object that will be injected into the JS interpreter. + * This method should not be overridden or changed. + */ @Override - public String toString() { + public final String toString() { return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE); } diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java index 6565c06bfbb..704cbf4373e 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java @@ -16,7 +16,7 @@ // under the License. package org.apache.cloudstack.storage.heuristics.presetvariables; -public class Domain extends GenericHeuristicPresetVariable{ +public class Domain extends GenericHeuristicPresetVariable { private String id; public String getId() { diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java index 28d4327954e..b85b7763eee 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java @@ -36,10 +36,12 @@ public class GenericHeuristicPresetVariable { fieldNamesToIncludeInToString.add("name"); } + /*** + * Converts the preset variable into a valid JSON object that will be injected into the JS interpreter. + * This method should not be overridden or changed. + */ @Override - public String toString() { - return String.format("GenericHeuristicPresetVariable %s", - ReflectionToStringBuilderUtils.reflectOnlySelectedFields( - this, fieldNamesToIncludeInToString.toArray(new String[0]))); + public final String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, fieldNamesToIncludeInToString.toArray(new String[0])); } } diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/AccountTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/AccountTest.java new file mode 100644 index 00000000000..f7610438b78 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/AccountTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class AccountTest { + + @Test + public void toStringTestReturnsValidJson() { + Account variable = new Account(); + variable.setName("test name"); + variable.setId("test id"); + + Domain domainVariable = new Domain(); + domainVariable.setId("domain id"); + domainVariable.setName("domain name"); + variable.setDomain(domainVariable); + + String expected = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(variable, "name", "id", "domain"); + String result = variable.toString(); + + Assert.assertEquals(expected, result); + } + +} diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/DomainTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/DomainTest.java new file mode 100644 index 00000000000..a1ec6854ecc --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/DomainTest.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class DomainTest { + + @Test + public void toStringTestReturnsValidJson() { + Domain variable = new Domain(); + variable.setName("test name"); + variable.setId("test id"); + + String expected = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(variable, "name", "id"); + String result = variable.toString(); + + Assert.assertEquals(expected, result); + } + +} diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariableTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariableTest.java new file mode 100644 index 00000000000..cd295e92caf --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariableTest.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class GenericHeuristicPresetVariableTest { + + @Test + public void toStringTestReturnsValidJson() { + GenericHeuristicPresetVariable variable = new GenericHeuristicPresetVariable(); + variable.setName("test name"); + + String expected = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(variable, "name"); + String result = variable.toString(); + + Assert.assertEquals(expected, result); + } + +} diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorageTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorageTest.java new file mode 100644 index 00000000000..a09386789cf --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorageTest.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class SecondaryStorageTest { + + @Test + public void toStringTestReturnsValidJson() { + SecondaryStorage variable = new SecondaryStorage(); + variable.setName("test name"); + variable.setId("test id"); + variable.setProtocol("test protocol"); + variable.setUsedDiskSize(1L); + variable.setTotalDiskSize(2L); + + String expected = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(variable, "name", "id", + "protocol", "usedDiskSize", "totalDiskSize"); + String result = variable.toString(); + + Assert.assertEquals(expected, result); + } + +} diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SnapshotTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SnapshotTest.java new file mode 100644 index 00000000000..b8476cd8e46 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/SnapshotTest.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class SnapshotTest { + + @Test + public void toStringTestReturnsValidJson() { + Snapshot variable = new Snapshot(); + variable.setName("test name"); + variable.setSize(1L); + variable.setHypervisorType(Hypervisor.HypervisorType.KVM); + + String expected = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(variable, "name", "size", + "hypervisorType"); + String result = variable.toString(); + + Assert.assertEquals(expected, result); + } + +} diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/TemplateTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/TemplateTest.java new file mode 100644 index 00000000000..2c1582befb2 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/TemplateTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class TemplateTest { + + @Test + public void toStringTestReturnsValidJson() { + Template variable = new Template(); + variable.setName("test name"); + variable.setTemplateType(Storage.TemplateType.USER); + variable.setHypervisorType(Hypervisor.HypervisorType.KVM); + variable.setFormat(Storage.ImageFormat.QCOW2); + + String expected = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(variable, "name", "templateType", + "hypervisorType", "format"); + String result = variable.toString(); + + Assert.assertEquals(expected, result); + } + +} diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/VolumeTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/VolumeTest.java new file mode 100644 index 00000000000..e74ddc93ec5 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/presetvariables/VolumeTest.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.storage.Storage; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class VolumeTest { + + @Test + public void toStringTestReturnsValidJson() { + Volume variable = new Volume(); + variable.setName("test name"); + variable.setFormat(Storage.ImageFormat.QCOW2); + variable.setSize(1L); + + String expected = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(variable, "name", "format", + "size"); + String result = variable.toString(); + + Assert.assertEquals(expected, result); + } + +} From c1ff799df23b7ad20dfef388efcc05a02f98bf52 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 24 Mar 2025 17:02:25 +0530 Subject: [PATCH 14/42] ui: fix considerlasthost for start vm (#10602) Signed-off-by: Abhishek Kumar --- ui/src/config/section/compute.js | 2 +- ui/src/views/compute/StartVirtualMachine.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index 0d5c8de6628..2887044a535 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -121,7 +121,7 @@ export default { dataView: true, groupAction: true, popup: true, - groupMap: (selection, values) => { return selection.map(x => { return { id: x, considerlasthost: values.considerlasthost } }) }, + groupMap: (selection, values) => { return selection.map(x => { return { id: x, considerlasthost: values.considerlasthost === true } }) }, args: (record, store) => { if (['Admin'].includes(store.userInfo.roletype)) { return ['considerlasthost'] diff --git a/ui/src/views/compute/StartVirtualMachine.vue b/ui/src/views/compute/StartVirtualMachine.vue index 7438ebc1beb..339d6c75719 100644 --- a/ui/src/views/compute/StartVirtualMachine.vue +++ b/ui/src/views/compute/StartVirtualMachine.vue @@ -237,7 +237,7 @@ export default { id: this.resource.id } for (const key in values) { - if (values[key]) { + if (values[key] || values[key] === false) { params[key] = values[key] } } From c9c02d030e5e6b83b61a87d07e604d28ed619208 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 24 Mar 2025 12:40:00 +0100 Subject: [PATCH 15/42] UI: fix list of vpc network offerings (#10595) --- ui/src/views/network/VpcTiersTab.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/views/network/VpcTiersTab.vue b/ui/src/views/network/VpcTiersTab.vue index ad06e3bdf0b..73410ca1d05 100644 --- a/ui/src/views/network/VpcTiersTab.vue +++ b/ui/src/views/network/VpcTiersTab.vue @@ -553,9 +553,9 @@ export default { if (this.publicLBExists && (idx === -1 || this.lbProviderMap.publicLb.vpc.indexOf(offering.service.map(svc => { return svc.provider[0].name })[idx]) === -1)) { filteredOfferings.push(offering) } else if (!this.publicLBExists && vpcLbServiceIndex > -1) { - const vpcLbServiceProvider = vpcLbServiceIndex === -1 ? undefined : this.resource.service[vpcLbServiceIndex].provider[0].name + const vpcLbServiceProviders = vpcLbServiceIndex === -1 ? undefined : this.resource.service[vpcLbServiceIndex].provider.map(provider => provider.name) const offeringLbServiceProvider = idx === -1 ? undefined : offering.service[idx].provider[0].name - if (vpcLbServiceProvider && (!offeringLbServiceProvider || (offeringLbServiceProvider && vpcLbServiceProvider === offeringLbServiceProvider))) { + if (vpcLbServiceProviders && (!offeringLbServiceProvider || (offeringLbServiceProvider && vpcLbServiceProviders.includes(offeringLbServiceProvider)))) { filteredOfferings.push(offering) } } else { From 9a3fa89e3239ac937b1861edcba430a9ebfe944c Mon Sep 17 00:00:00 2001 From: dahn Date: Tue, 25 Mar 2025 08:46:26 +0100 Subject: [PATCH 16/42] remove trailing white space --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index 17429a27519..43f0351bc7c 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -49,7 +49,7 @@ github: squash: true rebase: false - collaborators: + collaborators: - acs-robot - gpordeus - hsato03 From 8a338ea12c394eec9082d86263f3490501c26bdc Mon Sep 17 00:00:00 2001 From: Imvedansh <113465074+Imvedansh@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:55:04 +0530 Subject: [PATCH 17/42] In Install.md->CloudStack UI Commands(npm) (#10593) --- INSTALL.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index e133e7d7b91..3685037bfe2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -37,6 +37,7 @@ Setup up NodeJS (LTS): Start the MySQL service: $ service mysqld start + $ mysql_secure_installation ### Using jenv and/or pyenv for Version Management @@ -86,13 +87,33 @@ Start the management server: If this works, you've successfully setup a single server Apache CloudStack installation. -Open the following URL on your browser to access the Management Server UI: - - http://localhost:8080/client/ +To access the Management Server UI, follow the following procedure: The default credentials are; user: admin, password: password and the domain field should be left blank which is defaulted to the ROOT domain. +## To bring up CloudStack UI + +Move to UI Directory + + $ cd /path/to/cloudstack/ui + +To install dependencies. + + $ npm install + +To build the project. + + $ npm build + +For Development Mode. + + $ npm start + +Make sure to set CS_URL=http://localhost:8080/client on .env.local file on ui. + +You should be able to run the management server on http://localhost:5050 + ## Building with non-redistributable plugins CloudStack supports several plugins that depend on libraries with distribution restrictions. From 7f3f1042959a68594353c66c55f30c0fe9bf613b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bernardo=20De=20Marco=20Gon=C3=A7alves?= Date: Tue, 25 Mar 2025 07:27:13 -0300 Subject: [PATCH 18/42] add UI support for filtering ISOs by account (#10488) --- ui/src/components/view/InfoCard.vue | 3 +++ ui/src/config/section/account.js | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/ui/src/components/view/InfoCard.vue b/ui/src/components/view/InfoCard.vue index d8b37e3c6e4..6ecf4885ce5 100644 --- a/ui/src/components/view/InfoCard.vue +++ b/ui/src/components/view/InfoCard.vue @@ -1211,6 +1211,9 @@ export default { if (item.name === 'template') { query.templatefilter = 'self' query.filter = 'self' + } else if (item.name === 'iso') { + query.isofilter = 'self' + query.filter = 'self' } if (item.param === 'account') { diff --git a/ui/src/config/section/account.js b/ui/src/config/section/account.js index 21996167705..b5e962752f8 100644 --- a/ui/src/config/section/account.js +++ b/ui/src/config/section/account.js @@ -61,6 +61,10 @@ export default { name: 'template', title: 'label.templates', param: 'account' + }, { + name: 'iso', + title: 'label.isos', + param: 'account' }], filters: () => { const filters = ['enabled', 'disabled', 'locked'] From fc1f260d5291846b9da40c197a795d41ca29d081 Mon Sep 17 00:00:00 2001 From: Imvedansh <113465074+Imvedansh@users.noreply.github.com> Date: Tue, 25 Mar 2025 17:20:25 +0530 Subject: [PATCH 19/42] Host status auto refresh (#10606) Co-authored-by: Rene Peinthor --- ui/src/config/section/infra/hosts.js | 18 ++++++++++---- ui/src/views/infra/HostEnableDisable.vue | 30 +++++++++++------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/ui/src/config/section/infra/hosts.js b/ui/src/config/section/infra/hosts.js index 79ec40287fb..b5880985a9d 100644 --- a/ui/src/config/section/infra/hosts.js +++ b/ui/src/config/section/infra/hosts.js @@ -111,9 +111,14 @@ export default { label: 'label.disable.host', message: 'message.confirm.disable.host', dataView: true, - show: (record) => { return record.resourcestate === 'Enabled' }, + show: (record) => record.resourcestate === 'Enabled', popup: true, - component: shallowRef(defineAsyncComponent(() => import('@/views/infra/HostEnableDisable'))) + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/HostEnableDisable'))), + events: { + 'refresh-data': () => { + store.dispatch('refreshCurrentPage') + } + } }, { api: 'updateHost', @@ -121,9 +126,14 @@ export default { label: 'label.enable.host', message: 'message.confirm.enable.host', dataView: true, - show: (record) => { return record.resourcestate === 'Disabled' }, + show: (record) => record.resourcestate === 'Disabled', popup: true, - component: shallowRef(defineAsyncComponent(() => import('@/views/infra/HostEnableDisable'))) + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/HostEnableDisable'))), + events: { + 'refresh-data': () => { + store.dispatch('refreshCurrentPage') + } + } }, { api: 'prepareHostForMaintenance', diff --git a/ui/src/views/infra/HostEnableDisable.vue b/ui/src/views/infra/HostEnableDisable.vue index bc71aa27080..59ceb92d18f 100644 --- a/ui/src/views/infra/HostEnableDisable.vue +++ b/ui/src/views/infra/HostEnableDisable.vue @@ -18,7 +18,7 @@