From 71e53ab01d612de99f759e2b34611bbdf731d766 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 28 Jan 2020 06:24:32 +0100 Subject: [PATCH] server: Capacity check should take vms in Migrating state into calculation (#3727) When we calculate a resource consumption of a host, we need to take the vms in following states into calculation: Running, Starting, Stopping, Migrating (to the host), and vms are Migrating from the host. Because, when stop a vm, the resource on host will be released when vm is stopped. When migrate a vm, the resource on destination host will be increased before migration starts, and resource on source host will be decreased after migraiton succeeds. In cloudstack, there is a task named CapacityChecked which run every 5 minutes (capacity.check.period =300000 ms by default). It recalculates capacity of all hosts. However, it takes only vms in Running and Starting into consideration. We have faced some issues in host maintenance due to it. Steps to reproduce the issue (1) migrate N vms from host A to host B, cpu/ram resource increases before the migration. (2) capacity check recalculate the capacity of hosts. used capacity of Host B will be reset to original value (not including the vms in Migrating). (3) migrate some more vms from other host to host B, the migrations are allowed by cloudstack (because used capacity is incorrect). If the actual used memory exceed the physical memory on the host, there might be some critical issues (for example, libvirt dies) --- .../src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java | 2 +- .../main/java/com/cloud/capacity/CapacityManagerImpl.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 1945969f543..e4f5dba65ed 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -416,7 +416,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem public List listUpByHostId(Long hostId) { SearchCriteria sc = HostUpSearch.create(); sc.setParameters("host", hostId); - sc.setParameters("states", new Object[] {State.Starting, State.Running}); + sc.setParameters("states", new Object[] {State.Starting, State.Running, State.Stopping, State.Migrating}); return listBy(sc); } diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index a7fee9603a0..b0121ca612d 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -590,6 +590,12 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, s_logger.debug("Found " + vms.size() + " VMs on host " + host.getId()); } + final List vosMigrating = _vmDao.listVmsMigratingFromHost(host.getId()); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Found " + vosMigrating.size() + " VMs are Migrating from host " + host.getId()); + } + vms.addAll(vosMigrating); + ClusterVO cluster = _clusterDao.findById(host.getClusterId()); ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio"); ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio");