From 8e727230ceca981196ab371729c752d0155428f6 Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Tue, 29 Mar 2011 11:18:37 -0700 Subject: [PATCH] Use SystemVmLoadScanner for console proxy VMs --- .../consoleproxy/ConsoleProxyManagerImpl.java | 281 ++++++++---------- .../src/com/cloud/dc/dao/DataCenterDao.java | 1 + .../com/cloud/dc/dao/DataCenterDaoImpl.java | 13 +- .../src/com/cloud/vm/SystemVmLoadScanner.java | 11 + 4 files changed, 142 insertions(+), 164 deletions(-) diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index e05f21ba4f6..4b81ad5bee1 100644 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -27,8 +27,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import javax.ejb.Local; import javax.naming.ConfigurationException; @@ -116,6 +114,9 @@ import com.cloud.utils.net.NetUtils; import com.cloud.vm.ConsoleProxyVO; import com.cloud.vm.NicProfile; import com.cloud.vm.ReservationContext; +import com.cloud.vm.SystemVmLoadScanHandler; +import com.cloud.vm.SystemVmLoadScanner; +import com.cloud.vm.SystemVmLoadScanner.AfterScanAction; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; @@ -141,13 +142,10 @@ import com.google.gson.GsonBuilder; // because sooner or later, it will be driven into Running state // @Local(value = { ConsoleProxyManager.class, ConsoleProxyService.class }) -public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProxyService, Manager, AgentHook, VirtualMachineGuru { +public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProxyService, Manager, AgentHook, VirtualMachineGuru, SystemVmLoadScanHandler { private static final Logger s_logger = Logger.getLogger(ConsoleProxyManagerImpl.class); - private static final int DEFAULT_CAPACITY_SCAN_INTERVAL = 30000; // 30 seconds - private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1000; // 1 second - - private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; // 3 seconds + private static final int DEFAULT_CAPACITY_SCAN_INTERVAL = 30000; // 30 seconds private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC = 180; // 3 minutes private static final int API_WAIT_TIMEOUT = 5000; // 5 seconds (in milliseconds) @@ -187,7 +185,9 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx @Inject private VirtualMachineManager _itMgr; +/* private final ScheduledExecutorService _capacityScanScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("CP-Scan")); +*/ private final ExecutorService _requestHandlerScheduler = Executors.newCachedThreadPool(new NamedThreadFactory("Request-handler")); private long _capacityScanInterval = DEFAULT_CAPACITY_SCAN_INTERVAL; @@ -204,7 +204,12 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx private int _proxySessionTimeoutValue = DEFAULT_PROXY_SESSION_TIMEOUT; private boolean _sslEnabled = false; - private final GlobalLock _capacityScanLock = GlobalLock.getInternLock(getCapacityScanLockName()); + // global load picture at zone basis + private SystemVmLoadScanner _loadScanner; + private Map _zoneHostInfoMap; // map + private Map _zoneProxyCountMap; // map + private Map _zoneVmCountMap; // map + private final GlobalLock _allocProxyLock = GlobalLock.getInternLock(getAllocProxyLockName()); @Override @@ -867,135 +872,6 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx return l.size() < launchLimit; } - private Runnable getCapacityScanTask() { - return new Runnable() { - - @Override - public void run() { - Transaction txn = Transaction.open(Transaction.CLOUD_DB); - try { - reallyRun(); - } catch (Throwable e) { - s_logger.warn("Unexpected exception " + e.getMessage(), e); - } finally { - StackMaid.current().exitCleanup(); - txn.close(); - } - } - - private void reallyRun() { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Begin console proxy capacity scan"); - } - - if(!reserveStandbyCapacity()) { - if(s_logger.isDebugEnabled()) { - s_logger.debug("Reserving standby capacity is disable, skip capacity scan"); - } - return; - } - - // config var for consoleproxy.restart check - String restart = _configDao.getValue("consoleproxy.restart"); - if (restart != null && restart.equalsIgnoreCase("false")) { - s_logger.debug("Capacity scan disabled purposefully, consoleproxy.restart = false. This happens when the primarystorage is in maintenance mode"); - return; - } - - - Map zoneHostInfoMap = getZoneHostInfo(); - if (isServiceReady(zoneHostInfoMap)) { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Service is ready, check to see if we need to allocate standby capacity"); - } - - if (!_capacityScanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Capacity scan lock is used by others, skip and wait for my turn"); - } - return; - } - - if (s_logger.isTraceEnabled()) { - s_logger.trace("*** Begining capacity scan... ***"); - } - - try { - // scan default data center first - long defaultId = 0; - - // proxy count info by data-centers (zone-id, zone-name, - // count) - List l = _consoleProxyDao.getDatacenterProxyLoadMatrix(); - - // running VM session count by data-centers (zone-id, - // zone-name, count) - List listVmCounts = _consoleProxyDao.getDatacenterSessionLoadMatrix(); - - // indexing load info by data-center id - Map mapVmCounts = new HashMap(); - if (listVmCounts != null) { - for (ConsoleProxyLoadInfo info : listVmCounts) { - mapVmCounts.put(info.getId(), info); - } - } - - for (ConsoleProxyLoadInfo info : l) { - if (info.getName().equals(_instance)) { - ConsoleProxyLoadInfo vmInfo = mapVmCounts.get(info.getId()); - - if (!checkCapacity(info, vmInfo != null ? vmInfo : new ConsoleProxyLoadInfo())) { - if (isZoneReady(zoneHostInfoMap, info.getId())) { - allocCapacity(info.getId()); - } else { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Zone " + info.getId() + " is not ready to alloc standy console proxy"); - } - } - } - - defaultId = info.getId(); - break; - } - } - - // scan rest of data-centers - for (ConsoleProxyLoadInfo info : l) { - if (info.getId() != defaultId) { - ConsoleProxyLoadInfo vmInfo = mapVmCounts.get(info.getId()); - - if (!checkCapacity(info, vmInfo != null ? vmInfo : new ConsoleProxyLoadInfo())) { - if (isZoneReady(zoneHostInfoMap, info.getId())) { - allocCapacity(info.getId()); - } else { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Zone " + info.getId() + " is not ready to alloc standy console proxy"); - } - } - } - } - } - - if (s_logger.isTraceEnabled()) { - s_logger.trace("*** Stop capacity scan ***"); - } - } finally { - _capacityScanLock.unlock(); - } - - } else { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Service is not ready for capacity preallocation, wait for next time"); - } - } - - if (s_logger.isTraceEnabled()) { - s_logger.trace("End of console proxy capacity scan"); - } - } - }; - } - private boolean checkCapacity(ConsoleProxyLoadInfo proxyCountInfo, ConsoleProxyLoadInfo vmCountInfo) { if (proxyCountInfo.getCount() * _capacityPerProxy - vmCountInfo.getCount() <= _standbyCapacity) { @@ -1050,19 +926,6 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx } } - public boolean isServiceReady(Map zoneHostInfoMap) { - for (ZoneHostInfo zoneHostInfo : zoneHostInfoMap.values()) { - if (isZoneHostReady(zoneHostInfo)) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Zone " + zoneHostInfo.getDcId() + " is ready to launch"); - } - return true; - } - } - - return false; - } - public boolean isZoneReady(Map zoneHostInfoMap, long dataCenterId) { ZoneHostInfo zoneHostInfo = zoneHostInfoMap.get(dataCenterId); if (zoneHostInfo != null && isZoneHostReady(zoneHostInfo)) { @@ -1137,14 +1000,8 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx if (s_logger.isInfoEnabled()) { s_logger.info("Stop console proxy manager"); } - _capacityScanScheduler.shutdownNow(); - - try { - _capacityScanScheduler.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - } - - _capacityScanLock.releaseRef(); + + this._loadScanner.stop(); _allocProxyLock.releaseRef(); return true; } @@ -1247,10 +1104,6 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx } } - private String getCapacityScanLockName() { - return "consoleproxy.capacity.scan"; - } - private String getAllocProxyLockName() { return "consoleproxy.alloc"; } @@ -1341,7 +1194,8 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx _serviceOffering.setUniqueName("Cloud.com-ConsoleProxy"); _serviceOffering = _offeringDao.persistSystemServiceOffering(_serviceOffering); - _capacityScanScheduler.scheduleAtFixedRate(getCapacityScanTask(), STARTUP_DELAY, _capacityScanInterval, TimeUnit.MILLISECONDS); + _loadScanner = new SystemVmLoadScanner(this); + _loadScanner.initScan(STARTUP_DELAY, _capacityScanInterval); if (s_logger.isInfoEnabled()) { s_logger.info("Console Proxy Manager is configured."); @@ -1560,4 +1414,105 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx @Override public void finalizeStop(VirtualMachineProfile profile, StopAnswer answer) { } + + @Override + public void onScanStart() { + // to reduce possible number of DB queries for capacity scan, we run following aggregated queries in preparation stage + _zoneHostInfoMap = getZoneHostInfo(); + + _zoneProxyCountMap = new HashMap(); + List listProxyCounts = _consoleProxyDao.getDatacenterProxyLoadMatrix(); + for (ConsoleProxyLoadInfo info : listProxyCounts) { + _zoneProxyCountMap.put(info.getId(), info); + } + + _zoneVmCountMap = new HashMap(); + List listVmCounts = _consoleProxyDao.getDatacenterSessionLoadMatrix(); + for (ConsoleProxyLoadInfo info : listVmCounts) { + _zoneVmCountMap.put(info.getId(), info); + } + } + + @Override + public boolean canScan() { + if(!reserveStandbyCapacity()) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Reserving standby capacity is disable, skip capacity scan"); + } + return false; + } + + // config var for consoleproxy.restart check + String restart = _configDao.getValue("consoleproxy.restart"); + if (restart != null && restart.equalsIgnoreCase("false")) { + s_logger.debug("Capacity scan disabled purposefully, consoleproxy.restart = false. This happens when the primarystorage is in maintenance mode"); + return false; + } + + return true; + } + + @Override + public Long[] getScannablePools() { + List zones = _dcDao.listEnabledZones(); + + Long[] dcIdList = new Long[zones.size()]; + int i = 0; + for(DataCenterVO dc : zones) { + dcIdList[i++] = dc.getId(); + } + + return dcIdList; + } + + public boolean isPoolReadyForScan(Long pool) { + // pool is at zone basis + long dataCenterId = pool.longValue(); + + if(!isZoneReady(_zoneHostInfoMap, dataCenterId)) { + if(s_logger.isDebugEnabled()) + s_logger.debug("Zone " + dataCenterId + " is not ready to launch console proxy yet"); + return false; + } + + if(s_logger.isDebugEnabled()) + s_logger.debug("Zone " + dataCenterId + " is ready to launch console proxy"); + return true; + } + + @Override + public AfterScanAction scanPool(Long pool) { + long dataCenterId = pool.longValue(); + + ConsoleProxyLoadInfo proxyInfo = this._zoneProxyCountMap.get(dataCenterId); + if(proxyInfo == null) + return AfterScanAction.nop; + + ConsoleProxyLoadInfo vmInfo = this._zoneVmCountMap.get(dataCenterId); + if(vmInfo == null) + vmInfo = new ConsoleProxyLoadInfo(); + + if (!checkCapacity(proxyInfo, vmInfo)) { + if(s_logger.isDebugEnabled()) + s_logger.debug("Expand console proxy standby capacity for zone " + proxyInfo.getName()); + + return AfterScanAction.expand; + } + + return AfterScanAction.nop; + } + + @Override + public void expandPool(Long pool) { + long dataCenterId = pool.longValue(); + allocCapacity(dataCenterId); + } + + @Override + public void shrinkPool(Long pool) { + } + + @Override + public void onScanEnd() { + } } diff --git a/server/src/com/cloud/dc/dao/DataCenterDao.java b/server/src/com/cloud/dc/dao/DataCenterDao.java index f13cbf97b1d..21fd5c7b86d 100644 --- a/server/src/com/cloud/dc/dao/DataCenterDao.java +++ b/server/src/com/cloud/dc/dao/DataCenterDao.java @@ -72,4 +72,5 @@ public interface DataCenterDao extends GenericDao { void saveDetails(DataCenterVO zone); List listDisabledZones(); + List listEnabledZones(); } diff --git a/server/src/com/cloud/dc/dao/DataCenterDaoImpl.java b/server/src/com/cloud/dc/dao/DataCenterDaoImpl.java index b646fe0cd0e..1b88cc997f8 100644 --- a/server/src/com/cloud/dc/dao/DataCenterDaoImpl.java +++ b/server/src/com/cloud/dc/dao/DataCenterDaoImpl.java @@ -292,6 +292,7 @@ public class DataCenterDaoImpl extends GenericDaoBase implem _detailsDao.persist(zone.getId(), details); } + @Override public List listDisabledZones(){ SearchCriteria sc = DisabledZonesSearch.create(); sc.setParameters("allocationState", Grouping.AllocationState.Disabled); @@ -299,5 +300,15 @@ public class DataCenterDaoImpl extends GenericDaoBase implem List dcs = listBy(sc); return dcs; - } + } + + @Override + public List listEnabledZones(){ + SearchCriteria sc = DisabledZonesSearch.create(); + sc.setParameters("allocationState", Grouping.AllocationState.Enabled); + + List dcs = listBy(sc); + + return dcs; + } } diff --git a/server/src/com/cloud/vm/SystemVmLoadScanner.java b/server/src/com/cloud/vm/SystemVmLoadScanner.java index 7649897699b..d37fb67d1b7 100644 --- a/server/src/com/cloud/vm/SystemVmLoadScanner.java +++ b/server/src/com/cloud/vm/SystemVmLoadScanner.java @@ -35,6 +35,17 @@ public class SystemVmLoadScanner { _capacityScanScheduler.scheduleAtFixedRate(getCapacityScanTask(), startupDelayMs, scanIntervalMs, TimeUnit.MILLISECONDS); } + public void stop() { + _capacityScanScheduler.shutdownNow(); + + try { + _capacityScanScheduler.awaitTermination(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + } + + _capacityScanLock.releaseRef(); + } + private Runnable getCapacityScanTask() { return new Runnable() {