From db7bc893b94d71e7d2876caee49292cfe8a9d8fd Mon Sep 17 00:00:00 2001 From: Alex Huang Date: Wed, 9 Feb 2011 09:34:18 -0800 Subject: [PATCH] added cluster awareness to vm start/stop --- server/src/com/cloud/ha/HaWorkVO.java | 2 +- .../ClusteredVirtualMachineManagerImpl.java | 3 ++ server/src/com/cloud/vm/ItWorkDao.java | 4 ++ server/src/com/cloud/vm/ItWorkDaoImpl.java | 18 +++++++++ server/src/com/cloud/vm/ItWorkVO.java | 5 +++ .../cloud/vm/VirtualMachineManagerImpl.java | 37 +++++++++++++++++++ 6 files changed, 68 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/ha/HaWorkVO.java b/server/src/com/cloud/ha/HaWorkVO.java index b8f32f09445..51c778ccf2e 100644 --- a/server/src/com/cloud/ha/HaWorkVO.java +++ b/server/src/com/cloud/ha/HaWorkVO.java @@ -192,6 +192,6 @@ public class HaWorkVO { @Override public String toString() { - return new StringBuilder("[HA-Work:id=").append(id).append(":type=").append(workType.toString()).append(":vm=").append(instanceId).append(":state=").append(previousState.toString()).append("]").toString(); + return new StringBuilder("HAWork[").append(id).append("-").append(workType).append("-").append(instanceId).append("-").append(previousState).append("-").append(step).append("]").toString(); } } diff --git a/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java b/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java index 7af989e3404..24bb88fe510 100644 --- a/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/ClusteredVirtualMachineManagerImpl.java @@ -39,6 +39,9 @@ public class ClusteredVirtualMachineManagerImpl extends VirtualMachineManagerImp @Override public void onManagementNodeLeft(List nodeList, long selfNodeId) { + for (ManagementServerHostVO node : nodeList) { + cancelWorkItems(node.getMsid()); + } } @Override diff --git a/server/src/com/cloud/vm/ItWorkDao.java b/server/src/com/cloud/vm/ItWorkDao.java index c9cc5dc4032..75679059931 100644 --- a/server/src/com/cloud/vm/ItWorkDao.java +++ b/server/src/com/cloud/vm/ItWorkDao.java @@ -17,6 +17,8 @@ */ package com.cloud.vm; +import java.util.List; + import com.cloud.utils.db.GenericDao; import com.cloud.vm.ItWorkVO.Step; import com.cloud.vm.VirtualMachine.State; @@ -38,4 +40,6 @@ public interface ItWorkDao extends GenericDao { void cleanup(long wait); boolean updateStep(ItWorkVO work, Step step); + + List listWorkInProgressFor(long nodeId); } diff --git a/server/src/com/cloud/vm/ItWorkDaoImpl.java b/server/src/com/cloud/vm/ItWorkDaoImpl.java index 7eb1541aea1..7168b0cd9ff 100644 --- a/server/src/com/cloud/vm/ItWorkDaoImpl.java +++ b/server/src/com/cloud/vm/ItWorkDaoImpl.java @@ -17,6 +17,8 @@ */ package com.cloud.vm; +import java.util.List; + import javax.ejb.Local; import com.cloud.utils.db.GenericDaoBase; @@ -32,6 +34,7 @@ public class ItWorkDaoImpl extends GenericDaoBase implements I protected final SearchBuilder AllFieldsSearch; protected final SearchBuilder CleanupSearch; protected final SearchBuilder OutstandingWorkSearch; + protected final SearchBuilder WorkInProgressSearch; protected ItWorkDaoImpl() { super(); @@ -52,6 +55,11 @@ public class ItWorkDaoImpl extends GenericDaoBase implements I OutstandingWorkSearch.and("op", OutstandingWorkSearch.entity().getType(), Op.EQ); OutstandingWorkSearch.and("step", OutstandingWorkSearch.entity().getStep(), Op.NEQ); OutstandingWorkSearch.done(); + + WorkInProgressSearch = createSearchBuilder(); + WorkInProgressSearch.and("server", WorkInProgressSearch.entity().getManagementServerId(), Op.EQ); + WorkInProgressSearch.and("step", WorkInProgressSearch.entity().getStep(), Op.NIN); + WorkInProgressSearch.done(); } @Override @@ -85,4 +93,14 @@ public class ItWorkDaoImpl extends GenericDaoBase implements I work.setStep(step); return update(work.getId(), work); } + + @Override + public List listWorkInProgressFor(long nodeId) { + SearchCriteria sc = WorkInProgressSearch.create(); + sc.setParameters("server", nodeId); + sc.setParameters("step", Step.Done); + + return search(sc, null); + + } } diff --git a/server/src/com/cloud/vm/ItWorkVO.java b/server/src/com/cloud/vm/ItWorkVO.java index bdc6a0c140c..59f9f6885b3 100644 --- a/server/src/com/cloud/vm/ItWorkVO.java +++ b/server/src/com/cloud/vm/ItWorkVO.java @@ -155,4 +155,9 @@ public class ItWorkVO { public long getSecondsTaskHasBeenCreated() { return InaccurateClock.getTimeInSeconds() - this.createdAt; } + + @Override + public String toString() { + return new StringBuilder("ItWork[").append(id).append("-").append(type.toString()).append("-").append(instanceId).append("-").append(step.toString()).append("]").toString(); + } } diff --git a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java index 236bfad8c34..47aa27804d0 100755 --- a/server/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/server/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -409,6 +409,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene @Override public boolean start() { _executor.scheduleAtFixedRate(new CleanupTask(), _cleanupInterval, _cleanupInterval, TimeUnit.SECONDS); + cancelWorkItems(_nodeId); return true; } @@ -446,6 +447,7 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene _nodeId = _clusterMgr.getId(); _agentMgr.registerForHostEvents(this, true, true, true); + return true; } @@ -1050,6 +1052,41 @@ public class VirtualMachineManagerImpl implements VirtualMachineManager, Listene } } + protected void cancelWorkItems(long nodeId) { + GlobalLock scanLock = GlobalLock.getInternLock(this.getClass().getName()); + + try { + if (scanLock.lock(3)) { + try { + List works = _workDao.listWorkInProgressFor(nodeId); + for (ItWorkVO work : works) { + s_logger.info("Handling unfinished work item: " + work); + try { + VMInstanceVO vm = _vmDao.findById(work.getInstanceId()); + if (vm != null) { + if (work.getType() == State.Starting) { + _haMgr.scheduleRestart(vm, true); + } else if (work.getType() == State.Stopping) { + _haMgr.scheduleStop(vm, vm.getHostId(), WorkType.CheckStop); + } else if (work.getType() == State.Migrating) { + _haMgr.scheduleMigration(vm); + } + } + work.setStep(Step.Done); + _workDao.update(work.getId(), work); + } catch (Exception e) { + s_logger.error("Error while handling " + work, e); + } + } + } finally { + scanLock.unlock(); + } + } + } finally { + scanLock.releaseRef(); + } + } + @Override public boolean migrateAway(VirtualMachine.Type vmType, long vmId, long srcHostId) throws InsufficientServerCapacityException { VirtualMachineGuru vmGuru = _vmGurus.get(vmType);