From 542880ae76c5e6eefbf61d42364e881495e6f3af Mon Sep 17 00:00:00 2001 From: Rene Moser Date: Thu, 24 Sep 2015 21:10:26 +0200 Subject: [PATCH 1/2] CLOUDSTACK-8848: ensure power state is up to date when handling missing VMs in powerReport There 2 things which has been changed. * We look on power_state_update_time instead of update_time. Didn't make sense to me at all to look at update_time. * Due DB update optimisation, powerState will only be updated if < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT. That is why we can not rely on these information unless we make sure these are up to date. --- .../com/cloud/vm/VirtualMachinePowerStateSyncImpl.java | 9 ++++++++- engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java | 2 ++ .../schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java index d3dcdec3144..19ed71c9931 100644 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java @@ -111,7 +111,14 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat for (VMInstanceVO instance : vmsThatAreMissingReport) { - Date vmStateUpdateTime = instance.getUpdateTime(); + // Make sure powerState is up to date for missing VMs + if (!_instanceDao.isPowerStateUpToDate(instance.getId())) { + s_logger.warn("Detected missing VM but power state is outdated, wait for another process report run for VM id: " + instance.getId()); + _instanceDao.resetVmPowerStateTracking(instance.getId()); + continue; + } + + Date vmStateUpdateTime = instance.getPowerStateUpdateTime(); if (vmStateUpdateTime == null) { s_logger.warn("VM state was updated but update time is null?! vm id: " + instance.getId()); vmStateUpdateTime = currentTime; diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java index 1d522dc6204..8d457fadff5 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java @@ -138,4 +138,6 @@ public interface VMInstanceDao extends GenericDao, StateDao< HashMap countVgpuVMs(Long dcId, Long podId, Long clusterId); VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId); + + boolean isPowerStateUpToDate(long instanceId); } diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java index 427c534a094..33583c83a64 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -804,6 +804,12 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem }); } + @Override + public boolean isPowerStateUpToDate(final long instanceId) { + VMInstanceVO instance = findById(instanceId); + return instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT; + } + @Override public void resetVmPowerStateTracking(final long instanceId) { Transaction.execute(new TransactionCallbackNoReturn() { From b128e567c4eacf4d22e6a8c0d6888c297c7fb726 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Fri, 2 Oct 2015 11:42:42 +0200 Subject: [PATCH 2/2] CLOUDSTACK-8848: added null pointer guard to new public method --- .../cloud/vm/VirtualMachinePowerStateSyncImpl.java | 13 +++++++++---- .../src/com/cloud/vm/dao/VMInstanceDaoImpl.java | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java index 19ed71c9931..3b9d6f596de 100644 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java @@ -24,13 +24,13 @@ import java.util.Map; import javax.inject.Inject; import org.apache.log4j.Logger; - import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import com.cloud.agent.api.HostVmStateReportEntry; import com.cloud.utils.DateUtil; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.dao.VMInstanceDao; public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStateSync { @@ -112,9 +112,14 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat for (VMInstanceVO instance : vmsThatAreMissingReport) { // Make sure powerState is up to date for missing VMs - if (!_instanceDao.isPowerStateUpToDate(instance.getId())) { - s_logger.warn("Detected missing VM but power state is outdated, wait for another process report run for VM id: " + instance.getId()); - _instanceDao.resetVmPowerStateTracking(instance.getId()); + try { + if (!_instanceDao.isPowerStateUpToDate(instance.getId())) { + s_logger.warn("Detected missing VM but power state is outdated, wait for another process report run for VM id: " + instance.getId()); + _instanceDao.resetVmPowerStateTracking(instance.getId()); + continue; + } + } catch (CloudRuntimeException e) { + s_logger.warn("Checked for missing powerstate of a none existing vm", e); continue; } diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java index 33583c83a64..d58ff9e3b91 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -807,6 +807,9 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem @Override public boolean isPowerStateUpToDate(final long instanceId) { VMInstanceVO instance = findById(instanceId); + if(instance == null) { + throw new CloudRuntimeException("checking power state update count on non existing instance " + instanceId); + } return instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT; }