diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java index d3dcdec3144..3b9d6f596de 100644 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java @@ -24,13 +24,13 @@ import java.util.Map; import javax.inject.Inject; import org.apache.log4j.Logger; - import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import com.cloud.agent.api.HostVmStateReportEntry; import com.cloud.utils.DateUtil; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.dao.VMInstanceDao; public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStateSync { @@ -111,7 +111,19 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat for (VMInstanceVO instance : vmsThatAreMissingReport) { - Date vmStateUpdateTime = instance.getUpdateTime(); + // Make sure powerState is up to date for missing VMs + try { + if (!_instanceDao.isPowerStateUpToDate(instance.getId())) { + s_logger.warn("Detected missing VM but power state is outdated, wait for another process report run for VM id: " + instance.getId()); + _instanceDao.resetVmPowerStateTracking(instance.getId()); + continue; + } + } catch (CloudRuntimeException e) { + s_logger.warn("Checked for missing powerstate of a none existing vm", e); + continue; + } + + Date vmStateUpdateTime = instance.getPowerStateUpdateTime(); if (vmStateUpdateTime == null) { s_logger.warn("VM state was updated but update time is null?! vm id: " + instance.getId()); vmStateUpdateTime = currentTime; diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java index 1d522dc6204..8d457fadff5 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java @@ -138,4 +138,6 @@ public interface VMInstanceDao extends GenericDao, StateDao< HashMap countVgpuVMs(Long dcId, Long podId, Long clusterId); VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId); + + boolean isPowerStateUpToDate(long instanceId); } diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java index 427c534a094..d58ff9e3b91 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -804,6 +804,15 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem }); } + @Override + public boolean isPowerStateUpToDate(final long instanceId) { + VMInstanceVO instance = findById(instanceId); + if(instance == null) { + throw new CloudRuntimeException("checking power state update count on non existing instance " + instanceId); + } + return instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT; + } + @Override public void resetVmPowerStateTracking(final long instanceId) { Transaction.execute(new TransactionCallbackNoReturn() {