From ab60e9eae9cd7b195f316fd5028ce33945b0d2fc Mon Sep 17 00:00:00 2001
From: Koushik Das <koushik@apache.org>
Date: Tue, 10 Sep 2013 11:54:32 +0530
Subject: [PATCH] CLOUDSTACK-4636: In a scaled up setup all Vm's in a cluster
 were stopped and/or started after management server restart Issue happens as
 there are more than one thread processing connect for a host simultaneously.
 The VM full sync. is not designed to work in this scenario and as a result
 user VMs may get stopped incorrectly. Direct agent scan task runs at regular
 intervals (direct.agent.scan.interval defaulted to 90 secs) and identifies
 hosts that needs to be processed for connect. In a normal scenario hosts
 mostly get connected within that interval and there are no issues. But if due
 to some reason the connect process takes more time and is not completed by
 the time next agent scan runs. In this case, based on the db. state same
 hosts may get picked up again. And then there will be situations where more
 than one thread is processing connect for the same host. The fix is to check
 if there is a thread already processing connect for a host and in this case
 all subsequent threads for that host will simply bail out. Also there may be
 a scenario where one thread already completed processing connect but another
 thread already got scheduled before that and will again repeat the same. This
 is also prevented by putting appropriate checks.

---
 .../cloud/agent/manager/AgentManagerImpl.java |  5 ++++-
 .../cloud/resource/ResourceManagerImpl.java   | 20 ++++++++++++++++---
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/server/src/com/cloud/agent/manager/AgentManagerImpl.java b/server/src/com/cloud/agent/manager/AgentManagerImpl.java
index 572f78b86b6..a7073f47c0d 100755
--- a/server/src/com/cloud/agent/manager/AgentManagerImpl.java
+++ b/server/src/com/cloud/agent/manager/AgentManagerImpl.java
@@ -1376,7 +1376,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     public boolean tapLoadingAgents(Long hostId, TapAgentsAction action) {
         synchronized (_loadingAgents) {
             if (action == TapAgentsAction.Add) {
-                _loadingAgents.add(hostId);
+                if (_loadingAgents.contains(hostId))
+                    return false;
+                else
+                    _loadingAgents.add(hostId);
             } else if (action == TapAgentsAction.Del) {
                 _loadingAgents.remove(hostId);
             } else if (action == TapAgentsAction.Contains) {
diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java
index 9da8c28f26e..b8f6a5acd3a 100755
--- a/server/src/com/cloud/resource/ResourceManagerImpl.java
+++ b/server/src/com/cloud/resource/ResourceManagerImpl.java
@@ -1920,9 +1920,23 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
     @Override
     public Host createHostAndAgent(Long hostId, ServerResource resource, Map<String, String> details, boolean old, List<String> hostTags,
             boolean forRebalance) {
-        _agentMgr.tapLoadingAgents(hostId, TapAgentsAction.Add);
-        Host host = createHostAndAgent(resource, details, old, hostTags, forRebalance);
-        _agentMgr.tapLoadingAgents(hostId, TapAgentsAction.Del);
+        Host host = null;
+        if (_agentMgr.tapLoadingAgents(hostId, TapAgentsAction.Add)) {
+            try {
+                AgentAttache agentattache = _agentMgr.findAttache(hostId);
+                if (agentattache == null) {
+                    s_logger.debug("Creating agent for host " + hostId);
+                    host = createHostAndAgent(resource, details, old, hostTags, forRebalance);
+                    s_logger.debug("Completed creating agent for host " + hostId);
+                } else {
+                    s_logger.debug("Agent already created in another thread for host " + hostId + ", ignore this");
+                }
+            } finally {
+                _agentMgr.tapLoadingAgents(hostId, TapAgentsAction.Del);
+            }
+        } else {
+            s_logger.debug("Agent creation already getting processed in another thread for host " + hostId + ", ignore this");
+        }
         return host;
     }