Merge branch 'main' into cks-enhancements-upstream

.asf.yaml

@@ -60,6 +60,7 @@ github:
     - bernardodemarco
     - abh1sar
     - FelipeM525
+    - lucas-a-martins
 
   protected_branches: ~
 

.github/ISSUE_TEMPLATE/bug.yml

@@ -26,6 +26,13 @@ body:
   attributes:
     label: problem
     value: The long description of your problem
+- type: markdown
+  attributes:
+    value: "## What versions of cloudstack and any infra components are you using"
+- type: textarea
+  attributes:
+    label: versions
+    value: The versions of ACS, hypervisors, storage, network etc..
 - type: textarea
   attributes:
     label: The steps to reproduce the bug

.python-version

@@ -1 +1 @@
-3.6
+3.10

README.md

@@ -1,6 +1,6 @@
 # Apache CloudStack [![Build Status](https://github.com/apache/cloudstack/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/build.yml) [![UI Build](https://github.com/apache/cloudstack/actions/workflows/ui.yml/badge.svg)](https://github.com/apache/cloudstack/actions/workflows/ui.yml) [![License Check](https://github.com/apache/cloudstack/actions/workflows/rat.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/rat.yml) [![Simulator CI](https://github.com/apache/cloudstack/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/ci.yml) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache_cloudstack&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache_cloudstack) [![codecov](https://codecov.io/gh/apache/cloudstack/branch/main/graph/badge.svg)](https://codecov.io/gh/apache/cloudstack)
 
-[![Apache CloudStack](tools/logo/acsxmas.jpg)](https://cloudstack.apache.org/)
+[![Apache CloudStack](tools/logo/apache_cloudstack.png)](https://cloudstack.apache.org/)
 
 Apache CloudStack is open source software designed to deploy and manage large
 networks of virtual machines, as a highly available, highly scalable

agent/conf/agent.properties

@@ -209,7 +209,7 @@ hypervisor.type=kvm
 # the management server would send.
 # In case of arm64 (aarch64), this will change the machine type to 'virt' and
 # adds a SCSI and a USB controller in the domain xml.
-# Possible values: x86_64 | aarch64
+# Possible values: x86_64 | aarch64 | s390x
 # If null (default), defaults to the VM's OS architecture
 #guest.cpu.arch=
 
@@ -286,6 +286,7 @@ hypervisor.type=kvm
 
 # The model of Watchdog timer to present to the Guest.
 # For all models refer to the libvirt documentation.
+# PLEASE NOTE: to disable the watchdogs definitions, use value: none
 #vm.watchdog.model=i6300esb
 
 # Action to take when the Guest/Instance is no longer notifying the Watchdog timer.
@@ -433,3 +434,10 @@ iscsi.session.cleanup.enabled=false
 
 # Implicit host tags managed by agent.properties
 # host.tags=
+
+# Timeout(in seconds) for SSL handshake when agent connects to server. When no value is set then default value of 30s
+# will be used
+#ssl.handshake.timeout=
+
+# Wait(in seconds) during agent reconnections. When no value is set then default value of 5s will be used
+#backoff.seconds=

agent/src/main/java/com/cloud/agent/AgentShell.java

@@ -16,29 +16,6 @@
 // under the License.
 package com.cloud.agent;
 
-import com.cloud.agent.Agent.ExitStatus;
-import com.cloud.agent.dao.StorageComponent;
-import com.cloud.agent.dao.impl.PropertiesStorage;
-import com.cloud.agent.properties.AgentProperties;
-import com.cloud.agent.properties.AgentPropertiesFileHandler;
-import com.cloud.resource.ServerResource;
-import com.cloud.utils.LogUtils;
-import com.cloud.utils.ProcessUtil;
-import com.cloud.utils.PropertiesUtil;
-import com.cloud.utils.backoff.BackoffAlgorithm;
-import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
-import com.cloud.utils.exception.CloudRuntimeException;
-import org.apache.commons.daemon.Daemon;
-import org.apache.commons.daemon.DaemonContext;
-import org.apache.commons.daemon.DaemonInitException;
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.core.config.Configurator;
-
-import javax.naming.ConfigurationException;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -53,6 +30,31 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.UUID;
 
+import javax.naming.ConfigurationException;
+
+import org.apache.commons.daemon.Daemon;
+import org.apache.commons.daemon.DaemonContext;
+import org.apache.commons.daemon.DaemonInitException;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configurator;
+
+import com.cloud.agent.Agent.ExitStatus;
+import com.cloud.agent.dao.StorageComponent;
+import com.cloud.agent.dao.impl.PropertiesStorage;
+import com.cloud.agent.properties.AgentProperties;
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
+import com.cloud.resource.ServerResource;
+import com.cloud.utils.LogUtils;
+import com.cloud.utils.ProcessUtil;
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.backoff.BackoffAlgorithm;
+import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
+import com.cloud.utils.exception.CloudRuntimeException;
+
 public class AgentShell implements IAgentShell, Daemon {
     protected static Logger LOGGER = LogManager.getLogger(AgentShell.class);
 
@@ -77,6 +79,7 @@ public class AgentShell implements IAgentShell, Daemon {
     private String hostToConnect;
     private String connectedHost;
     private Long preferredHostCheckInterval;
+    private boolean connectionTransfer = false;
     protected AgentProperties agentProperties = new AgentProperties();
 
     public AgentShell() {
@@ -215,6 +218,14 @@ public class AgentShell implements IAgentShell, Daemon {
             _storage.persist(name, value);
     }
 
+    public boolean isConnectionTransfer() {
+        return connectionTransfer;
+    }
+
+    public void setConnectionTransfer(boolean connectionTransfer) {
+        this.connectionTransfer = connectionTransfer;
+    }
+
     void loadProperties() throws ConfigurationException {
         final File file = PropertiesUtil.findConfigFile("agent.properties");
 
@@ -406,7 +417,9 @@ public class AgentShell implements IAgentShell, Daemon {
 
         LOGGER.info("Defaulting to the constant time backoff algorithm");
         _backoff = new ConstantTimeBackoff();
-        _backoff.configure("ConstantTimeBackoff", new HashMap<String, Object>());
+        Map<String, Object> map = new HashMap<>();
+        map.put("seconds", _properties.getProperty("backoff.seconds"));
+        _backoff.configure("ConstantTimeBackoff", map);
     }
 
     private void launchAgent() throws ConfigurationException {
@@ -455,6 +468,11 @@ public class AgentShell implements IAgentShell, Daemon {
         agent.start();
     }
 
+    @Override
+    public Integer getSslHandshakeTimeout() {
+        return AgentPropertiesFileHandler.getPropertyValue(AgentProperties.SSL_HANDSHAKE_TIMEOUT);
+    }
+
     public synchronized int getNextAgentId() {
         return _nextAgentId++;
     }

agent/src/main/java/com/cloud/agent/IAgentShell.java

@@ -70,4 +70,10 @@ public interface IAgentShell {
     String getConnectedHost();
 
     void launchNewAgent(ServerResource resource) throws ConfigurationException;
+
+    boolean isConnectionTransfer();
+
+    void setConnectionTransfer(boolean connectionTransfer);
+
+    Integer getSslHandshakeTimeout();
 }

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -383,7 +383,7 @@ public class AgentProperties{
     /**
      * This param will set the CPU architecture for the domain to override what the management server would send.<br>
      * In case of arm64 (aarch64), this will change the machine type to 'virt' and add a SCSI and a USB controller in the domain XML.<br>
-     * Possible values: x86_64 | aarch64 <br>
+     * Possible values: x86_64 | aarch64 | s390x <br>
      * Data type: String.<br>
      * Default value: <code>null</code> (will set use the architecture of the VM's OS).
      */
@@ -516,6 +516,7 @@ public class AgentProperties{
     /**
      * The model of Watchdog timer to present to the Guest.<br>
      * For all models refer to the libvirt documentation.<br>
+     * PLEASE NOTE: to disable the watchdogs definitions, use value: none
      * Data type: String.<br>
      * Default value: <code>i6300esb</code>
      */
@@ -810,6 +811,13 @@ public class AgentProperties{
      */
     public static final Property<String> HOST_TAGS = new Property<>("host.tags", null, String.class);
 
+    /**
+     * Timeout for SSL handshake in seconds
+     * Data type: Integer.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<Integer> SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", null, Integer.class);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

agent/src/test/java/com/cloud/agent/AgentShellTest.java

@@ -362,4 +362,11 @@ public class AgentShellTest {
 
         Assert.assertEquals(expected, shell.getConnectedHost());
     }
+
+    @Test
+    public void testGetSslHandshakeTimeout() {
+        Integer expected = 1;
+        agentPropertiesFileHandlerMocked.when(() -> AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.SSL_HANDSHAKE_TIMEOUT))).thenReturn(expected);
+        Assert.assertEquals(expected, agentShellSpy.getSslHandshakeTimeout());
+    }
 }

agent/src/test/java/com/cloud/agent/AgentTest.java

@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import javax.naming.ConfigurationException;
+
+import org.apache.logging.log4j.Logger;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import com.cloud.resource.ServerResource;
+import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
+import com.cloud.utils.nio.Link;
+import com.cloud.utils.nio.NioConnection;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AgentTest {
+    Agent agent;
+    private AgentShell shell;
+    private ServerResource serverResource;
+    private Logger logger;
+
+    @Before
+    public void setUp() throws ConfigurationException {
+        shell = mock(AgentShell.class);
+        serverResource = mock(ServerResource.class);
+        doReturn(true).when(serverResource).configure(any(), any());
+        doReturn(1).when(shell).getWorkers();
+        doReturn(1).when(shell).getPingRetries();
+        agent = new Agent(shell, 1, serverResource);
+        logger = mock(Logger.class);
+        ReflectionTestUtils.setField(agent, "logger", logger);
+    }
+
+    @Test
+    public void testGetLinkLogNullLinkReturnsEmptyString() {
+        Link link = null;
+        String result = agent.getLinkLog(link);
+        assertEquals("", result);
+    }
+
+    @Test
+    public void testGetLinkLogLinkWithTraceEnabledReturnsLinkLogWithHashCode() {
+        Link link = mock(Link.class);
+        InetSocketAddress socketAddress = new InetSocketAddress("192.168.1.100", 1111);
+        when(link.getSocketAddress()).thenReturn(socketAddress);
+        when(logger.isTraceEnabled()).thenReturn(true);
+
+        String result = agent.getLinkLog(link);
+        System.out.println(result);
+        assertTrue(result.startsWith(System.identityHashCode(link) + "-"));
+        assertTrue(result.contains("192.168.1.100"));
+    }
+
+    @Test
+    public void testGetAgentNameWhenServerResourceIsNull() {
+        ReflectionTestUtils.setField(agent, "serverResource", null);
+        assertEquals("Agent", agent.getAgentName());
+    }
+
+    @Test
+    public void testGetAgentNameWhenAppendAgentNameIsTrue() {
+        when(serverResource.isAppendAgentNameToLogs()).thenReturn(true);
+        when(serverResource.getName()).thenReturn("TestAgent");
+
+        String agentName = agent.getAgentName();
+        assertEquals("TestAgent", agentName);
+    }
+
+    @Test
+    public void testGetAgentNameWhenAppendAgentNameIsFalse() {
+        when(serverResource.isAppendAgentNameToLogs()).thenReturn(false);
+
+        String agentName = agent.getAgentName();
+        assertEquals("Agent", agentName);
+    }
+
+    @Test
+    public void testAgentInitialization() {
+        Runtime.getRuntime().removeShutdownHook(agent.shutdownThread);
+        when(shell.getPingRetries()).thenReturn(3);
+        when(shell.getWorkers()).thenReturn(5);
+        agent.setupShutdownHookAndInitExecutors();
+        assertNotNull(agent.selfTaskExecutor);
+        assertNotNull(agent.outRequestHandler);
+        assertNotNull(agent.requestHandler);
+    }
+
+    @Test
+    public void testAgentShutdownHookAdded() {
+        Runtime.getRuntime().removeShutdownHook(agent.shutdownThread);
+        agent.setupShutdownHookAndInitExecutors();
+        verify(logger).trace("Adding shutdown hook");
+    }
+
+    @Test
+    public void testGetResourceGuidValidGuidAndResourceName() {
+        when(shell.getGuid()).thenReturn("12345");
+        String result = agent.getResourceGuid();
+        assertTrue(result.startsWith("12345-" + ServerResource.class.getSimpleName()));
+    }
+
+    @Test
+    public void testGetZoneReturnsValidZone() {
+        when(shell.getZone()).thenReturn("ZoneA");
+        String result = agent.getZone();
+        assertEquals("ZoneA", result);
+    }
+
+    @Test
+    public void testGetPodReturnsValidPod() {
+        when(shell.getPod()).thenReturn("PodA");
+        String result = agent.getPod();
+        assertEquals("PodA", result);
+    }
+
+    @Test
+    public void testSetLinkAssignsLink() {
+        Link mockLink = mock(Link.class);
+        agent.setLink(mockLink);
+        assertEquals(mockLink, agent.link);
+    }
+
+    @Test
+    public void testGetResourceReturnsServerResource() {
+        ServerResource mockResource = mock(ServerResource.class);
+        ReflectionTestUtils.setField(agent, "serverResource", mockResource);
+        ServerResource result = agent.getResource();
+        assertSame(mockResource, result);
+    }
+
+    @Test
+    public void testGetResourceName() {
+        String result = agent.getResourceName();
+        assertTrue(result.startsWith(ServerResource.class.getSimpleName()));
+    }
+
+    @Test
+    public void testUpdateLastPingResponseTimeUpdatesCurrentTime() {
+        long beforeUpdate = System.currentTimeMillis();
+        agent.updateLastPingResponseTime();
+        long updatedTime = agent.lastPingResponseTime.get();
+        assertTrue(updatedTime >= beforeUpdate);
+        assertTrue(updatedTime <= System.currentTimeMillis());
+    }
+
+    @Test
+    public void testGetNextSequenceIncrementsSequence() {
+        long initialSequence = agent.getNextSequence();
+        long nextSequence = agent.getNextSequence();
+        assertEquals(initialSequence + 1, nextSequence);
+        long thirdSequence = agent.getNextSequence();
+        assertEquals(nextSequence + 1, thirdSequence);
+    }
+
+    @Test
+    public void testRegisterControlListenerAddsListener() {
+        IAgentControlListener listener = mock(IAgentControlListener.class);
+        agent.registerControlListener(listener);
+        assertTrue(agent.controlListeners.contains(listener));
+    }
+
+    @Test
+    public void testUnregisterControlListenerRemovesListener() {
+        IAgentControlListener listener = mock(IAgentControlListener.class);
+        agent.registerControlListener(listener);
+        assertTrue(agent.controlListeners.contains(listener));
+        agent.unregisterControlListener(listener);
+        assertFalse(agent.controlListeners.contains(listener));
+    }
+
+    @Test
+    public void testCloseAndTerminateLinkLinkIsNullDoesNothing() {
+        agent.closeAndTerminateLink(null);
+    }
+
+    @Test
+    public void testCloseAndTerminateLinkValidLinkCallsCloseAndTerminate() {
+        Link mockLink = mock(Link.class);
+        agent.closeAndTerminateLink(mockLink);
+        verify(mockLink).close();
+        verify(mockLink).terminated();
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionConnectionIsNullDoesNothing() {
+        agent.connection = null;
+        agent.stopAndCleanupConnection(false);
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionValidConnectionNoWaitStopsAndCleansUp() throws IOException {
+        NioConnection mockConnection = mock(NioConnection.class);
+        agent.connection = mockConnection;
+        agent.stopAndCleanupConnection(false);
+        verify(mockConnection).stop();
+        verify(mockConnection).cleanUp();
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionCleanupThrowsIOExceptionLogsWarning() throws IOException {
+        NioConnection mockConnection = mock(NioConnection.class);
+        agent.connection = mockConnection;
+        doThrow(new IOException("Cleanup failed")).when(mockConnection).cleanUp();
+        agent.stopAndCleanupConnection(false);
+        verify(mockConnection).stop();
+        verify(logger).warn(eq("Fail to clean up old connection. {}"), any(IOException.class));
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionValidConnectionWaitForStopWaitsForStartupToStop() throws IOException {
+        NioConnection mockConnection = mock(NioConnection.class);
+        ConstantTimeBackoff mockBackoff = mock(ConstantTimeBackoff.class);
+        mockBackoff.setTimeToWait(0);
+        agent.connection = mockConnection;
+        when(shell.getBackoffAlgorithm()).thenReturn(mockBackoff);
+        when(mockConnection.isStartup()).thenReturn(true, true, false);
+        agent.stopAndCleanupConnection(true);
+        verify(mockConnection).stop();
+        verify(mockConnection).cleanUp();
+        verify(mockBackoff, times(3)).waitBeforeRetry();
+    }
+}

api/src/main/java/com/cloud/host/Host.java

@@ -177,6 +177,8 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
      */
     Long getManagementServerId();
 
+    Long getLastManagementServerId();
+
     /*
      *@return removal date
      */

api/src/main/java/com/cloud/host/Status.java

@@ -127,6 +127,7 @@ public enum Status {
         s_fsm.addTransition(Status.Connecting, Event.HostDown, Status.Down);
         s_fsm.addTransition(Status.Connecting, Event.Ping, Status.Connecting);
         s_fsm.addTransition(Status.Connecting, Event.ManagementServerDown, Status.Disconnected);
+        s_fsm.addTransition(Status.Connecting, Event.StartAgentRebalance, Status.Rebalancing);
         s_fsm.addTransition(Status.Connecting, Event.AgentDisconnected, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.PingTimeout, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.AgentDisconnected, Status.Alert);

api/src/main/java/com/cloud/resource/ResourceService.java

@@ -23,11 +23,11 @@ import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd;
 import org.apache.cloudstack.api.command.admin.cluster.UpdateClusterCmd;
 import org.apache.cloudstack.api.command.admin.host.AddHostCmd;
 import org.apache.cloudstack.api.command.admin.host.AddSecondaryStorageCmd;
-import org.apache.cloudstack.api.command.admin.host.CancelMaintenanceCmd;
+import org.apache.cloudstack.api.command.admin.host.CancelHostMaintenanceCmd;
 import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd;
 import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd;
 import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd;
-import org.apache.cloudstack.api.command.admin.host.PrepareForMaintenanceCmd;
+import org.apache.cloudstack.api.command.admin.host.PrepareForHostMaintenanceCmd;
 import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd;
 import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd;
 
@@ -51,7 +51,7 @@ public interface ResourceService {
 
     Host autoUpdateHostAllocationState(Long hostId, ResourceState.Event resourceEvent) throws NoTransitionException;
 
-    Host cancelMaintenance(CancelMaintenanceCmd cmd);
+    Host cancelMaintenance(CancelHostMaintenanceCmd cmd);
 
     Host reconnectHost(ReconnectHostCmd cmd) throws AgentUnavailableException;
 
@@ -69,7 +69,7 @@ public interface ResourceService {
 
     List<? extends Host> discoverHosts(AddSecondaryStorageCmd cmd) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
 
-    Host maintain(PrepareForMaintenanceCmd cmd);
+    Host maintain(PrepareForHostMaintenanceCmd cmd);
 
     Host declareHostAsDegraded(DeclareHostAsDegradedCmd cmd) throws NoTransitionException;
 

api/src/main/java/com/cloud/server/ManagementServerHostStats.java

@@ -19,6 +19,7 @@
 package com.cloud.server;
 
 import java.util.Date;
+import java.util.List;
 
 /**
  * management server related stats
@@ -70,6 +71,10 @@ public interface ManagementServerHostStats {
 
     String getOsDistribution();
 
+    List<String> getLastAgents();
+
+    List<String> getAgents();
+
     int getAgentCount();
 
     long getHeapMemoryUsed();

api/src/main/java/org/apache/cloudstack/acl/RoleService.java

@@ -30,6 +30,11 @@ public interface RoleService {
     ConfigKey<Boolean> EnableDynamicApiChecker = new ConfigKey<>("Advanced", Boolean.class, "dynamic.apichecker.enabled", "false",
             "If set to true, this enables the dynamic role-based api access checker and disables the default static role-based api access checker.", true);
 
+    ConfigKey<Integer> DynamicApiCheckerCachePeriod = new ConfigKey<>("Advanced", Integer.class,
+            "dynamic.apichecker.cache.period", "0",
+            "Defines the expiration time in seconds for the Dynamic API Checker cache, determining how long cached data is retained before being refreshed. If set to zero then caching will be disabled",
+            false);
+
     boolean isEnabled();
 
     /**

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -1157,9 +1157,12 @@ public class ApiConstants {
     public static final String LOGOUT = "logout";
     public static final String LIST_IDPS = "listIdps";
 
-    public static final String READY_FOR_SHUTDOWN = "readyforshutdown";
+    public static final String MAINTENANCE_INITIATED = "maintenanceinitiated";
     public static final String SHUTDOWN_TRIGGERED = "shutdowntriggered";
+    public static final String READY_FOR_SHUTDOWN = "readyforshutdown";
     public static final String PENDING_JOBS_COUNT = "pendingjobscount";
+    public static final String AGENTS_COUNT = "agentscount";
+    public static final String AGENTS = "agents";
 
     public static final String PUBLIC_MTU = "publicmtu";
     public static final String PRIVATE_MTU = "privatemtu";

api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java

@@ -100,7 +100,7 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd {
             dv = EnumSet.of(DomainDetails.all);
         } else {
             try {
-                ArrayList<DomainDetails> dc = new ArrayList<DomainDetails>();
+                ArrayList<DomainDetails> dc = new ArrayList<>();
                 for (String detail : viewDetails) {
                     dc.add(DomainDetails.valueOf(detail));
                 }
@@ -142,7 +142,10 @@ public class ListDomainsCmd extends BaseListCmd implements UserCmd {
         if (CollectionUtils.isEmpty(response)) {
             return;
         }
-        _resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag());
+        EnumSet<DomainDetails> details = getDetails();
+        if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
+            _resourceLimitService.updateTaggedResourceLimitsAndCountsForDomains(response, getTag());
+        }
         if (!getShowIcon()) {
             return;
         }

api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelHostMaintenanceCmd.java

@@ -33,7 +33,7 @@ import com.cloud.user.Account;
 
 @APICommand(name = "cancelHostMaintenance", description = "Cancels host maintenance.", responseObject = HostResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class CancelMaintenanceCmd extends BaseAsyncCmd {
+public class CancelHostMaintenanceCmd extends BaseAsyncCmd {
 
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java

@@ -31,6 +31,7 @@ import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.ClusterResponse;
 import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.ManagementServerResponse;
 import org.apache.cloudstack.api.response.PodResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
@@ -105,6 +106,9 @@ public class ListHostsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "hypervisor type of host: XenServer,KVM,VMware,Hyperv,BareMetal,Simulator")
     private String hypervisor;
 
+    @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "the id of the management server", since="4.21.0")
+    private Long managementServerId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -189,6 +193,10 @@ public class ListHostsCmd extends BaseListCmd {
         return outOfBandManagementPowerState;
     }
 
+    public Long getManagementServerId() {
+        return managementServerId;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/host/PrepareForHostMaintenanceCmd.java

@@ -33,7 +33,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
 
 @APICommand(name = "prepareHostForMaintenance", description = "Prepares a host for maintenance.", responseObject = HostResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class PrepareForMaintenanceCmd extends BaseAsyncCmd {
+public class PrepareForHostMaintenanceCmd extends BaseAsyncCmd {
 
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java

@@ -157,7 +157,10 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC
         if (CollectionUtils.isEmpty(response)) {
             return;
         }
-        _resourceLimitService.updateTaggedResourceLimitsAndCountsForAccounts(response, getTag());
+        EnumSet<DomainDetails> details = getDetails();
+        if (details.contains(DomainDetails.all) || details.contains(DomainDetails.resource)) {
+            _resourceLimitService.updateTaggedResourceLimitsAndCountsForAccounts(response, getTag());
+        }
         if (!getShowIcon()) {
             return;
         }

api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreateFirewallRuleCmd.java

@@ -19,6 +19,7 @@ package org.apache.cloudstack.api.command.user.firewall;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.collections.CollectionUtils;
 
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
@@ -40,6 +41,7 @@ import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.IpAddress;
 import com.cloud.network.rules.FirewallRule;
 import com.cloud.user.Account;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.net.NetUtils;
 
 @APICommand(name = "createFirewallRule", description = "Creates a firewall rule for a given IP address", responseObject = FirewallResponse.class, entityType = {FirewallRule.class},
@@ -125,14 +127,13 @@ public class CreateFirewallRuleCmd extends BaseAsyncCreateCmd implements Firewal
 
     @Override
     public List<String> getSourceCidrList() {
-        if (cidrlist != null) {
+        if (CollectionUtils.isNotEmpty(cidrlist) && !(cidrlist.size() == 1 && StringUtils.isBlank(cidrlist.get(0)))) {
             return cidrlist;
         } else {
-            List<String> oneCidrList = new ArrayList<String>();
+            List<String> oneCidrList = new ArrayList<>();
             oneCidrList.add(NetUtils.ALL_IP4_CIDRS);
             return oneCidrList;
         }
-
     }
 
     // ///////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/AsyncJobResponse.java

@@ -83,9 +83,13 @@ public class AsyncJobResponse extends BaseResponse {
     @Param(description = "the unique ID of the instance/entity object related to the job")
     private String jobInstanceId;
 
-    @SerializedName("managementserverid")
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID)
     @Param(description = "the msid of the management server on which the job is running", since = "4.19")
-    private Long msid;
+    private String managementServerId;
+
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_NAME)
+    @Param(description = "the management server name of the host", since = "4.21.0")
+    private String managementServerName;
 
     @SerializedName(ApiConstants.CREATED)
     @Param(description = "  the created date of the job")
@@ -156,7 +160,11 @@ public class AsyncJobResponse extends BaseResponse {
         this.removed = removed;
     }
 
-    public void setMsid(Long msid) {
-        this.msid = msid;
+    public void setManagementServerId(String managementServerId) {
+        this.managementServerId = managementServerId;
+    }
+
+    public void setManagementServerName(String managementServerName) {
+        this.managementServerName = managementServerName;
     }
 }

api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java

@@ -16,465 +16,20 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
-import java.util.Date;
-
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.BaseResponse;
 import org.apache.cloudstack.api.EntityReference;
 
 import com.cloud.host.Host;
-import com.cloud.host.Status;
 import com.cloud.serializer.Param;
 import com.google.gson.annotations.SerializedName;
 
 @EntityReference(value = Host.class)
-public class HostForMigrationResponse extends BaseResponse {
-    @SerializedName(ApiConstants.ID)
-    @Param(description = "the ID of the host")
-    private String id;
-
-    @SerializedName(ApiConstants.NAME)
-    @Param(description = "the name of the host")
-    private String name;
-
-    @SerializedName(ApiConstants.STATE)
-    @Param(description = "the state of the host")
-    private Status state;
-
-    @SerializedName("disconnected")
-    @Param(description = "true if the host is disconnected. False otherwise.")
-    private Date disconnectedOn;
-
-    @SerializedName(ApiConstants.TYPE)
-    @Param(description = "the host type")
-    private Host.Type hostType;
-
-    @SerializedName("oscategoryid")
-    @Param(description = "the OS category ID of the host")
-    private String osCategoryId;
-
-    @SerializedName("oscategoryname")
-    @Param(description = "the OS category name of the host")
-    private String osCategoryName;
-
-    @SerializedName(ApiConstants.IP_ADDRESS)
-    @Param(description = "the IP address of the host")
-    private String ipAddress;
-
-    @SerializedName(ApiConstants.ZONE_ID)
-    @Param(description = "the Zone ID of the host")
-    private String zoneId;
-
-    @SerializedName(ApiConstants.ZONE_NAME)
-    @Param(description = "the Zone name of the host")
-    private String zoneName;
-
-    @SerializedName(ApiConstants.POD_ID)
-    @Param(description = "the Pod ID of the host")
-    private String podId;
-
-    @SerializedName("podname")
-    @Param(description = "the Pod name of the host")
-    private String podName;
-
-    @SerializedName("version")
-    @Param(description = "the host version")
-    private String version;
-
-    @SerializedName(ApiConstants.HYPERVISOR)
-    @Param(description = "the host hypervisor")
-    private String hypervisor;
-
-    @SerializedName("cpunumber")
-    @Param(description = "the CPU number of the host")
-    private Integer cpuNumber;
-
-    @SerializedName("cpuspeed")
-    @Param(description = "the CPU speed of the host")
-    private Long cpuSpeed;
-
-    @Deprecated
-    @SerializedName("cpuallocated")
-    @Param(description = "the amount of the host's CPU currently allocated")
-    private String cpuAllocated;
-
-    @SerializedName("cpuallocatedvalue")
-    @Param(description = "the amount of the host's CPU currently allocated in MHz")
-    private Long cpuAllocatedValue;
-
-    @SerializedName("cpuallocatedpercentage")
-    @Param(description = "the amount of the host's CPU currently allocated in percentage")
-    private String cpuAllocatedPercentage;
-
-    @SerializedName("cpuallocatedwithoverprovisioning")
-    @Param(description = "the amount of the host's CPU currently allocated after applying the cpu.overprovisioning.factor")
-    private String cpuAllocatedWithOverprovisioning;
-
-    @SerializedName("cpuused")
-    @Param(description = "the amount of the host's CPU currently used")
-    private String cpuUsed;
-
-    @SerializedName("cpuwithoverprovisioning")
-    @Param(description = "the amount of the host's CPU after applying the cpu.overprovisioning.factor ")
-    private String cpuWithOverprovisioning;
-
-    @Deprecated
-    @SerializedName("memorytotal")
-    @Param(description = "the memory total of the host, this parameter is deprecated use memorywithoverprovisioning")
-    private Long memoryTotal;
-
-    @SerializedName("memorywithoverprovisioning")
-    @Param(description = "the amount of the host's memory after applying the mem.overprovisioning.factor ")
-    private String memWithOverprovisioning;
-
-    @SerializedName("averageload")
-    @Param(description = "the cpu average load on the host")
-    private Long averageLoad;
-
-    @SerializedName("networkkbsread")
-    @Param(description = "the incoming network traffic on the host")
-    private Long networkKbsRead;
-
-    @SerializedName("networkkbswrite")
-    @Param(description = "the outgoing network traffic on the host")
-    private Long networkKbsWrite;
-
-    @Deprecated
-    @SerializedName("memoryallocated")
-    @Param(description = "the amount of the host's memory currently allocated")
-    private String memoryAllocated;
-
-    @SerializedName("memoryallocatedpercentage")
-    @Param(description = "the amount of the host's memory currently allocated in percentage")
-    private String memoryAllocatedPercentage;
-
-    @SerializedName("memoryallocatedbytes")
-    @Param(description = "the amount of the host's memory currently allocated in bytes")
-    private Long memoryAllocatedBytes;
-
-    @SerializedName("memoryused")
-    @Param(description = "the amount of the host's memory currently used")
-    private Long memoryUsed;
-
-    @SerializedName("disksizetotal")
-    @Param(description = "the total disk size of the host")
-    private Long diskSizeTotal;
-
-    @SerializedName("disksizeallocated")
-    @Param(description = "the host's currently allocated disk size")
-    private Long diskSizeAllocated;
-
-    @SerializedName("capabilities")
-    @Param(description = "capabilities of the host")
-    private String capabilities;
-
-    @SerializedName("lastpinged")
-    @Param(description = "the date and time the host was last pinged")
-    private Date lastPinged;
-
-    @SerializedName("managementserverid")
-    @Param(description = "the management server ID of the host")
-    private Long managementServerId;
-
-    @SerializedName("clusterid")
-    @Param(description = "the cluster ID of the host")
-    private String clusterId;
-
-    @SerializedName("clustername")
-    @Param(description = "the cluster name of the host")
-    private String clusterName;
-
-    @SerializedName("clustertype")
-    @Param(description = "the cluster type of the cluster that host belongs to")
-    private String clusterType;
-
-    @SerializedName("islocalstorageactive")
-    @Param(description = "true if local storage is active, false otherwise")
-    private Boolean localStorageActive;
-
-    @SerializedName(ApiConstants.CREATED)
-    @Param(description = "the date and time the host was created")
-    private Date created;
-
-    @SerializedName("removed")
-    @Param(description = "the date and time the host was removed")
-    private Date removed;
-
-    @SerializedName("events")
-    @Param(description = "events available for the host")
-    private String events;
-
-    @SerializedName("hosttags")
-    @Param(description = "comma-separated list of tags for the host")
-    private String hostTags;
-
-    @SerializedName("explicithosttags")
-    @Param(description = "comma-separated list of explicit host tags for the host", since = "4.20.0")
-    private String explicitHostTags;
-
-    @SerializedName("implicithosttags")
-    @Param(description = "comma-separated list of implicit host tags for the host", since = "4.20.0")
-    private String implicitHostTags;
-
-    @SerializedName("hasenoughcapacity")
-    @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise")
-    private Boolean hasEnoughCapacity;
-
-    @SerializedName("suitableformigration")
-    @Param(description = "true if this host is suitable(has enough capacity and satisfies all conditions like hosttags, " +
-        "max guests vm limit etc) to migrate a VM to it , false otherwise")
-    private Boolean suitableForMigration;
+public class HostForMigrationResponse extends HostResponse {
 
     @SerializedName("requiresStorageMotion")
     @Param(description = "true if migrating a vm to this host requires storage motion, false otherwise")
     private Boolean requiresStorageMotion;
 
-    @SerializedName("resourcestate")
-    @Param(description = "the resource state of the host")
-    private String resourceState;
-
-    @SerializedName(ApiConstants.HYPERVISOR_VERSION)
-    @Param(description = "the hypervisor version")
-    private String hypervisorVersion;
-
-    @SerializedName(ApiConstants.HA_HOST)
-    @Param(description = "true if the host is Ha host (dedicated to vms started by HA process; false otherwise")
-    private Boolean haHost;
-
-    @Override
-    public String getObjectId() {
-        return getId();
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    public void setId(String id) {
-        this.id = id;
-    }
-
-    public void setName(String name) {
-        this.name = name;
-    }
-
-    public void setState(Status state) {
-        this.state = state;
-    }
-
-    public void setDisconnectedOn(Date disconnectedOn) {
-        this.disconnectedOn = disconnectedOn;
-    }
-
-    public void setHostType(Host.Type hostType) {
-        this.hostType = hostType;
-    }
-
-    public void setOsCategoryId(String osCategoryId) {
-        this.osCategoryId = osCategoryId;
-    }
-
-    public void setOsCategoryName(String osCategoryName) {
-        this.osCategoryName = osCategoryName;
-    }
-
-    public void setIpAddress(String ipAddress) {
-        this.ipAddress = ipAddress;
-    }
-
-    public void setZoneId(String zoneId) {
-        this.zoneId = zoneId;
-    }
-
-    public void setZoneName(String zoneName) {
-        this.zoneName = zoneName;
-    }
-
-    public void setPodId(String podId) {
-        this.podId = podId;
-    }
-
-    public void setPodName(String podName) {
-        this.podName = podName;
-    }
-
-    public void setVersion(String version) {
-        this.version = version;
-    }
-
-    public void setHypervisor(String hypervisor) {
-        this.hypervisor = hypervisor;
-    }
-
-    public void setCpuNumber(Integer cpuNumber) {
-        this.cpuNumber = cpuNumber;
-    }
-
-    public void setCpuSpeed(Long cpuSpeed) {
-        this.cpuSpeed = cpuSpeed;
-    }
-
-    public String getCpuAllocated() {
-        return cpuAllocated;
-    }
-
-    public void setCpuAllocated(String cpuAllocated) {
-        this.cpuAllocated = cpuAllocated;
-    }
-
-    public void setCpuAllocatedValue(Long cpuAllocatedValue) {
-        this.cpuAllocatedValue = cpuAllocatedValue;
-    }
-
-    public void setCpuAllocatedPercentage(String cpuAllocatedPercentage) {
-        this.cpuAllocatedPercentage = cpuAllocatedPercentage;
-    }
-
-    public void setCpuAllocatedWithOverprovisioning(String cpuAllocatedWithOverprovisioning) {
-        this.cpuAllocatedWithOverprovisioning = cpuAllocatedWithOverprovisioning;
-    }
-
-    public void setCpuUsed(String cpuUsed) {
-        this.cpuUsed = cpuUsed;
-    }
-
-    public void setAverageLoad(Long averageLoad) {
-        this.averageLoad = averageLoad;
-    }
-
-    public void setNetworkKbsRead(Long networkKbsRead) {
-        this.networkKbsRead = networkKbsRead;
-    }
-
-    public void setNetworkKbsWrite(Long networkKbsWrite) {
-        this.networkKbsWrite = networkKbsWrite;
-    }
-
-    public void setMemoryAllocated(String memoryAllocated) {
-        this.memoryAllocated = memoryAllocated;
-    }
-
-    public void setMemoryAllocatedPercentage(String memoryAllocatedPercentage) {
-        this.memoryAllocatedPercentage = memoryAllocatedPercentage;
-    }
-
-    public void setMemoryAllocatedBytes(Long memoryAllocatedBytes) {
-        this.memoryAllocatedBytes = memoryAllocatedBytes;
-    }
-
-    public void setMemoryUsed(Long memoryUsed) {
-        this.memoryUsed = memoryUsed;
-    }
-
-    public void setDiskSizeTotal(Long diskSizeTotal) {
-        this.diskSizeTotal = diskSizeTotal;
-    }
-
-    public void setDiskSizeAllocated(Long diskSizeAllocated) {
-        this.diskSizeAllocated = diskSizeAllocated;
-    }
-
-    public void setCapabilities(String capabilities) {
-        this.capabilities = capabilities;
-    }
-
-    public void setLastPinged(Date lastPinged) {
-        this.lastPinged = lastPinged;
-    }
-
-    public void setManagementServerId(Long managementServerId) {
-        this.managementServerId = managementServerId;
-    }
-
-    public void setClusterId(String clusterId) {
-        this.clusterId = clusterId;
-    }
-
-    public void setClusterName(String clusterName) {
-        this.clusterName = clusterName;
-    }
-
-    public void setClusterType(String clusterType) {
-        this.clusterType = clusterType;
-    }
-
-    public void setLocalStorageActive(Boolean localStorageActive) {
-        this.localStorageActive = localStorageActive;
-    }
-
-    public void setCreated(Date created) {
-        this.created = created;
-    }
-
-    public void setRemoved(Date removed) {
-        this.removed = removed;
-    }
-
-    public void setEvents(String events) {
-        this.events = events;
-    }
-
-    public String getHostTags() {
-        return hostTags;
-    }
-
-    public void setHostTags(String hostTags) {
-        this.hostTags = hostTags;
-    }
-
-    public void setExplicitHostTags(String explicitHostTags) {
-        this.explicitHostTags = explicitHostTags;
-    }
-
-    public void setImplicitHostTags(String implicitHostTags) {
-        this.implicitHostTags = implicitHostTags;
-    }
-
-    public void setHasEnoughCapacity(Boolean hasEnoughCapacity) {
-        this.hasEnoughCapacity = hasEnoughCapacity;
-    }
-
-    public void setSuitableForMigration(Boolean suitableForMigration) {
-        this.suitableForMigration = suitableForMigration;
-    }
-
     public void setRequiresStorageMotion(Boolean requiresStorageMotion) {
         this.requiresStorageMotion = requiresStorageMotion;
     }
-
-    public String getResourceState() {
-        return resourceState;
-    }
-
-    public void setResourceState(String resourceState) {
-        this.resourceState = resourceState;
-    }
-
-    public String getCpuWithOverprovisioning() {
-        return cpuWithOverprovisioning;
-    }
-
-    public void setCpuWithOverprovisioning(String cpuWithOverprovisioning) {
-        this.cpuWithOverprovisioning = cpuWithOverprovisioning;
-    }
-
-    public void setMemWithOverprovisioning(String memWithOverprovisioning){
-        this.memWithOverprovisioning=memWithOverprovisioning;
-    }
-
-    public void setHypervisorVersion(String hypervisorVersion) {
-        this.hypervisorVersion = hypervisorVersion;
-    }
-
-    public Boolean getHaHost() {
-        return haHost;
-    }
-
-    public void setHaHost(Boolean haHost) {
-        this.haHost = haHost;
-    }
-
-    public void setMemoryTotal(Long memoryTotal) {
-        this.memoryTotal = memoryTotal;
-    }
 }

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -186,10 +186,18 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "the date and time the host was last pinged")
     private Date lastPinged;
 
-    @SerializedName("managementserverid")
+    @SerializedName(ApiConstants.VIRTUAL_MACHINE_ID)
+    @Param(description = "the virtual machine id for host type ConsoleProxy and SecondaryStorageVM", since = "4.21.0")
+    private String virtualMachineId;
+
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID)
     @Param(description = "the management server ID of the host")
     private String managementServerId;
 
+    @SerializedName(ApiConstants.MANAGEMENT_SERVER_NAME)
+    @Param(description = "the management server name of the host", since = "4.21.0")
+    private String managementServerName;
+
     @SerializedName("clusterid")
     @Param(description = "the cluster ID of the host")
     private String clusterId;
@@ -435,10 +443,18 @@ public class HostResponse extends BaseResponseWithAnnotations {
         this.lastPinged = lastPinged;
     }
 
+    public void setVirtualMachineId(String virtualMachineId) {
+        this.virtualMachineId = virtualMachineId;
+    }
+
     public void setManagementServerId(String managementServerId) {
         this.managementServerId = managementServerId;
     }
 
+    public void setManagementServerName(String managementServerName) {
+        this.managementServerName = managementServerName;
+    }
+
     public void setClusterId(String clusterId) {
         this.clusterId = clusterId;
     }
@@ -723,10 +739,18 @@ public class HostResponse extends BaseResponseWithAnnotations {
         return lastPinged;
     }
 
+    public String getVirtualMachineId() {
+        return virtualMachineId;
+    }
+
     public String getManagementServerId() {
         return managementServerId;
     }
 
+    public String getManagementServerName() {
+        return managementServerName;
+    }
+
     public String getClusterId() {
         return clusterId;
     }

api/src/main/java/org/apache/cloudstack/api/response/LoginCmdResponse.java

@@ -86,6 +86,10 @@ public class LoginCmdResponse extends AuthenticationCmdResponse {
     @Param(description = "Two factor authentication issuer", since = "4.18.0.0")
     private String issuerFor2FA;
 
+    @SerializedName(value = ApiConstants.MANAGEMENT_SERVER_ID)
+    @Param(description = "Management Server ID that the user logged to", since = "4.21.0.0")
+    private String managementServerId;
+
     public String getUsername() {
         return username;
     }
@@ -211,4 +215,12 @@ public class LoginCmdResponse extends AuthenticationCmdResponse {
     public void setIssuerFor2FA(String issuerFor2FA) {
         this.issuerFor2FA = issuerFor2FA;
     }
+
+    public String getManagementServerId() {
+        return managementServerId;
+    }
+
+    public void setManagementServerId(String managementServerId) {
+        this.managementServerId = managementServerId;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java

@@ -82,6 +82,14 @@ public class ManagementServerResponse extends BaseResponse {
     @Param(description = "the Management Server Peers")
     private List<PeerManagementServerNodeResponse> peers;
 
+    @SerializedName(ApiConstants.AGENTS_COUNT)
+    @Param(description = "the number of host agents this Management Server is responsible for", since = "4.21.0.0")
+    private Long agentsCount;
+
+    @SerializedName(ApiConstants.PENDING_JOBS_COUNT)
+    @Param(description = "the number of pending jobs in this Management Server", since = "4.21.0.0")
+    private Long pendingJobsCount;
+
     public String getId() {
         return this.id;
     }
@@ -126,6 +134,14 @@ public class ManagementServerResponse extends BaseResponse {
         return serviceIp;
     }
 
+    public Long getAgentsCount() {
+        return this.agentsCount;
+    }
+
+    public Long getPendingJobsCount() {
+        return this.pendingJobsCount;
+    }
+
     public void setId(String id) {
         this.id = id;
     }
@@ -174,6 +190,14 @@ public class ManagementServerResponse extends BaseResponse {
         this.serviceIp = serviceIp;
     }
 
+    public void setAgentsCount(Long agentsCount) {
+        this.agentsCount = agentsCount;
+    }
+
+    public void setPendingJobsCount(Long pendingJobsCount) {
+        this.pendingJobsCount = pendingJobsCount;
+    }
+
     public String getKernelVersion() {
         return kernelVersion;
     }

api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java

@@ -196,6 +196,10 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement
     @Param(description = "true network requires restart")
     private Boolean restartRequired;
 
+    @SerializedName(ApiConstants.SPECIFY_VLAN)
+    @Param(description = "true if network supports specifying vlan, false otherwise")
+    private Boolean specifyVlan;
+
     @SerializedName(ApiConstants.SPECIFY_IP_RANGES)
     @Param(description = "true if network supports specifying ip ranges, false otherwise")
     private Boolean specifyIpRanges;
@@ -516,6 +520,10 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement
         this.restartRequired = restartRequired;
     }
 
+    public void setSpecifyVlan(Boolean specifyVlan) {
+        this.specifyVlan = specifyVlan;
+    }
+
     public void setSpecifyIpRanges(Boolean specifyIpRanges) {
         this.specifyIpRanges = specifyIpRanges;
     }

api/src/main/java/org/apache/cloudstack/management/ManagementServerHost.java

@@ -22,7 +22,7 @@ import org.apache.cloudstack.api.InternalIdentity;
 
 public interface ManagementServerHost extends InternalIdentity, Identity, ControlledEntity {
     enum State {
-        Up, Down, PreparingToShutDown, ReadyToShutDown, ShuttingDown
+        Up, Down, PreparingForMaintenance, Maintenance, PreparingForShutDown, ReadyToShutDown, ShuttingDown
     }
 
     long getMsid();

api/src/main/java/org/apache/cloudstack/outofbandmanagement/OutOfBandManagementService.java

@@ -39,7 +39,7 @@ public interface OutOfBandManagementService {
     long getId();
     boolean isOutOfBandManagementEnabled(Host host);
     void submitBackgroundPowerSyncTask(Host host);
-    boolean transitionPowerStateToDisabled(List<? extends Host> hosts);
+    boolean transitionPowerStateToDisabled(List<Long> hostIds);
 
     OutOfBandManagementResponse enableOutOfBandManagement(DataCenter zone);
     OutOfBandManagementResponse enableOutOfBandManagement(Cluster cluster);

api/src/test/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmdTest.java

@@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.admin.domain;
 
 import java.util.List;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.junit.Assert;
 import org.junit.Test;
@@ -71,7 +72,17 @@ public class ListDomainsCmdTest {
         cmd._resourceLimitService = resourceLimitService;
         ReflectionTestUtils.setField(cmd, "tag", "abc");
         cmd.updateDomainResponse(List.of(Mockito.mock(DomainResponse.class)));
-        Mockito.verify(resourceLimitService, Mockito.times(1)).updateTaggedResourceLimitsAndCountsForDomains(Mockito.any(), Mockito.any());
+        Mockito.verify(resourceLimitService).updateTaggedResourceLimitsAndCountsForDomains(Mockito.any(), Mockito.any());
+    }
+
+    @Test
+    public void testUpdateDomainResponseWithDomainsMinDetails() {
+        ListDomainsCmd cmd = new ListDomainsCmd();
+        ReflectionTestUtils.setField(cmd, "viewDetails", List.of(ApiConstants.DomainDetails.min.toString()));
+        cmd._resourceLimitService = resourceLimitService;
+        ReflectionTestUtils.setField(cmd, "tag", "abc");
+        cmd.updateDomainResponse(List.of(Mockito.mock(DomainResponse.class)));
+        Mockito.verify(resourceLimitService, Mockito.never()).updateTaggedResourceLimitsAndCountsForDomains(Mockito.any(), Mockito.any());
     }
 
 }

api/src/test/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmdTest.java

@@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.user.account;
 
 import java.util.List;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.response.AccountResponse;
 import org.junit.Assert;
 import org.junit.Test;
@@ -58,7 +59,7 @@ public class ListAccountsCmdTest {
     }
 
     @Test
-    public void testUpdateDomainResponseNoDomains() {
+    public void testUpdateAccountResponseNoAccounts() {
         ListAccountsCmd cmd = new ListAccountsCmd();
         cmd._resourceLimitService = resourceLimitService;
         cmd.updateAccountResponse(null);
@@ -66,11 +67,21 @@ public class ListAccountsCmdTest {
     }
 
     @Test
-    public void testUpdateDomainResponseWithDomains() {
+    public void testUpdateDomainResponseWithAccounts() {
         ListAccountsCmd cmd = new ListAccountsCmd();
         cmd._resourceLimitService = resourceLimitService;
         ReflectionTestUtils.setField(cmd, "tag", "abc");
         cmd.updateAccountResponse(List.of(Mockito.mock(AccountResponse.class)));
         Mockito.verify(resourceLimitService, Mockito.times(1)).updateTaggedResourceLimitsAndCountsForAccounts(Mockito.any(), Mockito.any());
     }
+
+    @Test
+    public void testUpdateDomainResponseWithAccountsMinDetails() {
+        ListAccountsCmd cmd = new ListAccountsCmd();
+        ReflectionTestUtils.setField(cmd, "viewDetails", List.of(ApiConstants.DomainDetails.min.toString()));
+        cmd._resourceLimitService = resourceLimitService;
+        ReflectionTestUtils.setField(cmd, "tag", "abc");
+        cmd.updateAccountResponse(List.of(Mockito.mock(AccountResponse.class)));
+        Mockito.verify(resourceLimitService, Mockito.never()).updateTaggedResourceLimitsAndCountsForAccounts(Mockito.any(), Mockito.any());
+    }
 }

api/src/test/java/org/apache/cloudstack/api/command/user/firewall/CreateFirewallRuleCmdTest.java

@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.firewall;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import com.cloud.utils.net.NetUtils;
+
+@RunWith(MockitoJUnitRunner.class)
+public class CreateFirewallRuleCmdTest {
+
+    private void validateAllIp4Cidr(final CreateFirewallRuleCmd cmd) {
+        Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
+        Assert.assertEquals(1, cmd.getSourceCidrList().size());
+        Assert.assertEquals(NetUtils.ALL_IP4_CIDRS, cmd.getSourceCidrList().get(0));
+    }
+
+    @Test
+    public void testGetSourceCidrList_Null() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        ReflectionTestUtils.setField(cmd, "cidrlist", null);
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_Empty() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        ReflectionTestUtils.setField(cmd, "cidrlist", new ArrayList<>());
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_NullFirstElement() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        List<String> list = new ArrayList<>();
+        list.add(null);
+        ReflectionTestUtils.setField(cmd, "cidrlist", list);
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_EmptyFirstElement() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        ReflectionTestUtils.setField(cmd, "cidrlist", Collections.singletonList("  "));
+        validateAllIp4Cidr(cmd);
+    }
+
+    @Test
+    public void testGetSourceCidrList_Valid() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        String cidr = "10.1.1.1/22";
+        ReflectionTestUtils.setField(cmd, "cidrlist", Collections.singletonList(cidr));
+        Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
+        Assert.assertEquals(1, cmd.getSourceCidrList().size());
+        Assert.assertEquals(cidr, cmd.getSourceCidrList().get(0));
+    }
+
+    @Test
+    public void testGetSourceCidrList_EmptyFirstElementButMore() {
+        final CreateFirewallRuleCmd cmd = new CreateFirewallRuleCmd();
+        String cidr = "10.1.1.1/22";
+        ReflectionTestUtils.setField(cmd, "cidrlist", Arrays.asList("  ", cidr));
+        Assert.assertTrue(CollectionUtils.isNotEmpty(cmd.getSourceCidrList()));
+        Assert.assertEquals(2, cmd.getSourceCidrList().size());
+        Assert.assertEquals(cidr, cmd.getSourceCidrList().get(1));
+    }
+}

client/conf/server.properties.in

@@ -32,6 +32,9 @@ session.timeout=30
 # Max allowed API request payload/content size in bytes
 request.content.size=1048576
 
+# Max allowed API request form keys
+request.max.form.keys=5000
+
 # Options to configure and enable HTTPS on the management server
 #
 # For the management server to pick up these configuration settings, the configured

client/pom.xml

@@ -624,7 +624,7 @@
         </dependency>
         <dependency>
            <groupId>org.apache.cloudstack</groupId>
-           <artifactId>cloud-plugin-shutdown</artifactId>
+           <artifactId>cloud-plugin-maintenance</artifactId>
            <version>${project.version}</version>
        </dependency>
         <dependency>

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -82,6 +82,8 @@ public class ServerDaemon implements Daemon {
     private static final String ACCESS_LOG = "access.log";
     private static final String REQUEST_CONTENT_SIZE_KEY = "request.content.size";
     private static final int DEFAULT_REQUEST_CONTENT_SIZE = 1048576;
+    private static final String REQUEST_MAX_FORM_KEYS_KEY = "request.max.form.keys";
+    private static final int DEFAULT_REQUEST_MAX_FORM_KEYS = 5000;
 
     ////////////////////////////////////////////////////////
     /////////////// Server Configuration ///////////////////
@@ -94,6 +96,7 @@ public class ServerDaemon implements Daemon {
     private int httpsPort = 8443;
     private int sessionTimeout = 30;
     private int maxFormContentSize = DEFAULT_REQUEST_CONTENT_SIZE;
+    private int maxFormKeys = DEFAULT_REQUEST_MAX_FORM_KEYS;
     private boolean httpsEnable = false;
     private String accessLogFile = "access.log";
     private String bindInterface = null;
@@ -141,6 +144,7 @@ public class ServerDaemon implements Daemon {
             setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log"));
             setSessionTimeout(Integer.valueOf(properties.getProperty(SESSION_TIMEOUT, "30")));
             setMaxFormContentSize(Integer.valueOf(properties.getProperty(REQUEST_CONTENT_SIZE_KEY, String.valueOf(DEFAULT_REQUEST_CONTENT_SIZE))));
+            setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS))));
         } catch (final IOException e) {
             logger.warn("Failed to read configuration from server.properties file", e);
         } finally {
@@ -192,6 +196,7 @@ public class ServerDaemon implements Daemon {
         // Extra config options
         server.setStopAtShutdown(true);
         server.setAttribute(ContextHandler.MAX_FORM_CONTENT_SIZE_KEY, maxFormContentSize);
+        server.setAttribute(ContextHandler.MAX_FORM_KEYS_KEY, maxFormKeys);
 
         // HTTPS Connector
         createHttpsConnector(httpConfig);
@@ -264,6 +269,7 @@ public class ServerDaemon implements Daemon {
         webApp.setContextPath(contextPath);
         webApp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
         webApp.setMaxFormContentSize(maxFormContentSize);
+        webApp.setMaxFormKeys(maxFormKeys);
 
         // GZIP handler
         final GzipHandler gzipHandler = new GzipHandler();
@@ -366,4 +372,8 @@ public class ServerDaemon implements Daemon {
     public void setMaxFormContentSize(int maxFormContentSize) {
         this.maxFormContentSize = maxFormContentSize;
     }
+
+    public void setMaxFormKeys(int maxFormKeys) {
+        this.maxFormKeys = maxFormKeys;
+    }
 }

core/src/main/java/com/cloud/agent/api/MigrateAgentConnectionAnswer.java

@@ -0,0 +1,38 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+public class MigrateAgentConnectionAnswer extends Answer {
+    public MigrateAgentConnectionAnswer() {
+    }
+
+    public MigrateAgentConnectionAnswer(boolean result) {
+        this.result = result;
+    }
+
+    public MigrateAgentConnectionAnswer(String details) {
+        this.result = false;
+        this.details = details;
+    }
+
+    public MigrateAgentConnectionAnswer(MigrateAgentConnectionCommand cmd, boolean result) {
+        super(cmd, result, null);
+    }
+}

core/src/main/java/com/cloud/agent/api/MigrateAgentConnectionCommand.java

@@ -0,0 +1,61 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+package com.cloud.agent.api;
+
+import java.util.List;
+
+public class MigrateAgentConnectionCommand extends Command {
+    private List<String> msList;
+    private List<String> avoidMsList;
+    private String lbAlgorithm;
+    private Long lbCheckInterval;
+
+    public MigrateAgentConnectionCommand() {
+    }
+
+    public MigrateAgentConnectionCommand(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) {
+        super();
+        this.msList = msList;
+        this.avoidMsList = avoidMsList;
+        this.lbAlgorithm = lbAlgorithm;
+        this.lbCheckInterval = lbCheckInterval;
+    }
+
+    public List<String> getMsList() {
+        return msList;
+    }
+
+    public List<String> getAvoidMsList() {
+        return avoidMsList;
+    }
+
+    public String getLbAlgorithm() {
+        return lbAlgorithm;
+    }
+
+    public Long getLbCheckInterval() {
+        return lbCheckInterval;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}

core/src/main/java/com/cloud/agent/api/StartupCommand.java

@@ -47,6 +47,7 @@ public class StartupCommand extends Command {
     String resourceName;
     String gatewayIpAddress;
     String msHostList;
+    boolean connectionTransferred;
     String arch;
 
     public StartupCommand(Host.Type type) {
@@ -291,6 +292,14 @@ public class StartupCommand extends Command {
         this.msHostList = msHostList;
     }
 
+    public boolean isConnectionTransferred() {
+        return connectionTransferred;
+    }
+
+    public void setConnectionTransferred(boolean connectionTransferred) {
+        this.connectionTransferred = connectionTransferred;
+    }
+
     public String getArch() {
         return arch;
     }

core/src/main/java/com/cloud/agent/api/TransferAgentCommand.java

@@ -25,6 +25,7 @@ public class TransferAgentCommand extends Command {
     protected long agentId;
     protected long futureOwner;
     protected long currentOwner;
+    protected boolean isConnectionTransfer;
     Event event;
 
     protected TransferAgentCommand() {
@@ -37,6 +38,11 @@ public class TransferAgentCommand extends Command {
         this.event = event;
     }
 
+    public TransferAgentCommand(long agentId, long currentOwner, long futureOwner, Event event, boolean isConnectionTransfer) {
+        this(agentId, currentOwner, futureOwner, event);
+        this.isConnectionTransfer = isConnectionTransfer;
+    }
+
     public long getAgentId() {
         return agentId;
     }
@@ -53,6 +59,10 @@ public class TransferAgentCommand extends Command {
         return currentOwner;
     }
 
+    public boolean isConnectionTransfer() {
+        return isConnectionTransfer;
+    }
+
     @Override
     public boolean executeInSequence() {
         return false;

core/src/main/java/com/cloud/resource/ServerResource.java

@@ -50,6 +50,10 @@ public interface ServerResource extends Manager {
      */
     StartupCommand[] initialize();
 
+    default StartupCommand[] initialize(boolean isTransferredConnection) {
+        return initialize();
+    }
+
     /**
      * @param id id of the server to put in the PingCommand
      * @return PingCommand
@@ -78,4 +82,12 @@ public interface ServerResource extends Manager {
 
     void setAgentControl(IAgentControl agentControl);
 
+    default boolean isExitOnFailures() {
+        return true;
+    }
+
+    default boolean isAppendAgentNameToLogs() {
+        return false;
+    }
+
 }

core/src/test/java/org/apache/cloudstack/api/agent/test/CheckOnHostCommandTest.java

@@ -189,6 +189,11 @@ public class CheckOnHostCommandTest {
             return 2L;
         };
 
+        @Override
+        public Long getLastManagementServerId() {
+            return null;
+        };
+
         @Override
         public Date getRemoved() {
             Date date = null;

engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java

@@ -22,7 +22,6 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.exception.ResourceAllocationException;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
@@ -38,6 +37,7 @@ import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.InsufficientServerCapacityException;
 import com.cloud.exception.OperationTimedoutException;
+import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.host.Host;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -101,6 +101,10 @@ public interface VirtualMachineManager extends Manager {
                     "refer documentation",
             true, ConfigKey.Scope.Zone);
 
+    ConfigKey<Boolean> VmSyncPowerStateTransitioning = new ConfigKey<>("Advanced", Boolean.class, "vm.sync.power.state.transitioning", "true",
+            "Whether to sync power states of the transitioning and stalled VMs while processing VM power reports.", false);
+
+
     interface Topics {
         String VM_POWER_STATE = "vm.powerstate";
     }
@@ -286,24 +290,22 @@ public interface VirtualMachineManager extends Manager {
 
     /**
      * Obtains statistics for a list of VMs; CPU and network utilization
-     * @param hostId ID of the host
-     * @param hostName name of the host
+     * @param host host
      * @param vmIds list of VM IDs
      * @return map of VM ID and stats entry for the VM
      */
-    HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, List<Long> vmIds);
+    HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, List<Long> vmIds);
     /**
      * Obtains statistics for a list of VMs; CPU and network utilization
-     * @param hostId ID of the host
-     * @param hostName name of the host
-     * @param vmMap map of VM IDs and the corresponding VirtualMachine object
+     * @param host host
+     * @param vmMap map of VM instanceName and its ID
      * @return map of VM ID and stats entry for the VM
      */
-    HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
+    HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, Map<String, Long> vmMap);
 
-    HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
+    HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(Host host, Map<String, Long> vmInstanceNameIdMap);
 
-    HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap);
+    HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(Host host, Map<String, Long> vmInstanceNameIdMap);
 
     Map<Long, Boolean> getDiskOfferingSuitabilityForVm(long vmId, List<Long> diskOfferingIds);
 

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java

@@ -82,6 +82,9 @@ public interface NetworkOrchestrationService {
     ConfigKey<Integer> NetworkLockTimeout = new ConfigKey<Integer>(Integer.class, NetworkLockTimeoutCK, "Network", "600",
         "Lock wait timeout (seconds) while implementing network", true, Scope.Global, null);
 
+    ConfigKey<String> DeniedRoutes = new ConfigKey<String>(String.class, "denied.routes", "Network", "",
+            "Routes that are denied, can not be used for Static Routes creation for the VPC Private Gateway", true, ConfigKey.Scope.Zone, null);
+
     ConfigKey<String> GuestDomainSuffix = new ConfigKey<String>(String.class, GuestDomainSuffixCK, "Network", "cloud.internal",
         "Default domain name for vms inside virtualized networks fronted by router", true, ConfigKey.Scope.Zone, null);
 

engine/components-api/src/main/java/com/cloud/agent/AgentManager.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.agent;
 
+import java.util.List;
 import java.util.Map;
 
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -170,4 +171,10 @@ public interface AgentManager {
     void notifyMonitorsOfRemovedHost(long hostId, long clusterId);
 
     void propagateChangeToAgents(Map<String, String> params);
+
+    boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs);
+
+    List<String> getLastAgents();
+
+    void setLastAgents(List<String> lastAgents);
 }

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -16,14 +16,11 @@
 // under the License.
 package com.cloud.capacity;
 
-import java.util.Map;
-
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
 import com.cloud.host.Host;
 import com.cloud.offering.ServiceOffering;
-import com.cloud.service.ServiceOfferingVO;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.utils.Pair;
 import com.cloud.vm.VirtualMachine;
@@ -130,6 +127,10 @@ public interface CapacityManager {
                     true,
                     ConfigKey.Scope.Zone);
 
+    ConfigKey<Integer> CapacityCalculateWorkers = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class,
+            "capacity.calculate.workers", "1",
+            "Number of worker threads to be used for capacities calculation", true);
+
     public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId);
 
     void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost);
@@ -145,8 +146,6 @@ public interface CapacityManager {
 
     void updateCapacityForHost(Host host);
 
-    void updateCapacityForHost(Host host, Map<Long, ServiceOfferingVO> offeringsMap);
-
     /**
      * @param pool storage pool
      * @param templateForVmCreation template that will be used for vm creation
@@ -163,12 +162,12 @@ public interface CapacityManager {
 
     /**
      * Check if specified host has capability to support cpu cores and speed freq
-     * @param hostId the host to be checked
+     * @param host the host to be checked
      * @param cpuNum cpu number to check
      * @param cpuSpeed cpu Speed to check
      * @return true if the count of host's running VMs >= hypervisor limit
      */
-    boolean checkIfHostHasCpuCapability(long hostId, Integer cpuNum, Integer cpuSpeed);
+    boolean checkIfHostHasCpuCapability(Host host, Integer cpuNum, Integer cpuSpeed);
 
     /**
      * Check if cluster will cross threshold if the cpu/memory requested are accommodated

engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java

@@ -84,6 +84,13 @@ public interface HighAvailabilityManager extends Manager {
         HA;         // Restart a VM.
     }
 
+    enum ReasonType {
+        Unknown,
+        HostMaintenance,
+        HostDown,
+        HostDegraded;
+    }
+
     enum Step {
         Scheduled, Investigating, Fencing, Stopping, Restarting, Migrating, Cancelled, Done, Error,
     }
@@ -92,7 +99,7 @@ public interface HighAvailabilityManager extends Manager {
      * Investigate why a host has disconnected and migrate the VMs on it
      * if necessary.
      *
-     * @param host - the host that has disconnected.
+     * @param hostId - the id of the host that has disconnected.
      */
     Status investigate(long hostId);
 
@@ -109,17 +116,19 @@ public interface HighAvailabilityManager extends Manager {
      * @param investigate must be investigated before we do anything with this vm.
      */
     void scheduleRestart(VMInstanceVO vm, boolean investigate);
+    void scheduleRestart(VMInstanceVO vm, boolean investigate, ReasonType reasonType);
 
     void cancelDestroy(VMInstanceVO vm, Long hostId);
 
-    boolean scheduleDestroy(VMInstanceVO vm, long hostId);
+    boolean scheduleDestroy(VMInstanceVO vm, long hostId, ReasonType reasonType);
 
     /**
      * Schedule restarts for all vms running on the host.
      * @param host host.
-     * @param investigate TODO
+     * @param investigate whether to investigate
+     * @param reasonType reason for HA work
      */
-    void scheduleRestartForVmsOnHost(HostVO host, boolean investigate);
+    void scheduleRestartForVmsOnHost(HostVO host, boolean investigate, ReasonType reasonType);
 
     /**
      * Schedule the vm for migration.
@@ -128,6 +137,7 @@ public interface HighAvailabilityManager extends Manager {
      * @return true if schedule worked.
      */
     boolean scheduleMigration(VMInstanceVO vm);
+    boolean scheduleMigration(VMInstanceVO vm, ReasonType reasonType);
 
     List<VMInstanceVO> findTakenMigrationWork();
 
@@ -140,10 +150,11 @@ public interface HighAvailabilityManager extends Manager {
      * 3. Check if a VM has been stopped: WorkType.CheckStop
      *
      * @param vm virtual machine to stop.
-     * @param host host the virtual machine is on.
+     * @param hostId the id of the host the virtual machine is on.
      * @param type which type of stop is requested.
      */
     boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type);
+    boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type, ReasonType reasonType);
 
     void cancelScheduledMigrations(HostVO host);
 

engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java

@@ -85,6 +85,8 @@ public interface ResourceManager extends ResourceService, Configurable {
 
     public Host createHostAndAgent(Long hostId, ServerResource resource, Map<String, String> details, boolean old, List<String> hostTags, boolean forRebalance);
 
+    public Host createHostAndAgent(Long hostId, ServerResource resource, Map<String, String> details, boolean old, List<String> hostTags, boolean forRebalance, boolean isTransferredConnection);
+
     public Host addHost(long zoneId, ServerResource resource, Type hostType, Map<String, String> hostDetails);
 
     public HostVO createHostVOForConnectedAgent(StartupCommand[] cmds);
@@ -138,13 +140,13 @@ public interface ResourceManager extends ResourceService, Configurable {
 
     public List<HostVO> listAllHostsInOneZoneNotInClusterByHypervisors(List<HypervisorType> types, long dcId, long clusterId);
 
-    public List<HypervisorType> listAvailHypervisorInZone(Long hostId, Long zoneId);
+    public List<HypervisorType> listAvailHypervisorInZone(Long zoneId);
 
     public HostVO findHostByGuid(String guid);
 
     public HostVO findHostByName(String name);
 
-    HostStats getHostStatistics(long hostId);
+    HostStats getHostStatistics(Host host);
 
     Long getGuestOSCategoryId(long hostId);
 

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -22,6 +22,7 @@ import java.util.Map;
 
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
+import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 
@@ -42,6 +43,7 @@ import com.cloud.offering.DiskOffering;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.utils.Pair;
+import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.DiskProfile;
 import com.cloud.vm.VMInstanceVO;
 
@@ -214,6 +216,10 @@ public interface StorageManager extends StorageService {
                     "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)",
             true, ConfigKey.Scope.Zone);
 
+    ConfigKey<Integer> StoragePoolHostConnectWorkers = new ConfigKey<>("Storage", Integer.class,
+            "storage.pool.host.connect.workers", "1",
+            "Number of worker threads to be used to connect hosts to a primary storage", true);
+
     /**
      * should we execute in sequence not involving any storages?
      * @return tru if commands should execute in sequence
@@ -365,6 +371,9 @@ public interface StorageManager extends StorageService {
 
     String getStoragePoolMountFailureReason(String error);
 
+    void connectHostsToPool(DataStore primaryStore, List<Long> hostIds, Scope scope,
+            boolean handleStorageConflictException, boolean errorOnNoUpHost) throws CloudRuntimeException;
+
     boolean connectHostToSharedPool(Host host, long poolId) throws StorageUnavailableException, StorageConflictException;
 
     void disconnectHostFromSharedPool(Host host, StoragePool pool) throws StorageUnavailableException, StorageConflictException;

engine/orchestration/pom.xml

@@ -70,7 +70,7 @@
         </dependency>
         <dependency>
             <groupId>org.apache.cloudstack</groupId>
-            <artifactId>cloud-plugin-shutdown</artifactId>
+            <artifactId>cloud-plugin-maintenance</artifactId>
             <version>${project.version}</version>
         </dependency>
     </dependencies>

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -16,8 +16,10 @@
 // under the License.
 package com.cloud.agent.manager;
 
+import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.net.SocketAddress;
 import java.nio.channels.ClosedChannelException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -25,23 +27,20 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.configuration.Config;
-import com.cloud.org.Cluster;
-import com.cloud.utils.NumbersUtil;
-import com.cloud.utils.db.GlobalLock;
 import org.apache.cloudstack.agent.lb.IndirectAgentLB;
 import org.apache.cloudstack.ca.CAManager;
 import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
@@ -50,11 +49,17 @@ import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.framework.jobs.AsyncJob;
 import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext;
+import org.apache.cloudstack.maintenance.ManagementServerMaintenanceListener;
+import org.apache.cloudstack.maintenance.ManagementServerMaintenanceManager;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
+import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.commons.collections.MapUtils;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.ThreadContext;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.Listener;
@@ -81,6 +86,9 @@ import com.cloud.agent.api.UnsupportedAnswer;
 import com.cloud.agent.transport.Request;
 import com.cloud.agent.transport.Response;
 import com.cloud.alert.AlertManager;
+import com.cloud.cluster.ManagementServerHostVO;
+import com.cloud.cluster.dao.ManagementServerHostDao;
+import com.cloud.configuration.Config;
 import com.cloud.configuration.ManagementServiceConfiguration;
 import com.cloud.dc.ClusterVO;
 import com.cloud.dc.DataCenterVO;
@@ -100,15 +108,18 @@ import com.cloud.host.Status.Event;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.hypervisor.HypervisorGuruManager;
+import com.cloud.org.Cluster;
 import com.cloud.resource.Discoverer;
 import com.cloud.resource.ResourceManager;
 import com.cloud.resource.ResourceState;
 import com.cloud.resource.ServerResource;
+import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.EntityManager;
+import com.cloud.utils.db.GlobalLock;
 import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchCriteria.Op;
 import com.cloud.utils.db.TransactionLegacy;
@@ -123,26 +134,23 @@ import com.cloud.utils.nio.Link;
 import com.cloud.utils.nio.NioServer;
 import com.cloud.utils.nio.Task;
 import com.cloud.utils.time.InaccurateClock;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.ThreadContext;
 
 /**
  * Implementation of the Agent Manager. This class controls the connection to the agents.
  **/
-public class AgentManagerImpl extends ManagerBase implements AgentManager, HandlerFactory, Configurable {
+public class AgentManagerImpl extends ManagerBase implements AgentManager, HandlerFactory, ManagementServerMaintenanceListener, Configurable {
 
     /**
      * _agents is a ConcurrentHashMap, but it is used from within a synchronized block. This will be reported by findbugs as JLM_JSR166_UTILCONCURRENT_MONITORENTER. Maybe a
      * ConcurrentHashMap is not the right thing to use here, but i'm not sure so i leave it alone.
      */
-    protected ConcurrentHashMap<Long, AgentAttache> _agents = new ConcurrentHashMap<Long, AgentAttache>(10007);
-    protected List<Pair<Integer, Listener>> _hostMonitors = new ArrayList<Pair<Integer, Listener>>(17);
-    protected List<Pair<Integer, Listener>> _cmdMonitors = new ArrayList<Pair<Integer, Listener>>(17);
-    protected List<Pair<Integer, StartupCommandProcessor>> _creationMonitors = new ArrayList<Pair<Integer, StartupCommandProcessor>>(17);
-    protected List<Long> _loadingAgents = new ArrayList<Long>();
+    protected ConcurrentHashMap<Long, AgentAttache> _agents = new ConcurrentHashMap<>(10007);
+    protected List<Pair<Integer, Listener>> _hostMonitors = new ArrayList<>(17);
+    protected List<Pair<Integer, Listener>> _cmdMonitors = new ArrayList<>(17);
+    protected List<Pair<Integer, StartupCommandProcessor>> _creationMonitors = new ArrayList<>(17);
+    protected List<Long> _loadingAgents = new ArrayList<>();
     protected Map<String, Integer> _commandTimeouts = new HashMap<>();
     private int _monitorId = 0;
-    private final Lock _agentStatusLock = new ReentrantLock();
 
     @Inject
     protected CAManager caService;
@@ -153,6 +161,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     @Inject
     protected HostDao _hostDao = null;
     @Inject
+    private ManagementServerHostDao _mshostDao;
+    @Inject
     protected OutOfBandManagementDao outOfBandManagementDao;
     @Inject
     protected DataCenterDao _dcDao = null;
@@ -174,6 +184,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     @Inject
     protected IndirectAgentLB indirectAgentLB;
 
+    @Inject
+    private ManagementServerMaintenanceManager managementServerMaintenanceManager;
+
     protected int _retry = 2;
 
     protected long _nodeId = -1;
@@ -186,26 +199,39 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
     private int _directAgentThreadCap;
 
+    private List<String> lastAgents = null;
+
     protected StateMachine2<Status, Status.Event, Host> _statusStateMachine = Status.getStateMachine();
-    private final ConcurrentHashMap<Long, Long> _pingMap = new ConcurrentHashMap<Long, Long>(10007);
+    private final ConcurrentHashMap<Long, Long> _pingMap = new ConcurrentHashMap<>(10007);
+    private int maxConcurrentNewAgentConnections;
+    private final ConcurrentHashMap<String, Long> newAgentConnections = new ConcurrentHashMap<>();
+    protected ScheduledExecutorService newAgentConnectionsMonitor;
 
     @Inject
     ResourceManager _resourceMgr;
     @Inject
     ManagementServiceConfiguration mgmtServiceConf;
 
-    protected final ConfigKey<Integer> Workers = new ConfigKey<Integer>("Advanced", Integer.class, "workers", "5",
+    protected final ConfigKey<Integer> Workers = new ConfigKey<>("Advanced", Integer.class, "workers", "5",
             "Number of worker threads handling remote agent connections.", false);
-    protected final ConfigKey<Integer> Port = new ConfigKey<Integer>("Advanced", Integer.class, "port", "8250", "Port to listen on for remote agent connections.", false);
-    protected final ConfigKey<Integer> AlertWait = new ConfigKey<Integer>("Advanced", Integer.class, "alert.wait", "1800",
+    protected final ConfigKey<Integer> Port = new ConfigKey<>("Advanced", Integer.class, "port", "8250", "Port to listen on for remote agent connections.", false);
+    protected final ConfigKey<Integer> RemoteAgentSslHandshakeTimeout = new ConfigKey<>("Advanced",
+            Integer.class, "agent.ssl.handshake.timeout", "30",
+            "Seconds after which SSL handshake times out during remote agent connections.", false);
+    protected final ConfigKey<Integer> RemoteAgentMaxConcurrentNewConnections = new ConfigKey<>("Advanced",
+            Integer.class, "agent.max.concurrent.new.connections", "0",
+            "Number of maximum concurrent new connections server allows for remote agents. " +
+                    "If set to zero (default value) then no limit will be enforced on concurrent new connections",
+            false);
+    protected final ConfigKey<Integer> AlertWait = new ConfigKey<>("Advanced", Integer.class, "alert.wait", "1800",
             "Seconds to wait before alerting on a disconnected agent", true);
-    protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<Integer>("Advanced", Integer.class, "direct.agent.load.size", "16",
+    protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<>("Advanced", Integer.class, "direct.agent.load.size", "16",
             "The number of direct agents to load each time", false);
-    protected final ConfigKey<Integer> DirectAgentPoolSize = new ConfigKey<Integer>("Advanced", Integer.class, "direct.agent.pool.size", "500",
+    protected final ConfigKey<Integer> DirectAgentPoolSize = new ConfigKey<>("Advanced", Integer.class, "direct.agent.pool.size", "500",
             "Default size for DirectAgentPool", false);
-    protected final ConfigKey<Float> DirectAgentThreadCap = new ConfigKey<Float>("Advanced", Float.class, "direct.agent.thread.cap", "1",
+    protected final ConfigKey<Float> DirectAgentThreadCap = new ConfigKey<>("Advanced", Float.class, "direct.agent.thread.cap", "1",
             "Percentage (as a value between 0 and 1) of direct.agent.pool.size to be used as upper thread cap for a single direct agent to process requests", false);
-    protected final ConfigKey<Boolean> CheckTxnBeforeSending = new ConfigKey<Boolean>("Developer", Boolean.class, "check.txn.before.sending.agent.commands", "false",
+    protected final ConfigKey<Boolean> CheckTxnBeforeSending = new ConfigKey<>("Developer", Boolean.class, "check.txn.before.sending.agent.commands", "false",
             "This parameter allows developers to enable a check to see if a transaction wraps commands that are sent to the resource.  This is not to be enabled on production systems.", true);
 
     @Override
@@ -213,8 +239,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
         logger.info("Ping Timeout is {}.", mgmtServiceConf.getPingTimeout());
 
-        final int threads = DirectAgentLoadSize.value();
-
         _nodeId = ManagementServerNode.getManagementServerId();
         logger.info("Configuring AgentManagerImpl. management server node id(msid): {}.", _nodeId);
 
@@ -225,24 +249,34 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
         registerForHostEvents(new SetHostParamsListener(), true, true, false);
 
-        _executor = new ThreadPoolExecutor(threads, threads, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentTaskPool"));
+        managementServerMaintenanceManager.registerListener(this);
 
-        _connectExecutor = new ThreadPoolExecutor(100, 500, 60l, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), new NamedThreadFactory("AgentConnectTaskPool"));
+        final int agentTaskThreads = DirectAgentLoadSize.value();
+
+        _executor = new ThreadPoolExecutor(agentTaskThreads, agentTaskThreads, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentTaskPool"));
+
+        _connectExecutor = new ThreadPoolExecutor(100, 500, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentConnectTaskPool"));
         // allow core threads to time out even when there are no items in the queue
         _connectExecutor.allowCoreThreadTimeOut(true);
 
-        _connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10, this, caService);
+        maxConcurrentNewAgentConnections = RemoteAgentMaxConcurrentNewConnections.value();
+
+        _connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10,
+                this, caService, RemoteAgentSslHandshakeTimeout.value());
         logger.info("Listening on {} with {} workers.", Port.value(), Workers.value());
 
+        final int directAgentPoolSize = DirectAgentPoolSize.value();
         // executes all agent commands other than cron and ping
-        _directAgentExecutor = new ScheduledThreadPoolExecutor(DirectAgentPoolSize.value(), new NamedThreadFactory("DirectAgent"));
+        _directAgentExecutor = new ScheduledThreadPoolExecutor(directAgentPoolSize, new NamedThreadFactory("DirectAgent"));
         // executes cron and ping agent commands
-        _cronJobExecutor = new ScheduledThreadPoolExecutor(DirectAgentPoolSize.value(), new NamedThreadFactory("DirectAgentCronJob"));
-        logger.debug("Created DirectAgentAttache pool with size: {}.", DirectAgentPoolSize.value());
-        _directAgentThreadCap = Math.round(DirectAgentPoolSize.value() * DirectAgentThreadCap.value()) + 1; // add 1 to always make the value > 0
+        _cronJobExecutor = new ScheduledThreadPoolExecutor(directAgentPoolSize, new NamedThreadFactory("DirectAgentCronJob"));
+        logger.debug("Created DirectAgentAttache pool with size: {}.", directAgentPoolSize);
+        _directAgentThreadCap = Math.round(directAgentPoolSize * DirectAgentThreadCap.value()) + 1; // add 1 to always make the value > 0
 
         _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor"));
 
+        newAgentConnectionsMonitor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NewAgentConnectionsMonitor"));
+
         initializeCommandTimeouts();
 
         return true;
@@ -253,22 +287,44 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         return new AgentHandler(type, link, data);
     }
 
+    @Override
+    public int getMaxConcurrentNewConnectionsCount() {
+        return maxConcurrentNewAgentConnections;
+    }
+
+    @Override
+    public int getNewConnectionsCount() {
+        return newAgentConnections.size();
+    }
+
+    @Override
+    public void registerNewConnection(SocketAddress address) {
+        logger.trace("Adding new agent connection from {}", address.toString());
+        newAgentConnections.putIfAbsent(address.toString(), System.currentTimeMillis());
+    }
+
+    @Override
+    public void unregisterNewConnection(SocketAddress address) {
+        logger.trace("Removing new agent connection for {}", address.toString());
+        newAgentConnections.remove(address.toString());
+    }
+
     @Override
     public int registerForHostEvents(final Listener listener, final boolean connections, final boolean commands, final boolean priority) {
         synchronized (_hostMonitors) {
             _monitorId++;
             if (connections) {
                 if (priority) {
-                    _hostMonitors.add(0, new Pair<Integer, Listener>(_monitorId, listener));
+                    _hostMonitors.add(0, new Pair<>(_monitorId, listener));
                 } else {
-                    _hostMonitors.add(new Pair<Integer, Listener>(_monitorId, listener));
+                    _hostMonitors.add(new Pair<>(_monitorId, listener));
                 }
             }
             if (commands) {
                 if (priority) {
-                    _cmdMonitors.add(0, new Pair<Integer, Listener>(_monitorId, listener));
+                    _cmdMonitors.add(0, new Pair<>(_monitorId, listener));
                 } else {
-                    _cmdMonitors.add(new Pair<Integer, Listener>(_monitorId, listener));
+                    _cmdMonitors.add(new Pair<>(_monitorId, listener));
                 }
             }
             logger.debug("Registering listener {} with id {}", listener.getClass().getSimpleName(), _monitorId);
@@ -281,9 +337,9 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         synchronized (_hostMonitors) {
             _monitorId++;
             if (priority) {
-                _creationMonitors.add(0, new Pair<Integer, StartupCommandProcessor>(_monitorId, creator));
+                _creationMonitors.add(0, new Pair<>(_monitorId, creator));
             } else {
-                _creationMonitors.add(new Pair<Integer, StartupCommandProcessor>(_monitorId, creator));
+                _creationMonitors.add(new Pair<>(_monitorId, creator));
             }
             return _monitorId;
         }
@@ -295,8 +351,47 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         _hostMonitors.remove(id);
     }
 
+    @Override
+    public void onManagementServerMaintenance() {
+        logger.debug("Management server maintenance enabled");
+        _monitorExecutor.shutdownNow();
+        if (_connection != null) {
+            _connection.stop();
+
+            try {
+                _connection.cleanUp();
+            } catch (final IOException e) {
+                logger.warn("Fail to clean up old connection", e);
+            }
+        }
+        _connectExecutor.shutdownNow();
+    }
+
+    @Override
+    public void onManagementServerCancelMaintenance() {
+        logger.debug("Management server maintenance disabled");
+        if (_connectExecutor.isShutdown()) {
+            _connectExecutor = new ThreadPoolExecutor(100, 500, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentConnectTaskPool"));
+            _connectExecutor.allowCoreThreadTimeOut(true);
+        }
+
+        startDirectlyConnectedHosts(true);
+        if (_connection != null) {
+            try {
+                _connection.start();
+            } catch (final NioConnectionException e) {
+                logger.error("Error when connecting to the NioServer!", e);
+            }
+        }
+
+        if (_monitorExecutor.isShutdown()) {
+            _monitorExecutor = new ScheduledThreadPoolExecutor(1, new NamedThreadFactory("AgentMonitor"));
+            _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS);
+        }
+    }
+
     private AgentControlAnswer handleControlCommand(final AgentAttache attache, final AgentControlCommand cmd) {
-        AgentControlAnswer answer = null;
+        AgentControlAnswer answer;
 
         for (final Pair<Integer, Listener> listener : _cmdMonitors) {
             answer = listener.second().processControlCommand(attache.getId(), cmd);
@@ -324,13 +419,23 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     }
 
     public AgentAttache findAttache(final long hostId) {
-        AgentAttache attache = null;
+        AgentAttache attache;
         synchronized (_agents) {
             attache = _agents.get(hostId);
         }
         return attache;
     }
 
+    @Override
+    public List<String> getLastAgents() {
+        return lastAgents;
+    }
+
+    @Override
+    public void setLastAgents(List<String> lastAgents) {
+        this.lastAgents = lastAgents;
+    }
+
     @Override
     public Answer sendTo(final Long dcId, final HypervisorType type, final Command cmd) {
         final List<ClusterVO> clusters = _clusterDao.listByDcHyType(dcId, type.toString());
@@ -366,12 +471,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         cmds.addCommand(cmd);
         send(hostId, cmds, cmd.getWait());
         final Answer[] answers = cmds.getAnswers();
-        if (answers != null && !(answers[0] instanceof UnsupportedAnswer)) {
-            return answers[0];
-        }
-
-        if (answers != null && answers[0] instanceof UnsupportedAnswer) {
-            logger.warn("Unsupported Command: {}", answers[0].getDetails());
+        if (answers != null) {
+            if (answers[0] instanceof UnsupportedAnswer) {
+                logger.warn("Unsupported Command: {}", answers[0].getDetails());
+            }
             return answers[0];
         }
 
@@ -402,8 +505,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     }
 
     /**
-     * @param commands
-     * @return
+     * @param commands object container of commands
+     * @return array of commands
      */
     private Command[] checkForCommandsAndTag(final Commands commands) {
         final Command[] cmds = commands.toCommands();
@@ -419,8 +522,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     }
 
     /**
-     * @param commands
-     * @param cmds
+     * @param commands object container of commands
+     * @param cmds array of commands
      */
     private void setEmptyAnswers(final Commands commands, final Command[] cmds) {
         if (cmds.length == 0) {
@@ -459,7 +562,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         String commandWaits = GranularWaitTimeForCommands.value().trim();
         if (StringUtils.isNotEmpty(commandWaits)) {
             _commandTimeouts = getCommandTimeoutsMap(commandWaits);
-            logger.info(String.format("Timeouts for management server internal commands successfully initialized from global setting commands.timeout: %s", _commandTimeouts));
+            logger.info("Timeouts for management server internal commands successfully initialized from global setting commands.timeout: {}", _commandTimeouts);
         }
     }
 
@@ -475,10 +578,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                     int commandTimeout = Integer.parseInt(parts[1].trim());
                     commandTimeouts.put(commandName, commandTimeout);
                 } catch (NumberFormatException e) {
-                    logger.error(String.format("Initialising the timeouts using commands.timeout: %s for management server internal commands failed with error %s", commandPair, e.getMessage()));
+                    logger.error("Initialising the timeouts using commands.timeout: {} for management server internal commands failed with error {}", commandPair, e.getMessage());
                 }
             } else {
-                logger.error(String.format("Error initialising the timeouts for management server internal commands. Invalid format in commands.timeout: %s", commandPair));
+                logger.error("Error initialising the timeouts for management server internal commands. Invalid format in commands.timeout: {}", commandPair);
             }
         }
         return commandTimeouts;
@@ -492,7 +595,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         }
 
         int wait = getTimeout(commands, timeout);
-        logger.debug(String.format("Wait time setting on %s is %d seconds", commands, wait));
+        logger.debug("Wait time setting on {} is {} seconds", commands, wait);
         for (Command cmd : commands) {
             String simpleCommandName = cmd.getClass().getSimpleName();
             Integer commandTimeout = _commandTimeouts.get(simpleCommandName);
@@ -579,7 +682,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         }
         final long hostId = attache.getId();
         logger.debug("Remove Agent : {}", attache);
-        AgentAttache removed = null;
+        AgentAttache removed;
         boolean conflict = false;
         synchronized (_agents) {
             removed = _agents.remove(hostId);
@@ -615,38 +718,32 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         final long hostId = attache.getId();
         final HostVO host = _hostDao.findById(hostId);
         for (final Pair<Integer, Listener> monitor : _hostMonitors) {
-            logger.debug("Sending Connect to listener: {}", monitor.second().getClass().getSimpleName());
+            logger.debug("Sending Connect to listener: {}, for rebalance: {}", monitor.second().getClass().getSimpleName(), forRebalance);
             for (int i = 0; i < cmd.length; i++) {
                 try {
+                    logger.debug("process connection to issue: {} for host: {}, forRebalance: {}, connection transferred: {}", ReflectionToStringBuilderUtils.reflectCollection(cmd[i]), hostId, forRebalance, cmd[i].isConnectionTransferred());
                     monitor.second().processConnect(host, cmd[i], forRebalance);
-                } catch (final Exception e) {
-                    if (e instanceof ConnectionException) {
-                        final ConnectionException ce = (ConnectionException)e;
-                        if (ce.isSetupError()) {
-                            logger.warn("Monitor {} says there is an error in the connect process for {} due to {}",
-                                    monitor.second().getClass().getSimpleName(), host, e.getMessage());
-                            handleDisconnectWithoutInvestigation(attache, Event.AgentDisconnected, true, true);
-                            throw ce;
-                        } else {
-                            logger.info("Monitor {} says not to continue the connect process for {} due to {}",
-                                    monitor.second().getClass().getSimpleName(), host, e.getMessage());
-                            handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
-                            return attache;
-                        }
-                    } else if (e instanceof HypervisorVersionChangedException) {
-                        handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
-                        throw new CloudRuntimeException(String.format("Unable to connect %s", attache), e);
-                    } else {
-                        logger.error("Monitor {} says there is an error in the connect process for {} due to {}",
-                                monitor.second().getClass().getSimpleName(), host, e.getMessage(), e);
+                } catch (final ConnectionException ce) {
+                    if (ce.isSetupError()) {
+                        logger.warn("Monitor {} says there is an error in the connect process for {} due to {}", monitor.second().getClass().getSimpleName(), hostId, ce.getMessage());
                         handleDisconnectWithoutInvestigation(attache, Event.AgentDisconnected, true, true);
-                        throw new CloudRuntimeException(String.format("Unable to connect %s", attache), e);
+                        throw ce;
+                    } else {
+                        logger.info("Monitor {} says not to continue the connect process for {} due to {}", monitor.second().getClass().getSimpleName(), hostId, ce.getMessage());
+                        handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
+                        return attache;
                     }
+                } catch (final HypervisorVersionChangedException hvce) {
+                    handleDisconnectWithoutInvestigation(attache, Event.ShutdownRequested, true, true);
+                    throw new CloudRuntimeException("Unable to connect " + attache.getId(), hvce);
+                } catch (final Exception e) {
+                    logger.error("Monitor {} says there is an error in the connect process for {} due to {}", monitor.second().getClass().getSimpleName(), hostId, e.getMessage(), e);
+                    handleDisconnectWithoutInvestigation(attache, Event.AgentDisconnected, true, true);
+                    throw new CloudRuntimeException("Unable to connect " + attache.getId(), e);
                 }
             }
         }
 
-        final Long dcId = host.getDataCenterId();
         final ReadyCommand ready = new ReadyCommand(host, NumbersUtil.enableHumanReadableSizes);
         ready.setWait(ReadyCommandWait.value());
         final Answer answer = easySend(hostId, ready);
@@ -679,7 +776,13 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
     @Override
     public boolean start() {
-        startDirectlyConnectedHosts();
+        ManagementServerHostVO msHost = _mshostDao.findByMsid(_nodeId);
+        if (msHost != null && (ManagementServerHost.State.Maintenance.equals(msHost.getState()) || ManagementServerHost.State.PreparingForMaintenance.equals(msHost.getState()))) {
+            _monitorExecutor.shutdownNow();
+            return true;
+        }
+
+        startDirectlyConnectedHosts(false);
 
         if (_connection != null) {
             try {
@@ -691,13 +794,17 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
         _monitorExecutor.scheduleWithFixedDelay(new MonitorTask(), mgmtServiceConf.getPingInterval(), mgmtServiceConf.getPingInterval(), TimeUnit.SECONDS);
 
+        final int cleanupTime = Wait.value();
+        newAgentConnectionsMonitor.scheduleAtFixedRate(new AgentNewConnectionsMonitorTask(), cleanupTime,
+                cleanupTime, TimeUnit.MINUTES);
+
         return true;
     }
 
-    public void startDirectlyConnectedHosts() {
+    public void startDirectlyConnectedHosts(final boolean forRebalance) {
         final List<HostVO> hosts = _resourceMgr.findDirectlyConnectedHosts();
         for (final HostVO host : hosts) {
-            loadDirectlyConnectedHost(host, false);
+            loadDirectlyConnectedHost(host, forRebalance);
         }
     }
 
@@ -709,25 +816,25 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             final Constructor<?> constructor = clazz.getConstructor();
             resource = (ServerResource)constructor.newInstance();
         } catch (final ClassNotFoundException e) {
-            logger.warn("Unable to find class " + host.getResource(), e);
+            logger.warn("Unable to find class {}", host.getResource(), e);
         } catch (final InstantiationException e) {
-            logger.warn("Unable to instantiate class " + host.getResource(), e);
+            logger.warn("Unable to instantiate class {}", host.getResource(), e);
         } catch (final IllegalAccessException e) {
-            logger.warn("Illegal access " + host.getResource(), e);
+            logger.warn("Illegal access {}", host.getResource(), e);
         } catch (final SecurityException e) {
-            logger.warn("Security error on " + host.getResource(), e);
+            logger.warn("Security error on {}", host.getResource(), e);
         } catch (final NoSuchMethodException e) {
-            logger.warn("NoSuchMethodException error on " + host.getResource(), e);
+            logger.warn("NoSuchMethodException error on {}", host.getResource(), e);
         } catch (final IllegalArgumentException e) {
-            logger.warn("IllegalArgumentException error on " + host.getResource(), e);
+            logger.warn("IllegalArgumentException error on {}", host.getResource(), e);
         } catch (final InvocationTargetException e) {
-            logger.warn("InvocationTargetException error on " + host.getResource(), e);
+            logger.warn("InvocationTargetException error on {}", host.getResource(), e);
         }
 
         if (resource != null) {
             _hostDao.loadDetails(host);
 
-            final HashMap<String, Object> params = new HashMap<String, Object>(host.getDetails().size() + 5);
+            final HashMap<String, Object> params = new HashMap<>(host.getDetails().size() + 5);
             params.putAll(host.getDetails());
 
             params.put("guid", host.getGuid());
@@ -737,7 +844,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             }
             if (host.getClusterId() != null) {
                 params.put("cluster", Long.toString(host.getClusterId()));
-                String guid = null;
+                String guid;
                 final ClusterVO cluster = _clusterDao.findById(host.getClusterId());
                 if (cluster.getGuid() == null) {
                     guid = host.getDetail("pool");
@@ -772,8 +879,12 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     }
 
     protected boolean loadDirectlyConnectedHost(final HostVO host, final boolean forRebalance) {
+        return loadDirectlyConnectedHost(host, forRebalance, false);
+    }
+
+    protected boolean loadDirectlyConnectedHost(final HostVO host, final boolean forRebalance, final boolean isTransferredConnection) {
         boolean initialized = false;
-        ServerResource resource = null;
+        ServerResource resource;
         try {
             // load the respective discoverer
             final Discoverer discoverer = _resourceMgr.getMatchingDiscover(host.getHypervisorType());
@@ -800,21 +911,21 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
         if (forRebalance) {
             tapLoadingAgents(host.getId(), TapAgentsAction.Add);
-            final Host h = _resourceMgr.createHostAndAgent(host.getId(), resource, host.getDetails(), false, null, true);
+            final Host h = _resourceMgr.createHostAndAgent(host.getId(), resource, host.getDetails(), false, null, true, isTransferredConnection);
             tapLoadingAgents(host.getId(), TapAgentsAction.Del);
 
-            return h == null ? false : true;
+            return h != null;
         } else {
             _executor.execute(new SimulateStartTask(host.getId(), host.getUuid(), host.getName(), resource, host.getDetails()));
             return true;
         }
     }
 
-    protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) throws ConnectionException {
+    protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) {
         logger.debug("create DirectAgentAttache for {}", host);
         final DirectAgentAttache attache = new DirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), resource, host.isInMaintenanceStates());
 
-        AgentAttache old = null;
+        AgentAttache old;
         synchronized (_agents) {
             old = _agents.put(host.getId(), attache);
         }
@@ -848,6 +959,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
         _connectExecutor.shutdownNow();
         _monitorExecutor.shutdownNow();
+        newAgentConnectionsMonitor.shutdownNow();
         return true;
     }
 
@@ -879,7 +991,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             try {
                 logger.info("Host {} is disconnecting with event {}",
                         attache, event);
-                Status nextStatus = null;
+                Status nextStatus;
                 final HostVO host = _hostDao.findById(hostId);
                 if (host == null) {
                     logger.warn("Can't find host with {} ({})", hostId, attache);
@@ -993,7 +1105,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         handleDisconnectWithoutInvestigation(attache, event, true, true);
         host = _hostDao.findById(hostId); // Maybe the host magically reappeared?
         if (host != null && host.getStatus() == Status.Down) {
-            _haMgr.scheduleRestartForVmsOnHost(host, true);
+            _haMgr.scheduleRestartForVmsOnHost(host, true, HighAvailabilityManager.ReasonType.HostDown);
         }
         return true;
     }
@@ -1012,7 +1124,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         @Override
         protected void runInContext() {
             try {
-                if (_investigate == true) {
+                if (_investigate) {
                     handleDisconnectWithInvestigation(_attache, _event);
                 } else {
                     handleDisconnectWithoutInvestigation(_attache, _event, true, false);
@@ -1064,8 +1176,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     public Answer[] send(final Long hostId, final Commands cmds) throws AgentUnavailableException, OperationTimedoutException {
         int wait = 0;
         if (cmds.size() > 1) {
-            logger.debug(String.format("Checking the wait time in seconds to be used for the following commands : %s. If there are multiple commands sent at once," +
-                    "then max wait time of those will be used", cmds));
+            logger.debug("Checking the wait time in seconds to be used for the following commands : {}. If there are multiple commands sent at once," +
+                    "then max wait time of those will be used", cmds);
         }
 
         for (final Command cmd : cmds) {
@@ -1128,7 +1240,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
     public boolean executeUserRequest(final long hostId, final Event event) throws AgentUnavailableException {
         if (event == Event.AgentDisconnected) {
-            AgentAttache attache = null;
+            AgentAttache attache;
             attache = findAttache(hostId);
             logger.debug("Received agent disconnect event for host {} ({})", hostId, attache);
             if (attache != null) {
@@ -1154,12 +1266,12 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         return agentAttache != null;
     }
 
-    protected AgentAttache createAttacheForConnect(final HostVO host, final Link link) throws ConnectionException {
+    protected AgentAttache createAttacheForConnect(final HostVO host, final Link link) {
         logger.debug("create ConnectedAgentAttache for {}", host);
         final AgentAttache attache = new ConnectedAgentAttache(this, host.getId(), host.getUuid(), host.getName(), link, host.isInMaintenanceStates());
         link.attach(attache);
 
-        AgentAttache old = null;
+        AgentAttache old;
         synchronized (_agents) {
             old = _agents.put(host.getId(), attache);
         }
@@ -1184,7 +1296,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             }
         }
         ready.setArch(host.getArch().getType());
-        AgentAttache attache = null;
+        AgentAttache attache;
         GlobalLock joinLock = getHostJoinLock(host.getId());
         if (joinLock.lock(60)) {
             try {
@@ -1210,7 +1322,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         return attache;
     }
 
-    private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[] startup, final Request request) {
+    private AgentAttache handleConnectedAgent(final Link link, final StartupCommand[] startup) {
         AgentAttache attache = null;
         ReadyCommand ready = null;
         try {
@@ -1238,7 +1350,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                 easySend(attache.getId(), ready);
             }
         } catch (final Exception e) {
-            logger.debug("Failed to send ready command:" + e.toString());
+            logger.debug("Failed to send ready command:", e);
         }
         return attache;
     }
@@ -1264,6 +1376,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             this.id = id;
             this.resource = resource;
             this.details = details;
+            this.uuid = uuid;
+            this.name = name;
         }
 
         @Override
@@ -1312,10 +1426,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                 startups[i] = (StartupCommand)_cmds[i];
             }
 
-            final AgentAttache attache = handleConnectedAgent(_link, startups, _request);
+            final AgentAttache attache = handleConnectedAgent(_link, startups);
             if (attache == null) {
                 logger.warn("Unable to create attache for agent: {}", _request);
             }
+            unregisterNewConnection(_link.getSocketAddress());
         }
     }
 
@@ -1332,7 +1447,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                 break;
             }
         }
-        Response response = null;
+        Response response;
         response = new Response(request, answers[0], _nodeId, -1);
         try {
             link.send(response.toBytes());
@@ -1413,7 +1528,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             }
 
             final long hostId = attache.getId();
-            final String hostName = attache.getName();
 
             if (logger.isDebugEnabled()) {
                 if (cmd instanceof PingRoutingCommand) {
@@ -1432,7 +1546,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             final Answer[] answers = new Answer[cmds.length];
             for (int i = 0; i < cmds.length; i++) {
                 cmd = cmds[i];
-                Answer answer = null;
+                Answer answer;
                 try {
                     if (cmd instanceof StartupRoutingCommand) {
                         final StartupRoutingCommand startup = (StartupRoutingCommand) cmd;
@@ -1466,7 +1580,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                             final long cmdHostId = ((PingCommand)cmd).getHostId();
                             boolean requestStartupCommand = false;
 
-                            final HostVO host = _hostDao.findById(Long.valueOf(cmdHostId));
+                            final HostVO host = _hostDao.findById(cmdHostId);
                             boolean gatewayAccessible = true;
                             // if the router is sending a ping, verify the
                             // gateway was pingable
@@ -1516,7 +1630,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             if (logD) {
                 logger.debug("SeqA {}-: Sending {}", attache.getId(), response.getSequence(), response);
             } else {
-                logger.trace("SeqA {}-: Sending {}" + attache.getId(), response.getSequence(), response);
+                logger.trace("SeqA {}-: Sending {} {}", response.getSequence(), response, attache.getId());
             }
             try {
                 link.send(response.toBytes());
@@ -1536,15 +1650,14 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
         @Override
         protected void doTask(final Task task) throws TaskExecutionException {
-            final TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
-            try {
+            try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
                 final Type type = task.getType();
-                if (type == Task.Type.DATA) {
+                if (type == Type.DATA) {
                     final byte[] data = task.getData();
                     try {
                         final Request event = Request.parse(data);
                         if (event instanceof Response) {
-                            processResponse(task.getLink(), (Response)event);
+                            processResponse(task.getLink(), (Response) event);
                         } else {
                             processRequest(task.getLink(), event);
                         }
@@ -1556,10 +1669,10 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                         logger.error(message);
                         throw new TaskExecutionException(message, e);
                     }
-                } else if (type == Task.Type.CONNECT) {
-                } else if (type == Task.Type.DISCONNECT) {
+                } else if (type == Type.CONNECT) {
+                } else if (type == Type.DISCONNECT) {
                     final Link link = task.getLink();
-                    final AgentAttache attache = (AgentAttache)link.attachment();
+                    final AgentAttache attache = (AgentAttache) link.attachment();
                     if (attache != null) {
                         disconnectWithInvestigation(attache, Event.AgentDisconnected);
                     } else {
@@ -1568,8 +1681,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                         link.terminated();
                     }
                 }
-            } finally {
-                txn.close();
             }
         }
     }
@@ -1598,21 +1709,16 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
     @Override
     public boolean agentStatusTransitTo(final HostVO host, final Status.Event e, final long msId) {
-        try {
-            _agentStatusLock.lock();
-            logger.debug("[Resource state = {}, Agent event = , Host = {}]",
-                    host.getResourceState(), e.toString(), host);
+        logger.debug("[Resource state = {}, Agent event = , Host = {}]",
+                host.getResourceState(), e.toString(), host);
 
-            host.setManagementServerId(msId);
-            try {
-                return _statusStateMachine.transitTo(host, e, host.getId(), _hostDao);
-            } catch (final NoTransitionException e1) {
-                logger.debug("Cannot transit agent status with event {} for host {}, management server id is {}", e, host, msId);
-                throw new CloudRuntimeException(String.format(
-                        "Cannot transit agent status with event %s for host %s, management server id is %d, %s", e, host, msId, e1.getMessage()));
-            }
-        } finally {
-            _agentStatusLock.unlock();
+        host.setManagementServerId(msId);
+        try {
+            return _statusStateMachine.transitTo(host, e, host.getId(), _hostDao);
+        } catch (final NoTransitionException e1) {
+            logger.debug("Cannot transit agent status with event {} for host {}, management server id is {}", e, host, msId);
+            throw new CloudRuntimeException(String.format(
+                    "Cannot transit agent status with event %s for host %s, management server id is %d, %s", e, host, msId, e1.getMessage()));
         }
     }
 
@@ -1801,7 +1907,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         }
 
         protected List<Long> findAgentsBehindOnPing() {
-            final List<Long> agentsBehind = new ArrayList<Long>();
+            final List<Long> agentsBehind = new ArrayList<>();
             final long cutoffTime = InaccurateClock.getTimeInSeconds() - mgmtServiceConf.getTimeout();
             for (final Map.Entry<Long, Long> entry : _pingMap.entrySet()) {
                 if (entry.getValue() < cutoffTime) {
@@ -1809,7 +1915,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                 }
             }
 
-            if (agentsBehind.size() > 0) {
+            if (!agentsBehind.isEmpty()) {
                 logger.info("Found the following agents behind on ping: {}", agentsBehind);
             }
 
@@ -1817,6 +1923,35 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         }
     }
 
+    protected class AgentNewConnectionsMonitorTask extends ManagedContextRunnable {
+        @Override
+        protected void runInContext() {
+            logger.trace("Agent New Connections Monitor is started.");
+            final int cleanupTime = Wait.value();
+            Set<Map.Entry<String, Long>> entrySet = newAgentConnections.entrySet();
+            long cutOff = System.currentTimeMillis() - (cleanupTime * 60 * 1000L);
+            if (logger.isDebugEnabled()) {
+                List<String> expiredConnections = newAgentConnections.entrySet()
+                        .stream()
+                        .filter(e -> e.getValue() <= cutOff)
+                        .map(Map.Entry::getKey)
+                        .collect(Collectors.toList());
+                logger.debug("Currently {} active new connections, of which {} have expired - {}",
+                        entrySet.size(),
+                        expiredConnections.size(),
+                        StringUtils.join(expiredConnections));
+            }
+            for (Map.Entry<String, Long> entry : entrySet) {
+                if (entry.getValue() <= cutOff) {
+                    if (logger.isTraceEnabled()) {
+                        logger.trace("Cleaning up new agent connection for {}", entry.getKey());
+                    }
+                    newAgentConnections.remove(entry.getKey());
+                }
+            }
+        }
+    }
+
     protected class BehindOnPingListener implements Listener {
         @Override
         public boolean isRecurring() {
@@ -1892,7 +2027,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
     @Override
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
-                DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait, GranularWaitTimeForCommands };
+                DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait,
+                GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections };
     }
 
     protected class SetHostParamsListener implements Listener {
@@ -1922,12 +2058,15 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
 
         @Override
         public void processConnect(final Host host, final StartupCommand cmd, final boolean forRebalance) {
-            if (cmd instanceof StartupRoutingCommand) {
-                if (((StartupRoutingCommand)cmd).getHypervisorType() == HypervisorType.KVM || ((StartupRoutingCommand)cmd).getHypervisorType() == HypervisorType.LXC) {
-                    Map<String, String> params = new HashMap<String, String>();
-                    params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
-                    params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
-                    params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId())));
+            if (!(cmd instanceof StartupRoutingCommand) || cmd.isConnectionTransferred()) {
+                return;
+            }
+
+            if (((StartupRoutingCommand)cmd).getHypervisorType() == HypervisorType.KVM || ((StartupRoutingCommand)cmd).getHypervisorType() == HypervisorType.LXC) {
+                Map<String, String> params = new HashMap<>();
+                params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString()));
+                params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString()));
+                params.put(NetworkOrchestrationService.TUNGSTEN_ENABLED.key(), String.valueOf(NetworkOrchestrationService.TUNGSTEN_ENABLED.valueIn(host.getDataCenterId())));
 
                     try {
                         SetHostParamsCommand cmds = new SetHostParamsCommand(params);
@@ -1939,8 +2078,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
                 }
             }
 
-        }
-
         @Override
         public boolean processDisconnect(final long agentId, final Status state) {
             return true;
@@ -1971,13 +2108,13 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         if (allHosts == null) {
             return null;
         }
-        Map<Long, List<Long>> hostsByZone = new HashMap<Long, List<Long>>();
+        Map<Long, List<Long>> hostsByZone = new HashMap<>();
         for (HostVO host : allHosts) {
             if (host.getHypervisorType() == HypervisorType.KVM || host.getHypervisorType() == HypervisorType.LXC) {
                 Long zoneId = host.getDataCenterId();
                 List<Long> hostIds = hostsByZone.get(zoneId);
                 if (hostIds == null) {
-                    hostIds = new ArrayList<Long>();
+                    hostIds = new ArrayList<>();
                 }
                 hostIds.add(host.getId());
                 hostsByZone.put(zoneId, hostIds);
@@ -2008,6 +2145,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
         }
     }
 
+    @Override
+    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) {
+        return true;
+    }
+
     private GlobalLock getHostJoinLock(Long hostId) {
         return GlobalLock.getInternLock(String.format("%s-%s", "Host-Join", hostId));
     }

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -47,14 +47,17 @@ import org.apache.cloudstack.framework.config.ConfigDepot;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.ha.dao.HAConfigDao;
+import org.apache.cloudstack.maintenance.ManagementServerMaintenanceManager;
+import org.apache.cloudstack.maintenance.command.BaseShutdownManagementServerHostCommand;
+import org.apache.cloudstack.maintenance.command.CancelMaintenanceManagementServerHostCommand;
+import org.apache.cloudstack.maintenance.command.CancelShutdownManagementServerHostCommand;
+import org.apache.cloudstack.maintenance.command.PrepareForMaintenanceManagementServerHostCommand;
+import org.apache.cloudstack.maintenance.command.PrepareForShutdownManagementServerHostCommand;
+import org.apache.cloudstack.maintenance.command.TriggerShutdownManagementServerHostCommand;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.managed.context.ManagedContextTimerTask;
+import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.outofbandmanagement.dao.OutOfBandManagementDao;
-import org.apache.cloudstack.shutdown.ShutdownManager;
-import org.apache.cloudstack.shutdown.command.CancelShutdownManagementServerHostCommand;
-import org.apache.cloudstack.shutdown.command.PrepareForShutdownManagementServerHostCommand;
-import org.apache.cloudstack.shutdown.command.BaseShutdownManagementServerHostCommand;
-import org.apache.cloudstack.shutdown.command.TriggerShutdownManagementServerHostCommand;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.cloudstack.utils.security.SSLUtils;
 
@@ -73,13 +76,15 @@ import com.cloud.cluster.ClusterManager;
 import com.cloud.cluster.ClusterManagerListener;
 import com.cloud.cluster.ClusterServicePdu;
 import com.cloud.cluster.ClusteredAgentRebalanceService;
-import org.apache.cloudstack.management.ManagementServerHost;
 import com.cloud.cluster.ManagementServerHostVO;
 import com.cloud.cluster.agentlb.AgentLoadBalancerPlanner;
 import com.cloud.cluster.agentlb.HostTransferMapVO;
 import com.cloud.cluster.agentlb.HostTransferMapVO.HostTransferState;
 import com.cloud.cluster.agentlb.dao.HostTransferMapDao;
 import com.cloud.cluster.dao.ManagementServerHostDao;
+import com.cloud.cluster.dao.ManagementServerHostPeerDao;
+import com.cloud.dc.DataCenterVO;
+import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.exception.AgentUnavailableException;
 import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.UnsupportedVersionException;
@@ -100,25 +105,30 @@ import com.cloud.utils.nio.Link;
 import com.cloud.utils.nio.Task;
 import com.google.gson.Gson;
 
+import org.apache.commons.collections.CollectionUtils;
+
 public class ClusteredAgentManagerImpl extends AgentManagerImpl implements ClusterManagerListener, ClusteredAgentRebalanceService {
-    private static final ScheduledExecutorService s_transferExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Cluster-AgentRebalancingExecutor"));
+    private static ScheduledExecutorService s_transferExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Cluster-AgentRebalancingExecutor"));
     private final long rebalanceTimeOut = 300000; // 5 mins - after this time remove the agent from the transfer list
 
     public final static long STARTUP_DELAY = 5000;
     public final static long SCAN_INTERVAL = 90000; // 90 seconds, it takes 60 sec for xenserver to fail login
     public final static int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 5; // 5 seconds
-    protected Set<Long> _agentToTransferIds = new HashSet<Long>();
+    protected Set<Long> _agentToTransferIds = new HashSet<>();
     Gson _gson;
     protected HashMap<String, SocketChannel> _peers;
     protected HashMap<String, SSLEngine> _sslEngines;
     private final Timer _timer = new Timer("ClusteredAgentManager Timer");
     boolean _agentLbHappened = false;
+    private int _mshostCounter = 0;
 
     @Inject
     protected ClusterManager _clusterMgr = null;
     @Inject
     protected ManagementServerHostDao _mshostDao;
     @Inject
+    protected ManagementServerHostPeerDao _mshostPeerDao;
+    @Inject
     protected HostTransferMapDao _hostTransferDao;
     @Inject
     protected List<AgentLoadBalancerPlanner> _lbPlanners;
@@ -133,23 +143,25 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     @Inject
     private CAManager caService;
     @Inject
-    private ShutdownManager shutdownManager;
+    private ManagementServerMaintenanceManager managementServerMaintenanceManager;
+    @Inject
+    private DataCenterDao dcDao;
 
     protected ClusteredAgentManagerImpl() {
         super();
     }
 
-    protected final ConfigKey<Boolean> EnableLB = new ConfigKey<Boolean>(Boolean.class, "agent.lb.enabled", "Advanced", "false", "Enable agent load balancing between management server nodes", true);
-    protected final ConfigKey<Double> ConnectedAgentThreshold = new ConfigKey<Double>(Double.class, "agent.load.threshold", "Advanced", "0.7",
+    protected final ConfigKey<Boolean> EnableLB = new ConfigKey<>(Boolean.class, "agent.lb.enabled", "Advanced", "false", "Enable agent load balancing between management server nodes", true);
+    protected final ConfigKey<Double> ConnectedAgentThreshold = new ConfigKey<>(Double.class, "agent.load.threshold", "Advanced", "0.7",
             "What percentage of the agents can be held by one management server before load balancing happens", true, EnableLB.key());
-    protected final ConfigKey<Integer> LoadSize = new ConfigKey<Integer>(Integer.class, "direct.agent.load.size", "Advanced", "16", "How many agents to connect to in each round", true);
-    protected final ConfigKey<Integer> ScanInterval = new ConfigKey<Integer>(Integer.class, "direct.agent.scan.interval", "Advanced", "90", "Interval between scans to load agents", false,
+    protected final ConfigKey<Integer> LoadSize = new ConfigKey<>(Integer.class, "direct.agent.load.size", "Advanced", "16", "How many agents to connect to in each round", true);
+    protected final ConfigKey<Integer> ScanInterval = new ConfigKey<>(Integer.class, "direct.agent.scan.interval", "Advanced", "90", "Interval between scans to load agents", false,
             ConfigKey.Scope.Global, 1000);
 
     @Override
     public boolean configure(final String name, final Map<String, Object> xmlParams) throws ConfigurationException {
-        _peers = new HashMap<String, SocketChannel>(7);
-        _sslEngines = new HashMap<String, SSLEngine>(7);
+        _peers = new HashMap<>(7);
+        _sslEngines = new HashMap<>(7);
         _nodeId = ManagementServerNode.getManagementServerId();
 
         logger.info("Configuring ClusterAgentManagerImpl. management server node id(msid): {}", _nodeId);
@@ -172,6 +184,13 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         _timer.schedule(new DirectAgentScanTimerTask(), STARTUP_DELAY, ScanInterval.value());
         logger.debug("Scheduled direct agent scan task to run at an interval of {} seconds", ScanInterval.value());
 
+        ManagementServerHostVO msHost = _mshostDao.findByMsid(_nodeId);
+        if (msHost != null && (ManagementServerHost.State.Maintenance.equals(msHost.getState()) || ManagementServerHost.State.PreparingForMaintenance.equals(msHost.getState()))) {
+            s_transferExecutor.shutdownNow();
+            cleanupTransferMap(_nodeId);
+            return true;
+        }
+
         // Schedule tasks for agent rebalancing
         if (isAgentRebalanceEnabled()) {
             cleanupTransferMap(_nodeId);
@@ -201,7 +220,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
 
         if (hosts != null) {
             hosts.addAll(appliances);
-            if (hosts.size() > 0) {
+            if (!hosts.isEmpty()) {
                 logger.debug("Found {} unmanaged direct hosts, processing connect for them...", hosts.size());
                 for (final HostVO host : hosts) {
                     try {
@@ -215,12 +234,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                                 continue;
                             }
                         }
-
-                        logger.debug("Loading directly connected host {}", host);
+                        logger.debug("Loading directly connected {}", host);
                         loadDirectlyConnectedHost(host, false);
                     } catch (final Throwable e) {
-                        logger.warn(" can not load directly connected host {}({}) due to ",
-                                host, e);
+                        logger.warn(" can not load directly connected {} due to ", host, e);
                     }
                 }
             }
@@ -248,10 +265,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         logger.debug("create forwarding ClusteredAgentAttache for {}", host);
         long id = host.getId();
         final AgentAttache attache = new ClusteredAgentAttache(this, id, host.getUuid(), host.getName());
-        AgentAttache old = null;
+        AgentAttache old;
         synchronized (_agents) {
-            old = _agents.get(id);
-            _agents.put(id, attache);
+            old = _agents.get(host.getId());
+            _agents.put(host.getId(), attache);
         }
         if (old != null) {
             logger.debug("Remove stale agent attache from current management server");
@@ -265,7 +282,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         logger.debug("create ClusteredAgentAttache for {}",  host);
         final AgentAttache attache = new ClusteredAgentAttache(this, host.getId(), host.getUuid(), host.getName(), link, host.isInMaintenanceStates());
         link.attach(attache);
-        AgentAttache old = null;
+        AgentAttache old;
         synchronized (_agents) {
             old = _agents.get(host.getId());
             _agents.put(host.getId(), attache);
@@ -280,7 +297,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     protected AgentAttache createAttacheForDirectConnect(final Host host, final ServerResource resource) {
         logger.debug("Create ClusteredDirectAgentAttache for {}.", host);
         final DirectAgentAttache attache = new ClusteredDirectAgentAttache(this, host.getId(), host.getUuid(), host.getName(), _nodeId, resource, host.isInMaintenanceStates());
-        AgentAttache old = null;
+        AgentAttache old;
         synchronized (_agents) {
             old = _agents.get(host.getId());
             _agents.put(host.getId(), attache);
@@ -399,12 +416,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     public boolean routeToPeer(final String peer, final byte[] bytes) {
         int i = 0;
         SocketChannel ch = null;
-        SSLEngine sslEngine = null;
+        SSLEngine sslEngine;
         while (i++ < 5) {
             ch = connectToPeer(peer, ch);
             if (ch == null) {
                 try {
-                    logD(bytes, "Unable to route to peer: " + Request.parse(bytes).toString());
+                    logD(bytes, "Unable to route to peer: " + Request.parse(bytes));
                 } catch (ClassNotFoundException | UnsupportedVersionException e) {
                     // Request.parse thrown exception when we try to log it, log as much as we can
                     logD(bytes, "Unable to route to peer, and Request.parse further caught exception" + e.getMessage());
@@ -422,7 +439,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 return true;
             } catch (final IOException e) {
                 try {
-                    logI(bytes, "Unable to route to peer: " + Request.parse(bytes).toString() + " due to " + e.getMessage());
+                    logI(bytes, "Unable to route to peer: " + Request.parse(bytes) + " due to " + e.getMessage());
                 } catch (ClassNotFoundException | UnsupportedVersionException ex) {
                     // Request.parse thrown exception when we try to log it, log as much as we can
                     logI(bytes, "Unable to route to peer due to" + e.getMessage() + ". Also caught exception when parsing request: " + ex.getMessage());
@@ -465,7 +482,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     public SocketChannel connectToPeer(final String peerName, final SocketChannel prevCh) {
         synchronized (_peers) {
             final SocketChannel ch = _peers.get(peerName);
-            SSLEngine sslEngine = null;
+            SSLEngine sslEngine;
             if (prevCh != null) {
                 try {
                     prevCh.close();
@@ -550,13 +567,13 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         AgentAttache agent = findAttache(hostId);
         if (agent == null || !agent.forForward()) {
             if (isHostOwnerSwitched(host)) {
-                logger.debug("Host {} has switched to another management server, need to update agent map with a forwarding agent attache",  host);
+                logger.debug("{} has switched to another management server, need to update agent map with a forwarding agent attache",  host);
                 agent = createAttache(host);
             }
         }
         if (agent == null) {
             final AgentUnavailableException ex = new AgentUnavailableException("Host with specified id is not in the right state: " + host.getStatus(), hostId);
-            ex.addProxyObject(_entityMgr.findById(Host.class, hostId).getUuid());
+            ex.addProxyObject(host.getUuid());
             throw ex;
         }
 
@@ -585,7 +602,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     }
 
     @Override
-    public void startDirectlyConnectedHosts() {
+    public void startDirectlyConnectedHosts(final boolean forRebalance) {
         // override and let it be dummy for purpose, we will scan and load direct agents periodically.
         // We may also pickup agents that have been left over from other crashed management server
     }
@@ -598,9 +615,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
 
         @Override
         protected void doTask(final Task task) throws TaskExecutionException {
-            final TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB);
-            try {
-                if (task.getType() != Task.Type.DATA) {
+            try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
+                if (task.getType() != Type.DATA) {
                     super.doTask(task);
                     return;
                 }
@@ -627,7 +643,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                         }
                         final Request req = Request.parse(data);
                         final Command[] cmds = req.getCommands();
-                        final CancelCommand cancel = (CancelCommand)cmds[0];
+                        final CancelCommand cancel = (CancelCommand) cmds[0];
                         logD(data, "Cancel request received");
                         agent.cancel(cancel.getSequence());
                         final Long current = agent._currentSequence;
@@ -651,10 +667,9 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                             // to deserialize this and send it through the agent attache.
                             final Request req = Request.parse(data);
                             agent.send(req, null);
-                            return;
                         } else {
                             if (agent instanceof Routable) {
-                                final Routable cluster = (Routable)agent;
+                                final Routable cluster = (Routable) agent;
                                 cluster.routeToAgent(data);
                             } else {
                                 agent.send(Request.parse(data));
@@ -671,13 +686,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                     if (mgmtId != -1 && mgmtId != _nodeId) {
                         routeToPeer(Long.toString(mgmtId), data);
                         if (Request.requiresSequentialExecution(data)) {
-                            final AgentAttache attache = (AgentAttache)link.attachment();
+                            final AgentAttache attache = (AgentAttache) link.attachment();
                             if (attache != null) {
                                 attache.sendNext(Request.getSequence(data));
                             }
-                            logD(data, "No attache to process " + Request.parse(data).toString());
+                            logD(data, "No attache to process " + Request.parse(data));
                         }
-                        return;
                     } else {
                         if (Request.isRequest(data)) {
                             super.doTask(task);
@@ -693,7 +707,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                                 logger.info("SeqA {}-{}: Response is not processed: {}", attache.getId(), response.getSequence(), response.toString());
                             }
                         }
-                        return;
                     }
                 }
             } catch (final ClassNotFoundException e) {
@@ -704,8 +717,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 final String message = String.format("UnsupportedVersionException occurred when executing tasks! Error '%s'", e.getMessage());
                 logger.error(message);
                 throw new TaskExecutionException(message, e);
-            } finally {
-                txn.close();
             }
         }
     }
@@ -742,12 +753,17 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
 
     @Override
     public boolean executeRebalanceRequest(final long agentId, final long currentOwnerId, final long futureOwnerId, final Event event) throws AgentUnavailableException, OperationTimedoutException {
+        return executeRebalanceRequest(agentId, currentOwnerId, futureOwnerId, event, false);
+    }
+
+    @Override
+    public boolean executeRebalanceRequest(final long agentId, final long currentOwnerId, final long futureOwnerId, final Event event, boolean isConnectionTransfer) throws AgentUnavailableException, OperationTimedoutException {
         boolean result = false;
         if (event == Event.RequestAgentRebalance) {
-            return setToWaitForRebalance(agentId, currentOwnerId, futureOwnerId);
+            return setToWaitForRebalance(agentId);
         } else if (event == Event.StartAgentRebalance) {
             try {
-                result = rebalanceHost(agentId, currentOwnerId, futureOwnerId);
+                result = rebalanceHost(agentId, currentOwnerId, futureOwnerId, isConnectionTransfer);
             } catch (final Exception e) {
                 logger.warn("Unable to rebalance host id={} ({})",  agentId, findAttache(agentId), e);
             }
@@ -799,7 +815,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         sc.and(sc.entity().getType(), Op.EQ, Host.Type.Routing);
         final List<HostVO> allManagedAgents = sc.list();
 
-        int avLoad = 0;
+        int avLoad;
 
         if (!allManagedAgents.isEmpty() && !allMS.isEmpty()) {
             avLoad = allManagedAgents.size() / allMS.size();
@@ -817,7 +833,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         for (final ManagementServerHostVO node : allMS) {
             if (node.getMsid() != _nodeId) {
 
-                List<HostVO> hostsToRebalance = new ArrayList<HostVO>();
+                List<HostVO> hostsToRebalance = new ArrayList<>();
                 for (final AgentLoadBalancerPlanner lbPlanner : _lbPlanners) {
                     hostsToRebalance = lbPlanner.getHostsToRebalance(node, avLoad);
                     if (hostsToRebalance != null && !hostsToRebalance.isEmpty()) {
@@ -843,7 +859,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                         HostTransferMapVO transfer = null;
                         try {
                             transfer = _hostTransferDao.startAgentTransfering(hostId, node.getMsid(), _nodeId);
-                            final Answer[] answer = sendRebalanceCommand(node.getMsid(), hostId, node.getMsid(), _nodeId, Event.RequestAgentRebalance);
+                            final Answer[] answer = sendRebalanceCommand(node.getMsid(), hostId, node.getMsid(), _nodeId);
                             if (answer == null) {
                                 logger.warn("Failed to get host {} from management server {}", host, node);
                                 result = false;
@@ -870,8 +886,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         }
     }
 
-    private Answer[] sendRebalanceCommand(final long peer, final long agentId, final long currentOwnerId, final long futureOwnerId, final Event event) {
-        final TransferAgentCommand transfer = new TransferAgentCommand(agentId, currentOwnerId, futureOwnerId, event);
+    private Answer[] sendRebalanceCommand(final long peer, final long agentId, final long currentOwnerId, final long futureOwnerId) {
+        return sendRebalanceCommand(peer, agentId, currentOwnerId, futureOwnerId, Event.RequestAgentRebalance, false);
+    }
+
+    private Answer[] sendRebalanceCommand(final long peer, final long agentId, final long currentOwnerId, final long futureOwnerId, final Event event, final boolean isConnectionTransfer) {
+        final TransferAgentCommand transfer = new TransferAgentCommand(agentId, currentOwnerId, futureOwnerId, event, isConnectionTransfer);
         final Commands commands = new Commands(Command.OnError.Stop);
         commands.addCommand(transfer);
 
@@ -882,8 +902,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             final String peerName = Long.toString(peer);
             final String cmdStr = _gson.toJson(cmds);
             final String ansStr = _clusterMgr.execute(peerName, agentId, cmdStr, true);
-            final Answer[] answers = _gson.fromJson(ansStr, Answer[].class);
-            return answers;
+            return _gson.fromJson(ansStr, Answer[].class);
         } catch (final Exception e) {
             logger.warn("Caught exception while talking to {}",  currentOwnerId, e);
             return null;
@@ -932,7 +951,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 try {
                     logger.trace("Clustered agent transfer scan check, management server id: {}",  _nodeId);
                     synchronized (_agentToTransferIds) {
-                        if (_agentToTransferIds.size() > 0) {
+                        if (!_agentToTransferIds.isEmpty()) {
                             logger.debug("Found {} agents to transfer", _agentToTransferIds.size());
                             // for (Long hostId : _agentToTransferIds) {
                             for (final Iterator<Long> iterator = _agentToTransferIds.iterator(); iterator.hasNext();) {
@@ -956,7 +975,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                                 }
 
                                 if (transferMap.getInitialOwner() != _nodeId || attache == null || attache.forForward()) {
-                                    logger.debug(String.format("Management server %d doesn't own host id=%d (%s) any more, skipping rebalance for the host", _nodeId, hostId, attache));
+                                    logger.debug("Management server {} doesn't own host id={} ({}) any more, skipping rebalance for the host", _nodeId, hostId, attache);
                                     iterator.remove();
                                     _hostTransferDao.completeAgentTransfer(hostId);
                                     continue;
@@ -976,9 +995,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                                         _executor.execute(new RebalanceTask(hostId, transferMap.getInitialOwner(), transferMap.getFutureOwner()));
                                     } catch (final RejectedExecutionException ex) {
                                         logger.warn("Failed to submit rebalance task for host id={} ({}); postponing the execution", hostId, attache);
-                                        continue;
                                     }
-
                                 } else {
                                     logger.debug("Agent {} ({}) can't be transferred yet as its request queue size is {} and listener queue size is {}",
                                             hostId, attache, attache.getQueueSize(), attache.getNonRecurringListenersSize());
@@ -988,7 +1005,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                             logger.trace("Found no agents to be transferred by the management server {}",  _nodeId);
                         }
                     }
-
                 } catch (final Throwable e) {
                     logger.error("Problem with the clustered agent transfer scan check!", e);
                 }
@@ -996,7 +1012,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         };
     }
 
-    private boolean setToWaitForRebalance(final long hostId, final long currentOwnerId, final long futureOwnerId) {
+    private boolean setToWaitForRebalance(final long hostId) {
         logger.debug("Adding agent {} ({}) to the list of agents to transfer", hostId, findAttache(hostId));
         synchronized (_agentToTransferIds) {
             return _agentToTransferIds.add(hostId);
@@ -1004,7 +1020,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     }
 
     protected boolean rebalanceHost(final long hostId, final long currentOwnerId, final long futureOwnerId) throws AgentUnavailableException {
+        return rebalanceHost(hostId, currentOwnerId, futureOwnerId, false);
+    }
 
+    protected boolean rebalanceHost(final long hostId, final long currentOwnerId, final long futureOwnerId, final boolean isConnectionTransfer) throws AgentUnavailableException {
         boolean result = true;
         if (currentOwnerId == _nodeId) {
             if (!startRebalance(hostId)) {
@@ -1013,7 +1032,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 return false;
             }
             try {
-                final Answer[] answer = sendRebalanceCommand(futureOwnerId, hostId, currentOwnerId, futureOwnerId, Event.StartAgentRebalance);
+                final Answer[] answer = sendRebalanceCommand(futureOwnerId, hostId, currentOwnerId, futureOwnerId, Event.StartAgentRebalance, isConnectionTransfer);
                 if (answer == null || !answer[0].getResult()) {
                     result = false;
                 }
@@ -1034,7 +1053,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         } else if (futureOwnerId == _nodeId) {
             final HostVO host = _hostDao.findById(hostId);
             try {
-                logger.debug("Disconnecting host {} as a part of rebalance process without notification", host);
+                logger.debug("Disconnecting {} as a part of rebalance process without notification", host);
 
                 final AgentAttache attache = findAttache(hostId);
                 if (attache != null) {
@@ -1043,7 +1062,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
 
                 if (result) {
                     logger.debug("Loading directly connected host {} to the management server {} as a part of rebalance process", host, _nodeId);
-                    result = loadDirectlyConnectedHost(host, true);
+                    result = loadDirectlyConnectedHost(host, true, isConnectionTransfer);
                 } else {
                     logger.warn("Failed to disconnect {} as a part of rebalance process without notification", host);
                 }
@@ -1054,9 +1073,9 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             }
 
             if (result) {
-                logger.debug("Successfully loaded directly connected host {} to the management server {} a part of rebalance process without notification", host, _nodeId);
+                logger.debug("Successfully loaded directly connected {} to the management server {} a part of rebalance process without notification", host, _nodeId);
             } else {
-                logger.warn("Failed to load directly connected host {} to the management server {} a part of rebalance process without notification", host, _nodeId);
+                logger.warn("Failed to load directly connected {} to the management server {} a part of rebalance process without notification", host, _nodeId);
             }
         }
 
@@ -1065,12 +1084,12 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
 
     protected void finishRebalance(final long hostId, final long futureOwnerId, final Event event) {
 
-        final boolean success = event == Event.RebalanceCompleted ? true : false;
+        final boolean success = event == Event.RebalanceCompleted;
 
         final AgentAttache attache = findAttache(hostId);
         logger.debug("Finishing rebalancing for the agent {} ({}) with event {}", hostId, attache, event);
 
-        if (attache == null || !(attache instanceof ClusteredAgentAttache)) {
+        if (!(attache instanceof ClusteredAgentAttache)) {
             logger.debug("Unable to find forward attache for the host id={} assuming that the agent disconnected already", hostId);
             _hostTransferDao.completeAgentTransfer(hostId);
             return;
@@ -1166,9 +1185,9 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     }
 
     protected class RebalanceTask extends ManagedContextRunnable {
-        Long hostId = null;
-        Long currentOwnerId = null;
-        Long futureOwnerId = null;
+        Long hostId;
+        Long currentOwnerId;
+        Long futureOwnerId;
 
         public RebalanceTask(final long hostId, final long currentOwnerId, final long futureOwnerId) {
             this.hostId = hostId;
@@ -1237,7 +1256,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
                 final ChangeAgentCommand cmd = (ChangeAgentCommand)cmds[0];
 
                 logger.debug("Intercepting command for agent change: agent {} event: {}", cmd.getAgentId(), cmd.getEvent());
-                boolean result = false;
+                boolean result;
                 try {
                     result = executeAgentUserRequest(cmd.getAgentId(), cmd.getEvent());
                     logger.debug("Result is {}", result);
@@ -1253,10 +1272,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             } else if (cmds.length == 1 && cmds[0] instanceof TransferAgentCommand) {
                 final TransferAgentCommand cmd = (TransferAgentCommand)cmds[0];
 
-                logger.debug("Intercepting command for agent rebalancing: agent {} event: {}", cmd.getAgentId(), cmd.getEvent());
-                boolean result = false;
+                logger.debug("Intercepting command for agent rebalancing: agent: {}, event: {}, connection transfer: {}", cmd.getAgentId(), cmd.getEvent(), cmd.isConnectionTransfer());
+                boolean result;
                 try {
-                    result = rebalanceAgent(cmd.getAgentId(), cmd.getEvent(), cmd.getCurrentOwner(), cmd.getFutureOwner());
+                    result = rebalanceAgent(cmd.getAgentId(), cmd.getEvent(), cmd.getCurrentOwner(), cmd.getFutureOwner(), cmd.isConnectionTransfer());
                     logger.debug("Result is {}", result);
 
                 } catch (final AgentUnavailableException e) {
@@ -1274,7 +1293,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
 
                 logger.debug("Intercepting command to propagate event {} for host {} ({})", () -> cmd.getEvent().name(), cmd::getHostId, () -> _hostDao.findById(cmd.getHostId()));
 
-                boolean result = false;
+                boolean result;
                 try {
                     result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent());
                     logger.debug("Result is {}", result);
@@ -1320,10 +1339,28 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         }
 
         private String handleShutdownManagementServerHostCommand(BaseShutdownManagementServerHostCommand cmd) {
-            if (cmd instanceof PrepareForShutdownManagementServerHostCommand) {
-                logger.debug("Received BaseShutdownManagementServerHostCommand - preparing to shut down");
+            if (cmd instanceof PrepareForMaintenanceManagementServerHostCommand) {
+                logger.debug("Received PrepareForMaintenanceManagementServerHostCommand - preparing for maintenance");
                 try {
-                    shutdownManager.prepareForShutdown();
+                    managementServerMaintenanceManager.prepareForMaintenance(((PrepareForMaintenanceManagementServerHostCommand) cmd).getLbAlgorithm());
+                    return "Successfully prepared for maintenance";
+                } catch(CloudRuntimeException e) {
+                    return e.getMessage();
+                }
+            }
+            if (cmd instanceof CancelMaintenanceManagementServerHostCommand) {
+                logger.debug("Received CancelMaintenanceManagementServerHostCommand - cancelling maintenance");
+                try {
+                    managementServerMaintenanceManager.cancelMaintenance();
+                    return "Successfully cancelled maintenance";
+                } catch(CloudRuntimeException e) {
+                    return e.getMessage();
+                }
+            }
+            if (cmd instanceof PrepareForShutdownManagementServerHostCommand) {
+                logger.debug("Received PrepareForShutdownManagementServerHostCommand - preparing to shut down");
+                try {
+                    managementServerMaintenanceManager.prepareForShutdown();
                     return "Successfully prepared for shutdown";
                 } catch(CloudRuntimeException e) {
                     return e.getMessage();
@@ -1332,7 +1369,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             if (cmd instanceof TriggerShutdownManagementServerHostCommand) {
                 logger.debug("Received TriggerShutdownManagementServerHostCommand - triggering a shut down");
                 try {
-                    shutdownManager.triggerShutdown();
+                    managementServerMaintenanceManager.triggerShutdown();
                     return "Successfully triggered shutdown";
                 } catch(CloudRuntimeException e) {
                     return e.getMessage();
@@ -1341,8 +1378,8 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
             if (cmd instanceof CancelShutdownManagementServerHostCommand) {
                 logger.debug("Received CancelShutdownManagementServerHostCommand - cancelling shut down");
                 try {
-                    shutdownManager.cancelShutdown();
-                    return "Successfully prepared for shutdown";
+                    managementServerMaintenanceManager.cancelShutdown();
+                    return "Successfully cancelled shutdown";
                 } catch(CloudRuntimeException e) {
                     return e.getMessage();
                 }
@@ -1351,6 +1388,127 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         }
     }
 
+    @Override
+    public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) {
+        if (timeoutDurationInMs <= 0) {
+            logger.debug("Not transferring direct agents from management server node {} (id: {}) to other nodes, invalid timeout duration", fromMsId, fromMsUuid);
+            return false;
+        }
+
+        long transferStartTime = System.currentTimeMillis();
+        if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId))) {
+            logger.info("No direct agent hosts available on management server node {} (id: {}), to transfer", fromMsId, fromMsUuid);
+            return true;
+        }
+
+        List<ManagementServerHostVO> msHosts = getUpMsHostsExcludingMs(fromMsId);
+        if (msHosts.isEmpty()) {
+            logger.warn("No management server nodes available to transfer agents from management server node {} (id: {})", fromMsId, fromMsUuid);
+            return false;
+        }
+
+        logger.debug("Transferring direct agents from management server node {} (id: {}) to other nodes", fromMsId, fromMsUuid);
+        int agentTransferFailedCount = 0;
+        List<DataCenterVO> dataCenterList = dcDao.listAll();
+        for (DataCenterVO dc : dataCenterList) {
+            List<HostVO> directAgentHostsInDc = getDirectAgentHostsInDc(fromMsId, dc.getId());
+            if (CollectionUtils.isEmpty(directAgentHostsInDc)) {
+                continue;
+            }
+            logger.debug("Transferring {} direct agents from management server node {} (id: {}) of zone {}", directAgentHostsInDc.size(), fromMsId, fromMsUuid, dc);
+            for (HostVO host : directAgentHostsInDc) {
+                long transferElapsedTimeInMs = System.currentTimeMillis() - transferStartTime;
+                if (transferElapsedTimeInMs >= timeoutDurationInMs) {
+                    logger.debug("Stop transferring remaining direct agents from management server node {} (id: {}), timed out", fromMsId, fromMsUuid);
+                    return false;
+                }
+
+                try {
+                    if (_mshostCounter >= msHosts.size()) {
+                        _mshostCounter = 0;
+                    }
+                    ManagementServerHostVO msHost = msHosts.get(_mshostCounter % msHosts.size());
+                    _mshostCounter++;
+
+                    _hostTransferDao.startAgentTransfering(host.getId(), fromMsId, msHost.getMsid());
+                    if (!rebalanceAgent(host.getId(), Event.StartAgentRebalance, fromMsId, msHost.getMsid(), true)) {
+                        agentTransferFailedCount++;
+                    } else {
+                        updateLastManagementServer(host.getId(), fromMsId);
+                    }
+                } catch (Exception e) {
+                    logger.warn("Failed to transfer direct agent of the host {} from management server node {} (id: {}), due to {}", host, fromMsId, fromMsUuid, e.getMessage());
+                }
+            }
+        }
+
+        return (agentTransferFailedCount == 0);
+    }
+
+    private List<HostVO> getDirectAgentHosts(long msId) {
+        List<HostVO> directAgentHosts = new ArrayList<>();
+        List<HostVO> hosts = _hostDao.listHostsByMs(msId);
+        for (HostVO host : hosts) {
+            AgentAttache agent = findAttache(host.getId());
+            if (agent instanceof DirectAgentAttache) {
+                directAgentHosts.add(host);
+            }
+        }
+
+        return directAgentHosts;
+    }
+
+    private List<HostVO> getDirectAgentHostsInDc(long msId, long dcId) {
+        List<HostVO> directAgentHosts = new ArrayList<>();
+        List<HostVO> hosts = _hostDao.listHostsByMsAndDc(msId, dcId);
+        for (HostVO host : hosts) {
+            AgentAttache agent = findAttache(host.getId());
+            if (agent instanceof DirectAgentAttache) {
+                directAgentHosts.add(host);
+            }
+        }
+
+        return directAgentHosts;
+    }
+
+    private List<ManagementServerHostVO> getUpMsHostsExcludingMs(long avoidMsId) {
+        final List<ManagementServerHostVO> msHosts = _mshostDao.listBy(ManagementServerHost.State.Up);
+        msHosts.removeIf(ms -> ms.getMsid() == avoidMsId || _mshostPeerDao.findByPeerMsAndState(ms.getId(), ManagementServerHost.State.Up) == null);
+
+        return msHosts;
+    }
+
+    private void updateLastManagementServer(long hostId, long msId) {
+        HostVO hostVO = _hostDao.findById(hostId);
+        if (hostVO != null) {
+            hostVO.setLastManagementServerId(msId);
+            _hostDao.update(hostId, hostVO);
+        }
+    }
+
+    @Override
+    public void onManagementServerMaintenance() {
+        logger.debug("Management server maintenance enabled");
+        s_transferExecutor.shutdownNow();
+        cleanupTransferMap(_nodeId);
+        _agentLbHappened = false;
+        super.onManagementServerMaintenance();
+    }
+
+    @Override
+    public void onManagementServerCancelMaintenance() {
+        logger.debug("Management server maintenance disabled");
+        super.onManagementServerCancelMaintenance();
+        if (isAgentRebalanceEnabled()) {
+            cleanupTransferMap(_nodeId);
+            if (s_transferExecutor.isShutdown()) {
+                s_transferExecutor = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Cluster-AgentRebalancingExecutor"));
+                s_transferExecutor.scheduleAtFixedRate(getAgentRebalanceScanTask(), 60000, 60000, TimeUnit.MILLISECONDS);
+                s_transferExecutor.scheduleAtFixedRate(getTransferScanTask(), 60000, ClusteredAgentRebalanceService.DEFAULT_TRANSFER_CHECK_INTERVAL, TimeUnit.MILLISECONDS);
+            }
+        }
+    }
+
     public boolean executeAgentUserRequest(final long agentId, final Event event) throws AgentUnavailableException {
         return executeUserRequest(agentId, event);
     }
@@ -1359,6 +1517,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
         return executeRebalanceRequest(agentId, currentOwnerId, futureOwnerId, event);
     }
 
+    public boolean rebalanceAgent(final long agentId, final Event event, final long currentOwnerId, final long futureOwnerId, boolean isConnectionTransfer) throws AgentUnavailableException, OperationTimedoutException {
+        return executeRebalanceRequest(agentId, currentOwnerId, futureOwnerId, event, isConnectionTransfer);
+    }
+
     public boolean isAgentRebalanceEnabled() {
         return EnableLB.value();
     }
@@ -1413,8 +1575,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust
     public ConfigKey<?>[] getConfigKeys() {
         final ConfigKey<?>[] keys = super.getConfigKeys();
 
-        final List<ConfigKey<?>> keysLst = new ArrayList<ConfigKey<?>>();
-        keysLst.addAll(Arrays.asList(keys));
+        final List<ConfigKey<?>> keysLst = new ArrayList<>(Arrays.asList(keys));
         keysLst.add(EnableLB);
         keysLst.add(ConnectedAgentThreshold);
         keysLst.add(LoadSize);

engine/orchestration/src/main/java/com/cloud/cluster/ClusteredAgentRebalanceService.java

@@ -27,4 +27,5 @@ public interface ClusteredAgentRebalanceService {
 
     boolean executeRebalanceRequest(long agentId, long currentOwnerId, long futureOwnerId, Event event) throws AgentUnavailableException, OperationTimedoutException;
 
+    boolean executeRebalanceRequest(long agentId, long currentOwnerId, long futureOwnerId, Event event, boolean isConnectionTransfer) throws AgentUnavailableException, OperationTimedoutException;
 }

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -85,6 +85,7 @@ import org.apache.cloudstack.resource.ResourceCleanupService;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
+import org.apache.cloudstack.utils.cache.SingleCache;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 import org.apache.cloudstack.vm.UnmanagedVMsManager;
@@ -406,6 +407,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     private DomainDao domainDao;
     @Inject
     ResourceCleanupService resourceCleanupService;
+    @Inject
+    VmWorkJobDao vmWorkJobDao;
+
+    private SingleCache<List<Long>> vmIdsInProgressCache;
 
     VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this);
 
@@ -450,6 +455,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
             Long.class, "systemvm.root.disk.size", "-1",
             "Size of root volume (in GB) of system VMs and virtual routers", true);
 
+    private boolean syncTransitioningVmPowerState;
+
     ScheduledExecutorService _executor = null;
 
     private long _nodeId;
@@ -816,6 +823,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
     @Override
     public boolean start() {
+        vmIdsInProgressCache = new SingleCache<>(10, vmWorkJobDao::listVmIdsWithPendingJob);
         _executor.scheduleAtFixedRate(new CleanupTask(), 5, VmJobStateReportInterval.value(), TimeUnit.SECONDS);
         _executor.scheduleAtFixedRate(new TransitionTask(),  VmOpCleanupInterval.value(), VmOpCleanupInterval.value(), TimeUnit.SECONDS);
         cancelWorkItems(_nodeId);
@@ -843,6 +851,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
 
         _messageBus.subscribe(VirtualMachineManager.Topics.VM_POWER_STATE, MessageDispatcher.getDispatcher(this));
 
+        syncTransitioningVmPowerState = Boolean.TRUE.equals(VmSyncPowerStateTransitioning.value());
+
         return true;
     }
 
@@ -3506,7 +3516,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         if (MIGRATE_VM_ACROSS_CLUSTERS.valueIn(host.getDataCenterId()) &&
                 (HypervisorType.VMware.equals(host.getHypervisorType()) || !checkIfVmHasClusterWideVolumes(vm.getId()))) {
             logger.info("Searching for hosts in the zone for vm migration");
-            List<Long> clustersToExclude = _clusterDao.listAllClusters(host.getDataCenterId());
+            List<Long> clustersToExclude = _clusterDao.listAllClusterIds(host.getDataCenterId());
             List<ClusterVO> clusterList = _clusterDao.listByDcHyType(host.getDataCenterId(), host.getHypervisorType().toString());
             for (ClusterVO cluster : clusterList) {
                 clustersToExclude.remove(cluster.getId());
@@ -3800,7 +3810,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 if (ping.getHostVmStateReport() != null) {
                     _syncMgr.processHostVmStatePingReport(agentId, ping.getHostVmStateReport(), ping.getOutOfBand());
                 }
-
                 scanStalledVMInTransitionStateOnUpHost(agentId);
                 processed = true;
             }
@@ -4757,7 +4766,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 VmOpLockStateRetry, VmOpWaitInterval, ExecuteInSequence, VmJobCheckInterval, VmJobTimeout, VmJobStateReportInterval,
                 VmConfigDriveLabel, VmConfigDriveOnPrimaryPool, VmConfigDriveForceHostCacheUse, VmConfigDriveUseHostCacheOnUnsupportedPool,
                 HaVmRestartHostUp, ResourceCountRunningVMsonly, AllowExposeHypervisorHostname, AllowExposeHypervisorHostnameAccountLevel, SystemVmRootDiskSize,
-                AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName
+                AllowExposeDomainInMetadata, MetadataCustomCloudName, VmMetadataManufacturer, VmMetadataProductName,
+                VmSyncPowerStateTransitioning
         };
     }
 
@@ -4955,20 +4965,46 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         }
     }
 
+    /**
+     * Scans stalled VMs in transition states on an UP host and processes them accordingly.
+     *
+     * <p>This method is executed only when the {@code syncTransitioningVmPowerState} flag is enabled. It identifies
+     * VMs stuck in specific states (e.g., Starting, Stopping, Migrating) on a host that is UP, except for those
+     * in the Expunging state, which require special handling.</p>
+     *
+     * <p>The following conditions are checked during the scan:
+     * <ul>
+     *     <li>No pending {@code VmWork} job exists for the VM.</li>
+     *     <li>The VM is associated with the given {@code hostId}, and the host is UP.</li>
+     * </ul>
+     * </p>
+     *
+     * <p>When a host is UP, a state report for the VMs will typically be received. However, certain scenarios
+     * (e.g., out-of-band changes or behavior specific to hypervisors like XenServer or KVM) might result in
+     * missing reports, preventing the state-sync logic from running. To address this, the method scans VMs
+     * based on their last update timestamp. If a VM remains stalled without a status update while its host is UP,
+     * it is assumed to be powered off, which is generally a safe assumption.</p>
+     *
+     * @param hostId the ID of the host to scan for stalled VMs in transition states.
+     */
     private void scanStalledVMInTransitionStateOnUpHost(final long hostId) {
-        final long stallThresholdInMs = VmJobStateReportInterval.value() + (VmJobStateReportInterval.value() >> 1);
-        final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs);
-        final List<Long> mostlikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostId, cutTime);
-        for (final Long vmId : mostlikelyStoppedVMs) {
-            final VMInstanceVO vm = _vmDao.findById(vmId);
-            assert vm != null;
+        if (!syncTransitioningVmPowerState) {
+            return;
+        }
+        if (!_hostDao.isHostUp(hostId)) {
+            return;
+        }
+        final long stallThresholdInMs = VmJobStateReportInterval.value() * 2;
+        final long cutTime = new Date(DateUtil.currentGMTTime().getTime() - stallThresholdInMs).getTime();
+        final List<VMInstanceVO> hostTransitionVms = _vmDao.listByHostAndState(hostId, State.Starting, State.Stopping, State.Migrating);
+
+        final List<VMInstanceVO> mostLikelyStoppedVMs = listStalledVMInTransitionStateOnUpHost(hostTransitionVms, cutTime);
+        for (final VMInstanceVO vm : mostLikelyStoppedVMs) {
             handlePowerOffReportWithNoPendingJobsOnVM(vm);
         }
 
-        final List<Long> vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostId, cutTime);
-        for (final Long vmId : vmsWithRecentReport) {
-            final VMInstanceVO vm = _vmDao.findById(vmId);
-            assert vm != null;
+        final List<VMInstanceVO> vmsWithRecentReport = listVMInTransitionStateWithRecentReportOnUpHost(hostTransitionVms, cutTime);
+        for (final VMInstanceVO vm : vmsWithRecentReport) {
             if (vm.getPowerState() == PowerState.PowerOn) {
                 handlePowerOnReportWithNoPendingJobsOnVM(vm);
             } else {
@@ -4977,6 +5013,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         }
     }
 
+
     private void scanStalledVMInTransitionStateOnDisconnectedHosts() {
         final Date cutTime = new Date(DateUtil.currentGMTTime().getTime() - VmOpWaitInterval.value() * 1000);
         final List<Long> stuckAndUncontrollableVMs = listStalledVMInTransitionStateOnDisconnectedHosts(cutTime);
@@ -4989,89 +5026,58 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
         }
     }
 
-    private List<Long> listStalledVMInTransitionStateOnUpHost(final long hostId, final Date cutTime) {
-        final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
-                "AND h.id = ? AND i.power_state_update_time < ? AND i.host_id = h.id " +
-                "AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
-                "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
-                "AND i.removed IS NULL";
-
-        final List<Long> l = new ArrayList<>();
-        try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
-            String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
-
-            try {
-                PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
-
-                pstmt.setLong(1, hostId);
-                pstmt.setString(2, cutTimeStr);
-                pstmt.setInt(3, JobInfo.Status.IN_PROGRESS.ordinal());
-                final ResultSet rs = pstmt.executeQuery();
-                while (rs.next()) {
-                    l.add(rs.getLong(1));
-                }
-            } catch (SQLException e) {
-                logger.error("Unable to execute SQL [{}] with params {\"h.id\": {}, \"i.power_state_update_time\": \"{}\"} due to [{}].", sql, hostId, cutTimeStr, e.getMessage(), e);
-            }
+    private List<VMInstanceVO> listStalledVMInTransitionStateOnUpHost(
+            final List<VMInstanceVO> transitioningVms, final long cutTime) {
+        if (CollectionUtils.isEmpty(transitioningVms)) {
+            return transitioningVms;
         }
-        return l;
+        List<Long> vmIdsInProgress = vmIdsInProgressCache.get();
+        return transitioningVms.stream()
+                .filter(v -> v.getPowerStateUpdateTime().getTime() < cutTime && !vmIdsInProgress.contains(v.getId()))
+                .collect(Collectors.toList());
     }
 
-    private List<Long> listVMInTransitionStateWithRecentReportOnUpHost(final long hostId, final Date cutTime) {
-        final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status = 'UP' " +
-                "AND h.id = ? AND i.power_state_update_time > ? AND i.host_id = h.id " +
-                "AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
-                "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
-                "AND i.removed IS NULL";
-
-        final List<Long> l = new ArrayList<>();
-        try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
-            String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
-            int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
-
-            try {
-                PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
-
-                pstmt.setLong(1, hostId);
-                pstmt.setString(2, cutTimeStr);
-                pstmt.setInt(3, jobStatusInProgress);
-                final ResultSet rs = pstmt.executeQuery();
-                while (rs.next()) {
-                    l.add(rs.getLong(1));
-                }
-            } catch (final SQLException e) {
-                logger.error("Unable to execute SQL [{}] with params {\"h.id\": {}, \"i.power_state_update_time\": \"{}\", \"j.job_status\": {}} due to [{}].", sql, hostId, cutTimeStr, jobStatusInProgress, e.getMessage(), e);
-            }
-            return l;
+    private List<VMInstanceVO> listVMInTransitionStateWithRecentReportOnUpHost(
+            final List<VMInstanceVO> transitioningVms, final long cutTime) {
+        if (CollectionUtils.isEmpty(transitioningVms)) {
+            return transitioningVms;
         }
+        List<Long> vmIdsInProgress = vmIdsInProgressCache.get();
+        return transitioningVms.stream()
+                .filter(v -> v.getPowerStateUpdateTime().getTime() > cutTime && !vmIdsInProgress.contains(v.getId()))
+                .collect(Collectors.toList());
     }
 
     private List<Long> listStalledVMInTransitionStateOnDisconnectedHosts(final Date cutTime) {
-        final String sql = "SELECT i.* FROM vm_instance as i, host as h WHERE h.status != 'UP' " +
-                "AND i.power_state_update_time < ? AND i.host_id = h.id " +
-                "AND (i.state ='Starting' OR i.state='Stopping' OR i.state='Migrating') " +
-                "AND i.id NOT IN (SELECT w.vm_instance_id FROM vm_work_job AS w JOIN async_job AS j ON w.id = j.id WHERE j.job_status = ?)" +
-                "AND i.removed IS NULL";
+        final String sql = "SELECT i.* " +
+                "FROM vm_instance AS i " +
+                "INNER JOIN host AS h ON i.host_id = h.id " +
+                "WHERE h.status != 'UP' " +
+                "  AND i.power_state_update_time < ? " +
+                "  AND i.state IN ('Starting', 'Stopping', 'Migrating') " +
+                "  AND i.id NOT IN (SELECT vm_instance_id FROM vm_work_job AS w " +
+                "                    INNER JOIN async_job AS j ON w.id = j.id " +
+                "                    WHERE j.job_status = ?) " +
+                "  AND i.removed IS NULL";
 
         final List<Long> l = new ArrayList<>();
-        try (TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB)) {
-            String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
-            int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        String cutTimeStr = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
+        int jobStatusInProgress = JobInfo.Status.IN_PROGRESS.ordinal();
 
-            try {
-                PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
+        try {
+            PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql);
 
-                pstmt.setString(1, cutTimeStr);
-                pstmt.setInt(2, jobStatusInProgress);
-                final ResultSet rs = pstmt.executeQuery();
-                while (rs.next()) {
-                    l.add(rs.getLong(1));
-                }
-            } catch (final SQLException e) {
-                logger.error("Unable to execute SQL [{}] with params {\"i.power_state_update_time\": \"{}\", \"j.job_status\": {}} due to [{}].", sql, cutTimeStr, jobStatusInProgress, e.getMessage(), e);
+            pstmt.setString(1, cutTimeStr);
+            pstmt.setInt(2, jobStatusInProgress);
+            final ResultSet rs = pstmt.executeQuery();
+            while (rs.next()) {
+                l.add(rs.getLong(1));
             }
-            return l;
+        } catch (final SQLException e) {
+            logger.error("Unable to execute SQL [{}] with params {\"i.power_state_update_time\": \"{}\", \"j.job_status\": {}} due to [{}].", sql, cutTimeStr, jobStatusInProgress, e.getMessage(), e);
         }
+        return l;
     }
 
     public class VmStateSyncOutcome extends OutcomeImpl<VirtualMachine> {
@@ -5953,29 +5959,23 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     }
 
     @Override
-    public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, List<Long> vmIds) {
+    public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, List<Long> vmIds) {
         HashMap<Long, VmStatsEntry> vmStatsById = new HashMap<>();
         if (CollectionUtils.isEmpty(vmIds)) {
             return vmStatsById;
         }
-        Map<Long, VMInstanceVO> vmMap = new HashMap<>();
-        for (Long vmId : vmIds) {
-            vmMap.put(vmId, _vmDao.findById(vmId));
-        }
-        return getVirtualMachineStatistics(hostId, hostName, vmMap);
+        Map<String, Long> vmMap = _vmDao.getNameIdMapForVmIds(vmIds);
+        return getVirtualMachineStatistics(host, vmMap);
     }
 
     @Override
-    public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
+    public HashMap<Long, ? extends VmStats> getVirtualMachineStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
         HashMap<Long, VmStatsEntry> vmStatsById = new HashMap<>();
-        if (MapUtils.isEmpty(vmMap)) {
+        if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
             return vmStatsById;
         }
-        Map<String, Long> vmNames = new HashMap<>();
-        for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
-            vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
-        }
-        Answer answer = _agentMgr.easySend(hostId, new GetVmStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
+        Answer answer = _agentMgr.easySend(host.getId(), new GetVmStatsCommand(
+                new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
         if (answer == null || !answer.getResult()) {
             logger.warn("Unable to obtain VM statistics.");
             return vmStatsById;
@@ -5986,23 +5986,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 return vmStatsById;
             }
             for (Map.Entry<String, VmStatsEntry> entry : vmStatsByName.entrySet()) {
-                vmStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
+                vmStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
             }
         }
         return vmStatsById;
     }
 
     @Override
-    public HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
+    public HashMap<Long, List<? extends VmDiskStats>> getVmDiskStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
         HashMap<Long, List<? extends  VmDiskStats>> vmDiskStatsById = new HashMap<>();
-        if (MapUtils.isEmpty(vmMap)) {
+        if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
             return vmDiskStatsById;
         }
-        Map<String, Long> vmNames = new HashMap<>();
-        for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
-            vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
-        }
-        Answer answer = _agentMgr.easySend(hostId, new GetVmDiskStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
+        Answer answer = _agentMgr.easySend(host.getId(), new GetVmDiskStatsCommand(
+                new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
         if (answer == null || !answer.getResult()) {
             logger.warn("Unable to obtain VM disk statistics.");
             return vmDiskStatsById;
@@ -6013,23 +6010,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 return vmDiskStatsById;
             }
             for (Map.Entry<String, List<VmDiskStatsEntry>> entry: vmDiskStatsByName.entrySet()) {
-                vmDiskStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
+                vmDiskStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
             }
         }
         return vmDiskStatsById;
     }
 
     @Override
-    public HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(long hostId, String hostName, Map<Long, ? extends VirtualMachine> vmMap) {
+    public HashMap<Long, List<? extends VmNetworkStats>> getVmNetworkStatistics(Host host, Map<String, Long> vmInstanceNameIdMap) {
         HashMap<Long, List<? extends VmNetworkStats>> vmNetworkStatsById = new HashMap<>();
-        if (MapUtils.isEmpty(vmMap)) {
+        if (MapUtils.isEmpty(vmInstanceNameIdMap)) {
             return vmNetworkStatsById;
         }
-        Map<String, Long> vmNames = new HashMap<>();
-        for (Map.Entry<Long, ? extends VirtualMachine> vmEntry : vmMap.entrySet()) {
-            vmNames.put(vmEntry.getValue().getInstanceName(), vmEntry.getKey());
-        }
-        Answer answer = _agentMgr.easySend(hostId, new GetVmNetworkStatsCommand(new ArrayList<>(vmNames.keySet()), _hostDao.findById(hostId).getGuid(), hostName));
+        Answer answer = _agentMgr.easySend(host.getId(), new GetVmNetworkStatsCommand(
+                new ArrayList<>(vmInstanceNameIdMap.keySet()), host.getGuid(), host.getName()));
         if (answer == null || !answer.getResult()) {
             logger.warn("Unable to obtain VM network statistics.");
             return vmNetworkStatsById;
@@ -6040,7 +6034,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
                 return vmNetworkStatsById;
             }
             for (Map.Entry<String, List<VmNetworkStatsEntry>> entry: vmNetworkStatsByName.entrySet()) {
-                vmNetworkStatsById.put(vmNames.get(entry.getKey()), entry.getValue());
+                vmNetworkStatsById.put(vmInstanceNameIdMap.get(entry.getKey()), entry.getValue());
             }
         }
         return vmNetworkStatsById;

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java

@@ -16,27 +16,29 @@
 // under the License.
 package com.cloud.vm;
 
-import java.text.SimpleDateFormat;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
-import com.cloud.host.Host;
-import com.cloud.host.HostVO;
-import com.cloud.host.dao.HostDao;
-import com.cloud.utils.Pair;
 import org.apache.cloudstack.framework.messagebus.MessageBus;
 import org.apache.cloudstack.framework.messagebus.PublishScope;
-import org.apache.logging.log4j.Logger;
+import org.apache.cloudstack.utils.cache.LazyCache;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import com.cloud.agent.api.HostVmStateReportEntry;
 import com.cloud.configuration.ManagementServiceConfiguration;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
 import com.cloud.utils.DateUtil;
-import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.dao.VMInstanceDao;
 
 public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStateSync {
@@ -47,7 +49,12 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
     @Inject HostDao hostDao;
     @Inject ManagementServiceConfiguration mgmtServiceConf;
 
+    private LazyCache<Long, VMInstanceVO> vmCache;
+    private LazyCache<Long, HostVO> hostCache;
+
     public VirtualMachinePowerStateSyncImpl() {
+        vmCache = new LazyCache<>(16, 10, this::getVmFromId);
+        hostCache = new LazyCache<>(16, 10, this::getHostFromId);
     }
 
     @Override
@@ -58,130 +65,141 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
 
     @Override
     public void processHostVmStateReport(long hostId, Map<String, HostVmStateReportEntry> report) {
-        HostVO host = hostDao.findById(hostId);
-        logger.debug("Process host VM state report. host: {}", host);
-
-        Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> translatedInfo = convertVmStateReport(report);
-        processReport(host, translatedInfo, false);
+        logger.debug("Process host VM state report. host: {}", hostCache.get(hostId));
+        Map<Long, VirtualMachine.PowerState> translatedInfo = convertVmStateReport(report);
+        processReport(hostId, translatedInfo, false);
     }
 
     @Override
     public void processHostVmStatePingReport(long hostId, Map<String, HostVmStateReportEntry> report, boolean force) {
-        HostVO host = hostDao.findById(hostId);
-        logger.debug("Process host VM state report from ping process. host: {}", host);
-
-        Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> translatedInfo = convertVmStateReport(report);
-        processReport(host, translatedInfo, force);
+        logger.debug("Process host VM state report from ping process. host: {}", hostCache.get(hostId));
+        Map<Long, VirtualMachine.PowerState> translatedInfo = convertVmStateReport(report);
+        processReport(hostId, translatedInfo, force);
     }
 
-    private void processReport(HostVO host, Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> translatedInfo, boolean force) {
-
-        logger.debug("Process VM state report. host: {}, number of records in report: {}.", host, translatedInfo.size());
-
-        for (Map.Entry<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> entry : translatedInfo.entrySet()) {
-
-            logger.debug("VM state report. host: {}, vm: {}, power state: {}", host, entry.getValue().second(), entry.getValue().first());
-
-            if (_instanceDao.updatePowerState(entry.getKey(), host.getId(), entry.getValue().first(), DateUtil.currentGMTTime())) {
-                logger.debug("VM state report is updated. host: {}, vm: {}, power state: {}", host, entry.getValue().second(), entry.getValue().first());
-
-                _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, entry.getKey());
-            } else {
-                logger.trace("VM power state does not change, skip DB writing. vm: {}", entry.getValue().second());
-            }
+    private void updateAndPublishVmPowerStates(long hostId, Map<Long, VirtualMachine.PowerState> instancePowerStates,
+           Date updateTime) {
+        if (instancePowerStates.isEmpty()) {
+            return;
         }
+        Set<Long> vmIds = instancePowerStates.keySet();
+        Map<Long, VirtualMachine.PowerState> notUpdated = _instanceDao.updatePowerState(instancePowerStates, hostId,
+                updateTime);
+        if (notUpdated.size() > vmIds.size()) {
+            return;
+        }
+        for (Long vmId : vmIds) {
+            if (!notUpdated.isEmpty() && !notUpdated.containsKey(vmId)) {
+                logger.debug("VM state report is updated. {}, {}, power state: {}",
+                        () -> hostCache.get(hostId), () -> vmCache.get(vmId), () -> instancePowerStates.get(vmId));
+                _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE,
+                        PublishScope.GLOBAL, vmId);
+                continue;
+            }
+            logger.trace("VM power state does not change, skip DB writing. {}", () -> vmCache.get(vmId));
+        }
+    }
 
+    private List<VMInstanceVO> filterOutdatedFromMissingVmReport(List<VMInstanceVO> vmsThatAreMissingReport) {
+        List<Long> outdatedVms = vmsThatAreMissingReport.stream()
+                .filter(v -> !_instanceDao.isPowerStateUpToDate(v))
+                .map(VMInstanceVO::getId)
+                .collect(Collectors.toList());
+        if (CollectionUtils.isEmpty(outdatedVms)) {
+            return vmsThatAreMissingReport;
+        }
+        _instanceDao.resetVmPowerStateTracking(outdatedVms);
+        return vmsThatAreMissingReport.stream()
+                .filter(v -> !outdatedVms.contains(v.getId()))
+                .collect(Collectors.toList());
+    }
+
+    private void processMissingVmReport(long hostId, Set<Long> vmIds, boolean force) {
         // any state outdates should be checked against the time before this list was retrieved
         Date startTime = DateUtil.currentGMTTime();
         // for all running/stopping VMs, we provide monitoring of missing report
-        List<VMInstanceVO> vmsThatAreMissingReport = _instanceDao.findByHostInStates(host.getId(), VirtualMachine.State.Running,
-                VirtualMachine.State.Stopping, VirtualMachine.State.Starting);
-        java.util.Iterator<VMInstanceVO> it = vmsThatAreMissingReport.iterator();
-        while (it.hasNext()) {
-            VMInstanceVO instance = it.next();
-            if (translatedInfo.get(instance.getId()) != null)
-                it.remove();
+        List<VMInstanceVO> vmsThatAreMissingReport = _instanceDao.findByHostInStatesExcluding(hostId, vmIds,
+                VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Starting);
+        // here we need to be wary of out of band migration as opposed to other, more unexpected state changes
+        if (vmsThatAreMissingReport.isEmpty()) {
+            return;
+        }
+        Date currentTime = DateUtil.currentGMTTime();
+        logger.debug("Run missing VM report. current time: {}", currentTime.getTime());
+        if (!force) {
+            vmsThatAreMissingReport = filterOutdatedFromMissingVmReport(vmsThatAreMissingReport);
         }
 
-        // here we need to be wary of out of band migration as opposed to other, more unexpected state changes
-        if (vmsThatAreMissingReport.size() > 0) {
-            Date currentTime = DateUtil.currentGMTTime();
-            logger.debug("Run missing VM report for host {}. current time: {}", host, currentTime.getTime());
-
-            // 2 times of sync-update interval for graceful period
-            long milliSecondsGracefullPeriod = mgmtServiceConf.getPingInterval() * 2000L;
-
-            for (VMInstanceVO instance : vmsThatAreMissingReport) {
-
-                // Make sure powerState is up to date for missing VMs
-                try {
-                    if (!force && !_instanceDao.isPowerStateUpToDate(instance.getId())) {
-                        logger.warn("Detected missing VM but power state is outdated, wait for another process report run for VM: {}", instance);
-                        _instanceDao.resetVmPowerStateTracking(instance.getId());
-                        continue;
-                    }
-                } catch (CloudRuntimeException e) {
-                    logger.warn("Checked for missing powerstate of a none existing vm {}", instance, e);
-                    continue;
-                }
-
-                Date vmStateUpdateTime = instance.getPowerStateUpdateTime();
+        // 2 times of sync-update interval for graceful period
+        long milliSecondsGracefulPeriod = mgmtServiceConf.getPingInterval() * 2000L;
+        Map<Long, VirtualMachine.PowerState> instancePowerStates = new HashMap<>();
+        for (VMInstanceVO instance : vmsThatAreMissingReport) {
+            Date vmStateUpdateTime = instance.getPowerStateUpdateTime();
+            if (vmStateUpdateTime == null) {
+                logger.warn("VM power state update time is null, falling back to update time for {}", instance);
+                vmStateUpdateTime = instance.getUpdateTime();
                 if (vmStateUpdateTime == null) {
-                    logger.warn("VM power state update time is null, falling back to update time for vm: {}", instance);
-                    vmStateUpdateTime = instance.getUpdateTime();
-                    if (vmStateUpdateTime == null) {
-                        logger.warn("VM update time is null, falling back to creation time for vm: {}", instance);
-                        vmStateUpdateTime = instance.getCreated();
-                    }
-                }
-
-                String lastTime = new SimpleDateFormat("yyyy/MM/dd'T'HH:mm:ss.SSS'Z'").format(vmStateUpdateTime);
-                logger.debug("Detected missing VM. host: {}, vm: {}, power state: {}, last state update: {}",
-                        host, instance, VirtualMachine.PowerState.PowerReportMissing, lastTime);
-
-                long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime();
-
-                if (force || milliSecondsSinceLastStateUpdate > milliSecondsGracefullPeriod) {
-                    logger.debug("vm: {} - time since last state update({}ms) has passed graceful period", instance, milliSecondsSinceLastStateUpdate);
-
-                    // this is were a race condition might have happened if we don't re-fetch the instance;
-                    // between the startime of this job and the currentTime of this missing-branch
-                    // an update might have occurred that we should not override in case of out of band migration
-                    if (_instanceDao.updatePowerState(instance.getId(), host.getId(), VirtualMachine.PowerState.PowerReportMissing, startTime)) {
-                        logger.debug("VM state report is updated. host: {}, vm: {}, power state: PowerReportMissing ", host, instance);
-
-                        _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, PublishScope.GLOBAL, instance.getId());
-                    } else {
-                        logger.debug("VM power state does not change, skip DB writing. vm: {}", instance);
-                    }
-                } else {
-                    logger.debug("vm: {} - time since last state update({} ms) has not passed graceful period yet", instance, milliSecondsSinceLastStateUpdate);
+                    logger.warn("VM update time is null, falling back to creation time for {}", instance);
+                    vmStateUpdateTime = instance.getCreated();
                 }
             }
+            logger.debug("Detected missing VM. host: {}, vm id: {}({}), power state: {}, last state update: {}",
+                    hostId,
+                    instance.getId(),
+                    instance.getUuid(),
+                    VirtualMachine.PowerState.PowerReportMissing,
+                    DateUtil.getOutputString(vmStateUpdateTime));
+            long milliSecondsSinceLastStateUpdate = currentTime.getTime() - vmStateUpdateTime.getTime();
+            if (force || (milliSecondsSinceLastStateUpdate > milliSecondsGracefulPeriod)) {
+                logger.debug("vm id: {} - time since last state update({} ms) has passed graceful period",
+                        instance.getId(), milliSecondsSinceLastStateUpdate);
+                // this is where a race condition might have happened if we don't re-fetch the instance;
+                // between the startime of this job and the currentTime of this missing-branch
+                // an update might have occurred that we should not override in case of out of band migration
+                instancePowerStates.put(instance.getId(), VirtualMachine.PowerState.PowerReportMissing);
+            } else {
+                logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period yet",
+                        instance.getId(), milliSecondsSinceLastStateUpdate);
+            }
         }
-
-        logger.debug("Done with process of VM state report. host: {}", host);
+        updateAndPublishVmPowerStates(hostId, instancePowerStates, startTime);
     }
 
-    public Map<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> convertVmStateReport(Map<String, HostVmStateReportEntry> states) {
-        final HashMap<Long, Pair<VirtualMachine.PowerState, VMInstanceVO>> map = new HashMap<>();
-        if (states == null) {
+    private void processReport(long hostId, Map<Long, VirtualMachine.PowerState> translatedInfo, boolean force) {
+        logger.debug("Process VM state report. {}, number of records in report: {}. VMs: [{}]",
+                () -> hostCache.get(hostId),
+                translatedInfo::size,
+                () -> translatedInfo.entrySet().stream().map(entry -> entry.getKey() + ":" + entry.getValue())
+                        .collect(Collectors.joining(", ")) + "]");
+        updateAndPublishVmPowerStates(hostId, translatedInfo, DateUtil.currentGMTTime());
+
+        processMissingVmReport(hostId, translatedInfo.keySet(), force);
+
+        logger.debug("Done with process of VM state report. host: {}", () -> hostCache.get(hostId));
+    }
+
+    public Map<Long, VirtualMachine.PowerState> convertVmStateReport(Map<String, HostVmStateReportEntry> states) {
+        final HashMap<Long, VirtualMachine.PowerState> map = new HashMap<>();
+        if (MapUtils.isEmpty(states)) {
             return map;
         }
-
+        Map<String, Long> nameIdMap = _instanceDao.getNameIdMapForVmInstanceNames(states.keySet());
         for (Map.Entry<String, HostVmStateReportEntry> entry : states.entrySet()) {
-            VMInstanceVO vm = findVM(entry.getKey());
-            if (vm != null) {
-                map.put(vm.getId(), new Pair<>(entry.getValue().getState(), vm));
+            Long id = nameIdMap.get(entry.getKey());
+            if (id != null) {
+                map.put(id, entry.getValue().getState());
             } else {
                 logger.debug("Unable to find matched VM in CloudStack DB. name: {} powerstate: {}", entry.getKey(), entry.getValue());
             }
         }
-
         return map;
     }
 
-    private VMInstanceVO findVM(String vmName) {
-        return _instanceDao.findVMByInstanceName(vmName);
+    protected VMInstanceVO getVmFromId(long vmId) {
+        return _instanceDao.findById(vmId);
+    }
+
+    protected HostVO getHostFromId(long hostId) {
+        return hostDao.findById(hostId);
     }
 }

engine/orchestration/src/main/java/org/apache/cloudstack/engine/datacenter/entity/api/db/EngineHostVO.java

@@ -372,6 +372,9 @@ public class EngineHostVO implements EngineHost, Identity {
     @Column(name = "mgmt_server_id")
     private Long managementServerId;
 
+    @Column(name = "last_mgmt_server_id")
+    private Long lastManagementServerId;
+
     @Column(name = "dom0_memory")
     private long dom0MinMemory;
 
@@ -556,6 +559,10 @@ public class EngineHostVO implements EngineHost, Identity {
         this.managementServerId = managementServerId;
     }
 
+    public void setLastManagementServerId(Long lastManagementServerId) {
+        this.lastManagementServerId = lastManagementServerId;
+    }
+
     @Override
     public long getLastPinged() {
         return lastPinged;
@@ -625,6 +632,11 @@ public class EngineHostVO implements EngineHost, Identity {
         return managementServerId;
     }
 
+    @Override
+    public Long getLastManagementServerId() {
+        return lastManagementServerId;
+    }
+
     @Override
     public Date getDisconnectedOn() {
         return disconnectedOn;

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -4263,7 +4263,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     @Override
     public void processConnect(final Host host, final StartupCommand cmd, final boolean forRebalance) throws ConnectionException {
-        if (!(cmd instanceof StartupRoutingCommand)) {
+        if (!(cmd instanceof StartupRoutingCommand) || cmd.isConnectionTransferred()) {
             return;
         }
         final long hostId = host.getId();
@@ -4872,7 +4872,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout,
+        return new ConfigKey<?>[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes,
                 GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion,
                 PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled,
                 TUNGSTEN_ENABLED, NSX_ENABLED };

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDao.java

@@ -28,6 +28,8 @@ import com.cloud.utils.db.GenericDao;
 public interface CapacityDao extends GenericDao<CapacityVO, Long> {
     CapacityVO findByHostIdType(Long hostId, short capacityType);
 
+    List<CapacityVO> listByHostIdTypes(Long hostId, List<Short> capacityTypes);
+
     List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone);
 
     List<Long> listHostsWithEnoughCapacity(int requiredCpu, long requiredRam, Long clusterId, String hostType);

engine/schema/src/main/java/com/cloud/capacity/dao/CapacityDaoImpl.java

@@ -671,6 +671,18 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
         return findOneBy(sc);
     }
 
+    @Override
+    public List<CapacityVO> listByHostIdTypes(Long hostId, List<Short> capacityTypes) {
+        SearchBuilder<CapacityVO> sb = createSearchBuilder();
+        sb.and("hostId", sb.entity().getHostOrPoolId(), SearchCriteria.Op.EQ);
+        sb.and("type", sb.entity().getCapacityType(), SearchCriteria.Op.IN);
+        sb.done();
+        SearchCriteria<CapacityVO> sc = sb.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("type", capacityTypes.toArray());
+        return listBy(sc);
+    }
+
     @Override
     public List<Long> listClustersInZoneOrPodByHostCapacities(long id, long vmId, int requiredCpu, long requiredRam, short capacityTypeForOrdering, boolean isZone) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();

engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.dc;
 
+import java.util.Collection;
 import java.util.Map;
 
 import com.cloud.utils.db.GenericDao;
@@ -29,6 +30,8 @@ public interface ClusterDetailsDao extends GenericDao<ClusterDetailsVO, Long> {
 
     ClusterDetailsVO findDetail(long clusterId, String name);
 
+    Map<String, String> findDetails(long clusterId, Collection<String> names);
+
     void deleteDetails(long clusterId);
 
     String getVmwareDcName(Long clusterId);

engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java

@@ -16,13 +16,16 @@
 // under the License.
 package com.cloud.dc;
 
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.ConfigKey.Scope;
 import org.apache.cloudstack.framework.config.ScopedConfigStorage;
+import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.utils.crypt.DBEncryptionUtil;
 import com.cloud.utils.db.GenericDaoBase;
@@ -82,6 +85,23 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase<ClusterDetailsVO, Long
         return details;
     }
 
+    @Override
+    public Map<String, String> findDetails(long clusterId, Collection<String> names) {
+        if (CollectionUtils.isEmpty(names)) {
+            return new HashMap<>();
+        }
+        SearchBuilder<ClusterDetailsVO> sb = createSearchBuilder();
+        sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.and("name", sb.entity().getName(), SearchCriteria.Op.IN);
+        sb.done();
+        SearchCriteria<ClusterDetailsVO> sc = sb.create();
+        sc.setParameters("clusterId", clusterId);
+        sc.setParameters("name", names.toArray());
+        List<ClusterDetailsVO> results = search(sc, null);
+        return results.stream()
+                .collect(Collectors.toMap(ClusterDetailsVO::getName, ClusterDetailsVO::getValue));
+    }
+
     @Override
     public void deleteDetails(long clusterId) {
         SearchCriteria<ClusterDetailsVO> sc = ClusterSearch.create();

engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java

@@ -16,15 +16,15 @@
 // under the License.
 package com.cloud.dc.dao;
 
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import com.cloud.cpu.CPU;
 import com.cloud.dc.ClusterVO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.utils.db.GenericDao;
 
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 public interface ClusterDao extends GenericDao<ClusterVO, Long> {
     List<ClusterVO> listByPodId(long podId);
 
@@ -36,7 +36,7 @@ public interface ClusterDao extends GenericDao<ClusterVO, Long> {
 
     List<HypervisorType> getAvailableHypervisorInZone(Long zoneId);
 
-    Set<HypervisorType> getDistictAvailableHypervisorsAcrossClusters();
+    Set<HypervisorType> getDistinctAvailableHypervisorsAcrossClusters();
 
     List<ClusterVO> listByDcHyType(long dcId, String hyType);
 
@@ -46,9 +46,13 @@ public interface ClusterDao extends GenericDao<ClusterVO, Long> {
 
     List<Long> listClustersWithDisabledPods(long zoneId);
 
+    Integer countAllByDcId(long zoneId);
+
+    Integer countAllManagedAndEnabledByDcId(long zoneId);
+
     List<ClusterVO> listClustersByDcId(long zoneId);
 
-    List<Long> listAllClusters(Long zoneId);
+    List<Long> listAllClusterIds(Long zoneId);
 
     boolean getSupportsResigning(long clusterId);
 

engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java

@@ -16,25 +16,6 @@
 // under the License.
 package com.cloud.dc.dao;
 
-import com.cloud.cpu.CPU;
-import com.cloud.dc.ClusterDetailsDao;
-import com.cloud.dc.ClusterDetailsVO;
-import com.cloud.dc.ClusterVO;
-import com.cloud.dc.HostPodVO;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.org.Grouping;
-import com.cloud.utils.db.GenericDaoBase;
-import com.cloud.utils.db.GenericSearchBuilder;
-import com.cloud.utils.db.JoinBuilder;
-import com.cloud.utils.db.SearchBuilder;
-import com.cloud.utils.db.SearchCriteria;
-import com.cloud.utils.db.SearchCriteria.Func;
-import com.cloud.utils.db.SearchCriteria.Op;
-import com.cloud.utils.db.TransactionLegacy;
-import com.cloud.utils.exception.CloudRuntimeException;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -46,6 +27,28 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import javax.inject.Inject;
+
+import org.springframework.stereotype.Component;
+
+import com.cloud.cpu.CPU;
+import com.cloud.dc.ClusterDetailsDao;
+import com.cloud.dc.ClusterDetailsVO;
+import com.cloud.dc.ClusterVO;
+import com.cloud.dc.HostPodVO;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.org.Grouping;
+import com.cloud.org.Managed;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.GenericSearchBuilder;
+import com.cloud.utils.db.JoinBuilder;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.SearchCriteria.Func;
+import com.cloud.utils.db.SearchCriteria.Op;
+import com.cloud.utils.db.TransactionLegacy;
+import com.cloud.utils.exception.CloudRuntimeException;
+
 @Component
 public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements ClusterDao {
 
@@ -58,7 +61,6 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
     protected final SearchBuilder<ClusterVO> ClusterSearch;
     protected final SearchBuilder<ClusterVO> ClusterDistinctArchSearch;
     protected final SearchBuilder<ClusterVO> ClusterArchSearch;
-
     protected GenericSearchBuilder<ClusterVO, Long> ClusterIdSearch;
 
     private static final String GET_POD_CLUSTER_MAP_PREFIX = "SELECT pod_id, id FROM cloud.cluster WHERE cluster.id IN( ";
@@ -98,6 +100,8 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
 
         ZoneClusterSearch = createSearchBuilder();
         ZoneClusterSearch.and("dataCenterId", ZoneClusterSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        ZoneClusterSearch.and("allocationState", ZoneClusterSearch.entity().getAllocationState(), Op.EQ);
+        ZoneClusterSearch.and("managedState", ZoneClusterSearch.entity().getManagedState(), Op.EQ);
         ZoneClusterSearch.done();
 
         ClusterIdSearch = createSearchBuilder(Long.class);
@@ -167,23 +171,15 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
             sc.setParameters("zoneId", zoneId);
         }
         List<ClusterVO> clusters = listBy(sc);
-        List<HypervisorType> hypers = new ArrayList<HypervisorType>(4);
-        for (ClusterVO cluster : clusters) {
-            hypers.add(cluster.getHypervisorType());
-        }
-
-        return hypers;
+        return clusters.stream()
+                .map(ClusterVO::getHypervisorType)
+                .distinct()
+                .collect(Collectors.toList());
     }
 
     @Override
-    public Set<HypervisorType> getDistictAvailableHypervisorsAcrossClusters() {
-        SearchCriteria<ClusterVO> sc = ClusterSearch.create();
-        List<ClusterVO> clusters = listBy(sc);
-        Set<HypervisorType> hypers = new HashSet<>();
-        for (ClusterVO cluster : clusters) {
-            hypers.add(cluster.getHypervisorType());
-        }
-        return hypers;
+    public Set<HypervisorType> getDistinctAvailableHypervisorsAcrossClusters() {
+        return new HashSet<>(getAvailableHypervisorInZone(null));
     }
 
     @Override
@@ -266,6 +262,23 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
         return customSearch(sc, null);
     }
 
+    @Override
+    public Integer countAllByDcId(long zoneId) {
+        SearchCriteria<ClusterVO> sc = ZoneClusterSearch.create();
+        sc.setParameters("dataCenterId", zoneId);
+        return getCount(sc);
+    }
+
+    @Override
+    public Integer countAllManagedAndEnabledByDcId(long zoneId) {
+        SearchCriteria<ClusterVO> sc = ZoneClusterSearch.create();
+        sc.setParameters("dataCenterId", zoneId);
+        sc.setParameters("allocationState", Grouping.AllocationState.Enabled);
+        sc.setParameters("managedState", Managed.ManagedState.Managed);
+
+        return getCount(sc);
+    }
+
     @Override
     public List<ClusterVO> listClustersByDcId(long zoneId) {
         SearchCriteria<ClusterVO> sc = ZoneClusterSearch.create();
@@ -289,7 +302,7 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
     }
 
     @Override
-    public List<Long> listAllClusters(Long zoneId) {
+    public List<Long> listAllClusterIds(Long zoneId) {
         SearchCriteria<Long> sc = ClusterIdSearch.create();
         if (zoneId != null) {
             sc.setParameters("dataCenterId", zoneId);

engine/schema/src/main/java/com/cloud/dc/dao/DataCenterIpAddressDaoImpl.java

@@ -294,8 +294,7 @@ public class DataCenterIpAddressDaoImpl extends GenericDaoBase<DataCenterIpAddre
         sc.addAnd("podId", SearchCriteria.Op.EQ, podId);
         sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, dcId);
 
-        List<DataCenterIpAddressVO> result = listBy(sc);
-        return result.size();
+        return getCount(sc);
     }
 
     public DataCenterIpAddressDaoImpl() {

engine/schema/src/main/java/com/cloud/dc/dao/DataCenterVnetDaoImpl.java

@@ -81,7 +81,7 @@ public class DataCenterVnetDaoImpl extends GenericDaoBase<DataCenterVnetVO, Long
     public int countAllocatedVnets(long physicalNetworkId) {
         SearchCriteria<DataCenterVnetVO> sc = DcSearchAllocated.create();
         sc.setParameters("physicalNetworkId", physicalNetworkId);
-        return listBy(sc).size();
+        return getCount(sc);
     }
 
     @Override

engine/schema/src/main/java/com/cloud/host/HostVO.java

@@ -404,6 +404,9 @@ public class HostVO implements Host {
     @Column(name = "mgmt_server_id")
     private Long managementServerId;
 
+    @Column(name = "last_mgmt_server_id")
+    private Long lastManagementServerId;
+
     @Column(name = "dom0_memory")
     private long dom0MinMemory;
 
@@ -570,6 +573,10 @@ public class HostVO implements Host {
         this.managementServerId = managementServerId;
     }
 
+    public void setLastManagementServerId(Long lastManagementServerId) {
+        this.lastManagementServerId = lastManagementServerId;
+    }
+
     @Override
     public long getLastPinged() {
         return lastPinged;
@@ -639,6 +646,11 @@ public class HostVO implements Host {
         return managementServerId;
     }
 
+    @Override
+    public Long getLastManagementServerId() {
+        return lastManagementServerId;
+    }
+
     @Override
     public Date getDisconnectedOn() {
         return disconnectedOn;

engine/schema/src/main/java/com/cloud/host/dao/HostDao.java

@@ -27,6 +27,7 @@ import com.cloud.hypervisor.Hypervisor;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.info.RunningHostCountInfo;
 import com.cloud.resource.ResourceState;
+import com.cloud.utils.Pair;
 import com.cloud.utils.db.GenericDao;
 import com.cloud.utils.fsm.StateDao;
 
@@ -39,8 +40,14 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     Integer countAllByType(final Host.Type type);
 
+    Integer countAllInClusterByTypeAndStates(Long clusterId, final Host.Type type, List<Status> status);
+
     Integer countAllByTypeInZone(long zoneId, final Host.Type type);
 
+    Integer countUpAndEnabledHostsInZone(long zoneId);
+
+    Pair<Integer, Integer> countAllHostsAndCPUSocketsByType(Type type);
+
     /**
      * Mark all hosts associated with a certain management server
      * as disconnected.
@@ -75,32 +82,41 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> findHypervisorHostInCluster(long clusterId);
 
+    HostVO findAnyStateHypervisorHostInCluster(long clusterId);
+
     HostVO findOldestExistentHypervisorHostInCluster(long clusterId);
 
     List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag);
 
     List<HostVO> findByDataCenterId(Long zoneId);
 
+    List<Long> listIdsByDataCenterId(Long zoneId);
+
     List<HostVO> findByPodId(Long podId);
 
+    List<Long> listIdsByPodId(Long podId);
+
     List<HostVO> findByClusterId(Long clusterId);
 
+    List<Long> listIdsByClusterId(Long clusterId);
+
+    List<Long> listIdsForUpRouting(Long zoneId, Long podId, Long clusterId);
+
+    List<Long> listIdsByType(Type type);
+
+    List<Long> listIdsForUpEnabledByZoneAndHypervisor(Long zoneId, HypervisorType hypervisorType);
+
     List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId);
 
     /**
-     * Returns hosts that are 'Up' and 'Enabled' from the given Data Center/Zone
+     * Returns host Ids that are 'Up' and 'Enabled' from the given Data Center/Zone
      */
-    List<HostVO> listByDataCenterId(long id);
+    List<Long> listEnabledIdsByDataCenterId(long id);
 
     /**
-     * Returns hosts that are from the given Data Center/Zone and at a given state (e.g. Creating, Enabled, Disabled, etc).
+     * Returns host Ids that are 'Up' and 'Disabled' from the given Data Center/Zone
      */
-    List<HostVO> listByDataCenterIdAndState(long id, ResourceState state);
-
-    /**
-     * Returns hosts that are 'Up' and 'Disabled' from the given Data Center/Zone
-     */
-    List<HostVO> listDisabledByDataCenterId(long id);
+    List<Long> listDisabledIdsByDataCenterId(long id);
 
     List<HostVO> listByDataCenterIdAndHypervisorType(long zoneId, Hypervisor.HypervisorType hypervisorType);
 
@@ -110,8 +126,6 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> listAllHostsThatHaveNoRuleTag(Host.Type type, Long clusterId, Long podId, Long dcId);
 
-    List<HostVO> listAllHostsByType(Host.Type type);
-
     HostVO findByPublicIp(String publicIp);
 
     List<Long> listClustersByHostTag(String hostTagOnOffering);
@@ -151,12 +165,23 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> listHostsWithActiveVMs(long offeringId);
 
+    List<HostVO> listHostsByMsAndDc(long msId, long dcId);
+
+    List<HostVO> listHostsByMs(long msId);
+
     /**
      * Retrieves the number of hosts/agents this {@see ManagementServer} has responsibility over.
-     * @param msid the id of the {@see ManagementServer}
+     * @param msId the id of the {@see ManagementServer}
      * @return the number of hosts/agents this {@see ManagementServer} has responsibility over
      */
-    int countByMs(long msid);
+    int countByMs(long msId);
+
+    /**
+     * Retrieves the host ids/agents this {@see ManagementServer} has responsibility over.
+     * @param msId the id of the {@see ManagementServer}
+     * @return the host ids/agents this {@see ManagementServer} has responsibility over
+     */
+    List<String> listByMs(long msId);
 
     /**
      * Retrieves the hypervisor versions of the hosts in the datacenter which are in Up state in ascending order
@@ -171,4 +196,14 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
     List<Long> findClustersThatMatchHostTagRule(String computeOfferingTags);
 
     List<Long> listSsvmHostsWithPendingMigrateJobsOrderedByJobCount();
+
+    boolean isHostUp(long hostId);
+
+    List<Long> findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long zoneId, final Long clusterId,
+            final List<ResourceState> resourceStates, final List<Type> types,
+            final List<Hypervisor.HypervisorType> hypervisorTypes);
+
+    List<HypervisorType> listDistinctHypervisorTypes(final Long zoneId);
+
+    List<HostVO> listByIds(final List<Long> ids);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java

@@ -20,6 +20,7 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -45,8 +46,8 @@ import com.cloud.dc.ClusterVO;
 import com.cloud.dc.dao.ClusterDao;
 import com.cloud.gpu.dao.HostGpuGroupsDao;
 import com.cloud.gpu.dao.VGPUTypesDao;
-import com.cloud.host.Host;
 import com.cloud.host.DetailVO;
+import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
 import com.cloud.host.HostTagVO;
 import com.cloud.host.HostVO;
@@ -59,6 +60,8 @@ import com.cloud.org.Grouping;
 import com.cloud.org.Managed;
 import com.cloud.resource.ResourceState;
 import com.cloud.utils.DateUtil;
+import com.cloud.utils.Pair;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.db.Attribute;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.Filter;
@@ -74,19 +77,17 @@ import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.UpdateBuilder;
 import com.cloud.utils.exception.CloudRuntimeException;
 
-import java.util.Arrays;
-
 @DB
 @TableGenerator(name = "host_req_sq", table = "op_host", pkColumnName = "id", valueColumnName = "sequence", allocationSize = 1)
 public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao { //FIXME: , ExternalIdDao {
 
-    private static final String LIST_HOST_IDS_BY_COMPUTETAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count "
-                                                             + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag) AS filtered "
-                                                             + "WHERE tag IN(%s) AND is_tag_a_rule = 0 "
+    private static final String LIST_HOST_IDS_BY_HOST_TAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count "
+                                                             + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag,is_tag_a_rule) AS filtered "
+                                                             + "WHERE tag IN (%s) AND (is_tag_a_rule = 0 OR is_tag_a_rule IS NULL) "
                                                              + "GROUP BY host_id "
                                                              + "HAVING tag_count = %s ";
     private static final String SEPARATOR = ",";
-    private static final String LIST_CLUSTERID_FOR_HOST_TAG = "select distinct cluster_id from host join ( %s ) AS selected_hosts ON host.id = selected_hosts.host_id";
+    private static final String LIST_CLUSTER_IDS_FOR_HOST_TAGS = "select distinct cluster_id from host join ( %s ) AS selected_hosts ON host.id = selected_hosts.host_id";
     private static final String GET_HOSTS_OF_ACTIVE_VMS = "select h.id " +
             "from vm_instance vm " +
             "join host h on (vm.host_id=h.id) " +
@@ -98,6 +99,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     protected SearchBuilder<HostVO> TypePodDcStatusSearch;
 
+    protected SearchBuilder<HostVO> IdsSearch;
     protected SearchBuilder<HostVO> IdStatusSearch;
     protected SearchBuilder<HostVO> TypeDcSearch;
     protected SearchBuilder<HostVO> TypeDcStatusSearch;
@@ -124,7 +126,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     protected SearchBuilder<HostVO> UnmanagedApplianceSearch;
     protected SearchBuilder<HostVO> MaintenanceCountSearch;
     protected SearchBuilder<HostVO> HostTypeCountSearch;
-    protected SearchBuilder<HostVO> ResponsibleMsCountSearch;
+    protected SearchBuilder<HostVO> ResponsibleMsSearch;
+    protected SearchBuilder<HostVO> ResponsibleMsDcSearch;
+    protected GenericSearchBuilder<HostVO, String> ResponsibleMsIdSearch;
+    protected SearchBuilder<HostVO> HostTypeClusterCountSearch;
     protected SearchBuilder<HostVO> HostTypeZoneCountSearch;
     protected SearchBuilder<HostVO> ClusterStatusSearch;
     protected SearchBuilder<HostVO> TypeNameZoneSearch;
@@ -136,8 +141,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     protected SearchBuilder<HostVO> ManagedRoutingServersSearch;
     protected SearchBuilder<HostVO> SecondaryStorageVMSearch;
 
-    protected GenericSearchBuilder<HostVO, Long> HostIdSearch;
-    protected GenericSearchBuilder<HostVO, Long> HostsInStatusSearch;
+    protected GenericSearchBuilder<HostVO, Long> HostsInStatusesSearch;
     protected GenericSearchBuilder<HostVO, Long> CountRoutingByDc;
     protected SearchBuilder<HostTransferMapVO> HostTransferSearch;
     protected SearchBuilder<ClusterVO> ClusterManagedSearch;
@@ -187,11 +191,30 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
         HostTypeCountSearch = createSearchBuilder();
         HostTypeCountSearch.and("type", HostTypeCountSearch.entity().getType(), SearchCriteria.Op.EQ);
+        HostTypeCountSearch.and("zoneId", HostTypeCountSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        HostTypeCountSearch.and("resourceState", HostTypeCountSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
         HostTypeCountSearch.done();
 
-        ResponsibleMsCountSearch = createSearchBuilder();
-        ResponsibleMsCountSearch.and("managementServerId", ResponsibleMsCountSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ);
-        ResponsibleMsCountSearch.done();
+        ResponsibleMsSearch = createSearchBuilder();
+        ResponsibleMsSearch.and("managementServerId", ResponsibleMsSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ);
+        ResponsibleMsSearch.done();
+
+        ResponsibleMsDcSearch = createSearchBuilder();
+        ResponsibleMsDcSearch.and("managementServerId", ResponsibleMsDcSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ);
+        ResponsibleMsDcSearch.and("dcId", ResponsibleMsDcSearch.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        ResponsibleMsDcSearch.done();
+
+        ResponsibleMsIdSearch = createSearchBuilder(String.class);
+        ResponsibleMsIdSearch.selectFields(ResponsibleMsIdSearch.entity().getUuid());
+        ResponsibleMsIdSearch.and("managementServerId", ResponsibleMsIdSearch.entity().getManagementServerId(), SearchCriteria.Op.EQ);
+        ResponsibleMsIdSearch.done();
+
+        HostTypeClusterCountSearch = createSearchBuilder();
+        HostTypeClusterCountSearch.and("cluster", HostTypeClusterCountSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
+        HostTypeClusterCountSearch.and("type", HostTypeClusterCountSearch.entity().getType(), SearchCriteria.Op.EQ);
+        HostTypeClusterCountSearch.and("status", HostTypeClusterCountSearch.entity().getStatus(), SearchCriteria.Op.IN);
+        HostTypeClusterCountSearch.and("removed", HostTypeClusterCountSearch.entity().getRemoved(), SearchCriteria.Op.NULL);
+        HostTypeClusterCountSearch.done();
 
         HostTypeZoneCountSearch = createSearchBuilder();
         HostTypeZoneCountSearch.and("type", HostTypeZoneCountSearch.entity().getType(), SearchCriteria.Op.EQ);
@@ -240,6 +263,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         TypeClusterStatusSearch.and("resourceState", TypeClusterStatusSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
         TypeClusterStatusSearch.done();
 
+        IdsSearch = createSearchBuilder();
+        IdsSearch.and("id", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
+        IdsSearch.done();
+
         IdStatusSearch = createSearchBuilder();
         IdStatusSearch.and("id", IdStatusSearch.entity().getId(), SearchCriteria.Op.EQ);
         IdStatusSearch.and("states", IdStatusSearch.entity().getStatus(), SearchCriteria.Op.IN);
@@ -386,14 +413,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         AvailHypevisorInZone.groupBy(AvailHypevisorInZone.entity().getHypervisorType());
         AvailHypevisorInZone.done();
 
-        HostsInStatusSearch = createSearchBuilder(Long.class);
-        HostsInStatusSearch.selectFields(HostsInStatusSearch.entity().getId());
-        HostsInStatusSearch.and("dc", HostsInStatusSearch.entity().getDataCenterId(), Op.EQ);
-        HostsInStatusSearch.and("pod", HostsInStatusSearch.entity().getPodId(), Op.EQ);
-        HostsInStatusSearch.and("cluster", HostsInStatusSearch.entity().getClusterId(), Op.EQ);
-        HostsInStatusSearch.and("type", HostsInStatusSearch.entity().getType(), Op.EQ);
-        HostsInStatusSearch.and("statuses", HostsInStatusSearch.entity().getStatus(), Op.IN);
-        HostsInStatusSearch.done();
+        HostsInStatusesSearch = createSearchBuilder(Long.class);
+        HostsInStatusesSearch.selectFields(HostsInStatusesSearch.entity().getId());
+        HostsInStatusesSearch.and("dc", HostsInStatusesSearch.entity().getDataCenterId(), Op.EQ);
+        HostsInStatusesSearch.and("pod", HostsInStatusesSearch.entity().getPodId(), Op.EQ);
+        HostsInStatusesSearch.and("cluster", HostsInStatusesSearch.entity().getClusterId(), Op.EQ);
+        HostsInStatusesSearch.and("type", HostsInStatusesSearch.entity().getType(), Op.EQ);
+        HostsInStatusesSearch.and("statuses", HostsInStatusesSearch.entity().getStatus(), Op.IN);
+        HostsInStatusesSearch.done();
 
         CountRoutingByDc = createSearchBuilder(Long.class);
         CountRoutingByDc.select(null, Func.COUNT, null);
@@ -456,11 +483,6 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         HostsInClusterSearch.and("server", HostsInClusterSearch.entity().getManagementServerId(), SearchCriteria.Op.NNULL);
         HostsInClusterSearch.done();
 
-        HostIdSearch = createSearchBuilder(Long.class);
-        HostIdSearch.selectFields(HostIdSearch.entity().getId());
-        HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ);
-        HostIdSearch.done();
-
         searchBuilderFindByRuleTag = _hostTagsDao.createSearchBuilder();
         searchBuilderFindByRuleTag.and("is_tag_a_rule", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.EQ);
         searchBuilderFindByRuleTag.or("tagDoesNotExist", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.NULL);
@@ -492,8 +514,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         sc.setParameters("resourceState", (Object[])states);
         sc.setParameters("cluster", clusterId);
 
-        List<HostVO> hosts = listBy(sc);
-        return hosts.size();
+        return getCount(sc);
     }
 
     @Override
@@ -504,36 +525,62 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     }
 
     @Override
-    public Integer countAllByTypeInZone(long zoneId, Type type) {
-        SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
-        sc.setParameters("type", type);
-        sc.setParameters("dc", zoneId);
+    public Integer countAllInClusterByTypeAndStates(Long clusterId, final Host.Type type, List<Status> status) {
+        SearchCriteria<HostVO> sc = HostTypeClusterCountSearch.create();
+        if (clusterId != null) {
+            sc.setParameters("cluster", clusterId);
+        }
+        if (type != null) {
+            sc.setParameters("type", type);
+        }
+        if (status != null) {
+            sc.setParameters("status", status.toArray());
+        }
         return getCount(sc);
     }
 
     @Override
-    public List<HostVO> listByDataCenterId(long id) {
-        return listByDataCenterIdAndState(id, ResourceState.Enabled);
+    public Integer countAllByTypeInZone(long zoneId, Type type) {
+        SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
+        sc.setParameters("type", type);
+        sc.setParameters("zoneId", zoneId);
+        return getCount(sc);
     }
 
     @Override
-    public List<HostVO> listByDataCenterIdAndState(long id, ResourceState state) {
-        SearchCriteria<HostVO> sc = scHostsFromZoneUpRouting(id);
-        sc.setParameters("resourceState", state);
-        return listBy(sc);
+    public Integer countUpAndEnabledHostsInZone(long zoneId) {
+        SearchCriteria<HostVO> sc = HostTypeCountSearch.create();
+        sc.setParameters("type", Type.Routing);
+        sc.setParameters("resourceState", ResourceState.Enabled);
+        sc.setParameters("zoneId", zoneId);
+        return getCount(sc);
     }
 
     @Override
-    public List<HostVO> listDisabledByDataCenterId(long id) {
-        return listByDataCenterIdAndState(id, ResourceState.Disabled);
+    public Pair<Integer, Integer> countAllHostsAndCPUSocketsByType(Type type) {
+        GenericSearchBuilder<HostVO, SumCount> sb = createSearchBuilder(SumCount.class);
+        sb.select("sum", Func.SUM, sb.entity().getCpuSockets());
+        sb.select("count", Func.COUNT, null);
+        sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<SumCount> sc = sb.create();
+        sc.setParameters("type", type);
+        SumCount result = customSearch(sc, null).get(0);
+        return new Pair<>((int)result.count, (int)result.sum);
     }
 
-    private SearchCriteria<HostVO> scHostsFromZoneUpRouting(long id) {
-        SearchCriteria<HostVO> sc = DcSearch.create();
-        sc.setParameters("dc", id);
-        sc.setParameters("status", Status.Up);
-        sc.setParameters("type", Host.Type.Routing);
-        return sc;
+    private List<Long> listIdsForRoutingByZoneIdAndResourceState(long zoneId, ResourceState state) {
+        return listIdsBy(Type.Routing, Status.Up, state, null, zoneId, null, null);
+    }
+
+    @Override
+    public List<Long> listEnabledIdsByDataCenterId(long id) {
+        return listIdsForRoutingByZoneIdAndResourceState(id, ResourceState.Enabled);
+    }
+
+    @Override
+    public List<Long> listDisabledIdsByDataCenterId(long id) {
+        return listIdsForRoutingByZoneIdAndResourceState(id, ResourceState.Disabled);
     }
 
     @Override
@@ -591,9 +638,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
             sb.append(" ");
         }
 
-        if (logger.isTraceEnabled()) {
-            logger.trace("Following hosts got reset: " + sb.toString());
-        }
+        logger.trace("Following hosts got reset: {}", sb);
     }
 
     /*
@@ -603,8 +648,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         SearchCriteria<Long> sc = ClustersOwnedByMSSearch.create();
         sc.setParameters("server", managementServerId);
 
-        List<Long> clusters = customSearch(sc, null);
-        return clusters;
+        return customSearch(sc, null);
     }
 
     /*
@@ -614,13 +658,11 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         SearchCriteria<Long> sc = ClustersForHostsNotOwnedByAnyMSSearch.create();
         sc.setJoinParameters("ClusterManagedSearch", "managed", Managed.ManagedState.Managed);
 
-        List<Long> clusters = customSearch(sc, null);
-        return clusters;
+        return customSearch(sc, null);
     }
 
     /**
      * This determines if hosts belonging to cluster(@clusterId) are up for grabs
-     *
      * This is used for handling following cases:
      * 1. First host added in cluster
      * 2. During MS restart all hosts in a cluster are without any MS
@@ -630,9 +672,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         sc.setParameters("cluster", clusterId);
 
         List<HostVO> hosts = search(sc, null);
-        boolean ownCluster = (hosts == null || hosts.size() == 0);
-
-        return ownCluster;
+        return (hosts == null || hosts.isEmpty());
     }
 
     @Override
@@ -649,14 +689,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
             logger.debug("Completed resetting hosts suitable for reconnect");
         }
 
-        List<HostVO> assignedHosts = new ArrayList<HostVO>();
+        List<HostVO> assignedHosts = new ArrayList<>();
 
         if (logger.isDebugEnabled()) {
             logger.debug("Acquiring hosts for clusters already owned by this management server");
         }
         List<Long> clusters = findClustersOwnedByManagementServer(managementServerId);
         txn.start();
-        if (clusters.size() > 0) {
+        if (!clusters.isEmpty()) {
             // handle clusters already owned by @managementServerId
             SearchCriteria<HostVO> sc = UnmanagedDirectConnectSearch.create();
             sc.setParameters("lastPinged", lastPingSecondsAfter);
@@ -671,13 +711,9 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
                 sb.append(host.getId());
                 sb.append(" ");
             }
-            if (logger.isTraceEnabled()) {
-                logger.trace("Following hosts got acquired for clusters already owned: " + sb.toString());
-            }
-        }
-        if (logger.isDebugEnabled()) {
-            logger.debug("Completed acquiring hosts for clusters already owned by this management server");
+            logger.trace("Following hosts got acquired for clusters already owned: {}", sb);
         }
+        logger.debug("Completed acquiring hosts for clusters already owned by this management server");
 
         if (assignedHosts.size() < limit) {
             if (logger.isDebugEnabled()) {
@@ -689,7 +725,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
             if (clusters.size() > limit) {
                 updatedClusters = clusters.subList(0, limit.intValue());
             }
-            if (updatedClusters.size() > 0) {
+            if (!updatedClusters.isEmpty()) {
                 SearchCriteria<HostVO> sc = UnmanagedDirectConnectSearch.create();
                 sc.setParameters("lastPinged", lastPingSecondsAfter);
                 sc.setJoinParameters("ClusterManagedSearch", "managed", Managed.ManagedState.Managed);
@@ -697,10 +733,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
                 List<HostVO> unmanagedHosts = lockRows(sc, null, true);
 
                 // group hosts based on cluster
-                Map<Long, List<HostVO>> hostMap = new HashMap<Long, List<HostVO>>();
+                Map<Long, List<HostVO>> hostMap = new HashMap<>();
                 for (HostVO host : unmanagedHosts) {
                     if (hostMap.get(host.getClusterId()) == null) {
-                        hostMap.put(host.getClusterId(), new ArrayList<HostVO>());
+                        hostMap.put(host.getClusterId(), new ArrayList<>());
                     }
                     hostMap.get(host.getClusterId()).add(host);
                 }
@@ -721,13 +757,9 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
                         break;
                     }
                 }
-                if (logger.isTraceEnabled()) {
-                    logger.trace("Following hosts got acquired from newly owned clusters: " + sb.toString());
-                }
-            }
-            if (logger.isDebugEnabled()) {
-                logger.debug("Completed acquiring hosts for clusters not owned by any management server");
+                logger.trace("Following hosts got acquired from newly owned clusters: {}", sb);
             }
+            logger.debug("Completed acquiring hosts for clusters not owned by any management server");
         }
         txn.commit();
 
@@ -782,6 +814,15 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     @Override
     public List<HostVO> listByHostTag(Host.Type type, Long clusterId, Long podId, Long dcId, String hostTag) {
+        return listHostsWithOrWithoutHostTags(type, clusterId, podId, dcId, hostTag, true);
+    }
+
+    private List<HostVO> listHostsWithOrWithoutHostTags(Host.Type type, Long clusterId, Long podId, Long dcId, String hostTags, boolean withHostTags) {
+        if (StringUtils.isEmpty(hostTags)) {
+            logger.debug("Host tags not specified, to list hosts");
+            return new ArrayList<>();
+        }
+
         SearchBuilder<HostVO> hostSearch = createSearchBuilder();
         HostVO entity = hostSearch.entity();
         hostSearch.and("type", entity.getType(), SearchCriteria.Op.EQ);
@@ -792,7 +833,9 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         hostSearch.and("resourceState", entity.getResourceState(), SearchCriteria.Op.EQ);
 
         SearchCriteria<HostVO> sc = hostSearch.create();
-        sc.setParameters("type", type.toString());
+        if (type != null) {
+            sc.setParameters("type", type.toString());
+        }
         if (podId != null) {
             sc.setParameters("pod", podId);
         }
@@ -805,27 +848,38 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         sc.setParameters("status", Status.Up.toString());
         sc.setParameters("resourceState", ResourceState.Enabled.toString());
 
-        List<HostVO> tmpHosts = listBy(sc);
-        List<HostVO> correctHostsByHostTags = new ArrayList();
-        List<Long> hostIdsByComputeOffTags = findHostByComputeOfferings(hostTag);
+        List<HostVO> upAndEnabledHosts = listBy(sc);
+        if (CollectionUtils.isEmpty(upAndEnabledHosts)) {
+            return new ArrayList<>();
+        }
 
-        tmpHosts.forEach((host) -> { if(hostIdsByComputeOffTags.contains(host.getId())) correctHostsByHostTags.add(host);});
+        List<Long> hostIdsByHostTags = findHostIdsByHostTags(hostTags);
+        if (CollectionUtils.isEmpty(hostIdsByHostTags)) {
+            return withHostTags ? new ArrayList<>() : upAndEnabledHosts;
+        }
 
-        return correctHostsByHostTags;
+        if (withHostTags) {
+            List<HostVO> upAndEnabledHostsWithHostTags = new ArrayList<>();
+            upAndEnabledHosts.forEach((host) -> { if (hostIdsByHostTags.contains(host.getId())) upAndEnabledHostsWithHostTags.add(host);});
+            return upAndEnabledHostsWithHostTags;
+        } else {
+            List<HostVO> upAndEnabledHostsWithoutHostTags = new ArrayList<>();
+            upAndEnabledHosts.forEach((host) -> { if (!hostIdsByHostTags.contains(host.getId())) upAndEnabledHostsWithoutHostTags.add(host);});
+            return upAndEnabledHostsWithoutHostTags;
+        }
     }
 
     @Override
     public List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag) {
+        if (StringUtils.isNotEmpty(haTag)) {
+            return listHostsWithOrWithoutHostTags(type, clusterId, podId, dcId, haTag, false);
+        }
+
         SearchBuilder<HostTagVO> hostTagSearch = _hostTagsDao.createSearchBuilder();
         hostTagSearch.and();
         hostTagSearch.op("isTagARule", hostTagSearch.entity().getIsTagARule(), Op.EQ);
         hostTagSearch.or("tagDoesNotExist", hostTagSearch.entity().getIsTagARule(), Op.NULL);
         hostTagSearch.cp();
-        if (haTag != null && !haTag.isEmpty()) {
-            hostTagSearch.and().op("tag", hostTagSearch.entity().getTag(), SearchCriteria.Op.NEQ);
-            hostTagSearch.or("tagNull", hostTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
-            hostTagSearch.cp();
-        }
 
         SearchBuilder<HostVO> hostSearch = createSearchBuilder();
 
@@ -836,18 +890,12 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         hostSearch.and("status", hostSearch.entity().getStatus(), SearchCriteria.Op.EQ);
         hostSearch.and("resourceState", hostSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
 
-
         hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER);
 
-
         SearchCriteria<HostVO> sc = hostSearch.create();
 
         sc.setJoinParameters("hostTagSearch", "isTagARule", false);
 
-        if (haTag != null && !haTag.isEmpty()) {
-            sc.setJoinParameters("hostTagSearch", "tag", haTag);
-        }
-
         if (type != null) {
             sc.setParameters("type", type);
         }
@@ -887,12 +935,12 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     @DB
     @Override
     public List<HostVO> findLostHosts(long timeout) {
-        List<HostVO> result = new ArrayList<HostVO>();
+        List<HostVO> result = new ArrayList<>();
         String sql = "select h.id from host h left join  cluster c on h.cluster_id=c.id where h.mgmt_server_id is not null and h.last_ping < ? and h.status in ('Up', 'Updating', 'Disconnected', 'Connecting') and h.type not in ('ExternalFirewall', 'ExternalLoadBalancer', 'TrafficMonitor', 'SecondaryStorage', 'LocalSecondaryStorage', 'L2Networking') and (h.cluster_id is null or c.managed_state = 'Managed') ;";
         try (TransactionLegacy txn = TransactionLegacy.currentTxn();
-                PreparedStatement pstmt = txn.prepareStatement(sql);) {
+                PreparedStatement pstmt = txn.prepareStatement(sql)) {
             pstmt.setLong(1, timeout);
-            try (ResultSet rs = pstmt.executeQuery();) {
+            try (ResultSet rs = pstmt.executeQuery()) {
                 while (rs.next()) {
                     long id = rs.getLong(1); //ID column
                     result.add(findById(id));
@@ -925,7 +973,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails = host.getGpuGroupDetails();
         if (groupDetails != null) {
             // Create/Update GPU group entries
-            _hostGpuGroupsDao.persist(host.getId(), new ArrayList<String>(groupDetails.keySet()));
+            _hostGpuGroupsDao.persist(host.getId(), new ArrayList<>(groupDetails.keySet()));
             // Create/Update VGPU types entries
             _vgpuTypesDao.persist(host.getId(), groupDetails);
         }
@@ -968,7 +1016,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
         boolean persisted = super.update(hostId, host);
         if (!persisted) {
-            return persisted;
+            return false;
         }
 
         saveDetails(host);
@@ -977,7 +1025,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
         txn.commit();
 
-        return persisted;
+        return true;
     }
 
     @Override
@@ -988,11 +1036,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
                 + "select h.data_center_id, h.type, count(*) as count from host as h INNER JOIN mshost as m ON h.mgmt_server_id=m.msid "
                 + "where h.status='Up' and h.type='Routing' and m.last_update > ? " + "group by h.data_center_id, h.type) as t " + "ORDER by t.data_center_id, t.type";
 
-        ArrayList<RunningHostCountInfo> l = new ArrayList<RunningHostCountInfo>();
+        ArrayList<RunningHostCountInfo> l = new ArrayList<>();
 
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        ;
-        PreparedStatement pstmt = null;
+        PreparedStatement pstmt;
         try {
             pstmt = txn.prepareAutoCloseStatement(sql);
             String gmtCutTime = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), cutTime);
@@ -1016,9 +1063,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     @Override
     public long getNextSequence(long hostId) {
-        if (logger.isTraceEnabled()) {
-            logger.trace("getNextSequence(), hostId: " + hostId);
-        }
+        logger.trace("getNextSequence(), hostId: {}", hostId);
 
         TableGenerator tg = _tgs.get("host_req_sq");
         assert tg != null : "how can this be wrong!";
@@ -1087,31 +1132,30 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
             HostVO ho = findById(host.getId());
             assert ho != null : "How how how? : " + host.getId();
 
+            // TODO handle this if(debug){}else{log.debug} it makes no sense
             if (logger.isDebugEnabled()) {
-
-                StringBuilder str = new StringBuilder("Unable to update host for event:").append(event.toString());
-                str.append(". Name=").append(host.getName());
-                str.append("; New=[status=").append(newStatus.toString()).append(":msid=").append(newStatus.lostConnection() ? "null" : host.getManagementServerId())
-                .append(":lastpinged=").append(host.getLastPinged()).append("]");
-                str.append("; Old=[status=").append(oldStatus.toString()).append(":msid=").append(host.getManagementServerId()).append(":lastpinged=").append(oldPingTime)
-                .append("]");
-                str.append("; DB=[status=").append(vo.getStatus().toString()).append(":msid=").append(vo.getManagementServerId()).append(":lastpinged=").append(vo.getLastPinged())
-                .append(":old update count=").append(oldUpdateCount).append("]");
-                logger.debug(str.toString());
+                String str = "Unable to update host for event:" + event +
+                        ". Name=" + host.getName() +
+                        "; New=[status=" + newStatus + ":msid=" + (newStatus.lostConnection() ? "null" : host.getManagementServerId()) +
+                        ":lastpinged=" + host.getLastPinged() + "]" +
+                        "; Old=[status=" + oldStatus.toString() + ":msid=" + host.getManagementServerId() + ":lastpinged=" + oldPingTime +
+                        "]" +
+                        "; DB=[status=" + vo.getStatus().toString() + ":msid=" + vo.getManagementServerId() + ":lastpinged=" + vo.getLastPinged() +
+                        ":old update count=" + oldUpdateCount + "]";
+                logger.debug(str);
             } else {
-                StringBuilder msg = new StringBuilder("Agent status update: [");
-                msg.append("id = " + host.getId());
-                msg.append("; name = " + host.getName());
-                msg.append("; old status = " + oldStatus);
-                msg.append("; event = " + event);
-                msg.append("; new status = " + newStatus);
-                msg.append("; old update count = " + oldUpdateCount);
-                msg.append("; new update count = " + newUpdateCount + "]");
-                logger.debug(msg.toString());
+                String msg = "Agent status update: [" + "id = " + host.getId() +
+                        "; name = " + host.getName() +
+                        "; old status = " + oldStatus +
+                        "; event = " + event +
+                        "; new status = " + newStatus +
+                        "; old update count = " + oldUpdateCount +
+                        "; new update count = " + newUpdateCount + "]";
+                logger.debug(msg);
             }
 
             if (ho.getState() == newStatus) {
-                logger.debug("Host " + ho.getName() + " state has already been updated to " + newStatus);
+                logger.debug("Host {} state has already been updated to {}", ho.getName(), newStatus);
                 return true;
             }
         }
@@ -1137,25 +1181,24 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         int result = update(ub, sc, null);
         assert result <= 1 : "How can this update " + result + " rows? ";
 
+        // TODO handle this if(debug){}else{log.debug} it makes no sense
         if (logger.isDebugEnabled() && result == 0) {
             HostVO ho = findById(host.getId());
             assert ho != null : "How how how? : " + host.getId();
 
-            StringBuilder str = new StringBuilder("Unable to update resource state: [");
-            str.append("m = " + host.getId());
-            str.append("; name = " + host.getName());
-            str.append("; old state = " + oldState);
-            str.append("; event = " + event);
-            str.append("; new state = " + newState + "]");
-            logger.debug(str.toString());
+            String str = "Unable to update resource state: [" + "m = " + host.getId() +
+                    "; name = " + host.getName() +
+                    "; old state = " + oldState +
+                    "; event = " + event +
+                    "; new state = " + newState + "]";
+            logger.debug(str);
         } else {
-            StringBuilder msg = new StringBuilder("Resource state update: [");
-            msg.append("id = " + host.getId());
-            msg.append("; name = " + host.getName());
-            msg.append("; old state = " + oldState);
-            msg.append("; event = " + event);
-            msg.append("; new state = " + newState + "]");
-            logger.debug(msg.toString());
+            String msg = "Resource state update: [" + "id = " + host.getId() +
+                    "; name = " + host.getName() +
+                    "; old state = " + oldState +
+                    "; event = " + event +
+                    "; new state = " + newState + "]";
+            logger.debug(msg);
         }
 
         return result > 0;
@@ -1178,6 +1221,11 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return listBy(sc);
     }
 
+    @Override
+    public List<Long> listIdsByDataCenterId(Long zoneId) {
+        return listIdsBy(Type.Routing, null, null, null, zoneId, null, null);
+    }
+
     @Override
     public List<HostVO> findByPodId(Long podId) {
         SearchCriteria<HostVO> sc = PodSearch.create();
@@ -1185,6 +1233,11 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return listBy(sc);
     }
 
+    @Override
+    public List<Long> listIdsByPodId(Long podId) {
+        return listIdsBy(null, null, null, null, null, podId, null);
+    }
+
     @Override
     public List<HostVO> findByClusterId(Long clusterId) {
         SearchCriteria<HostVO> sc = ClusterSearch.create();
@@ -1192,6 +1245,63 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return listBy(sc);
     }
 
+    protected List<Long> listIdsBy(Host.Type type, Status status, ResourceState resourceState,
+             HypervisorType hypervisorType, Long zoneId, Long podId, Long clusterId) {
+        GenericSearchBuilder<HostVO, Long> sb = createSearchBuilder(Long.class);
+        sb.selectFields(sb.entity().getId());
+        sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
+        sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ);
+        sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.EQ);
+        sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ);
+        sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ);
+        sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<Long> sc = sb.create();
+        if (type != null) {
+            sc.setParameters("type", type);
+        }
+        if (status != null) {
+            sc.setParameters("status", status);
+        }
+        if (resourceState != null) {
+            sc.setParameters("resourceState", resourceState);
+        }
+        if (hypervisorType != null) {
+            sc.setParameters("hypervisorType", hypervisorType);
+        }
+        if (zoneId != null) {
+            sc.setParameters("zoneId", zoneId);
+        }
+        if (podId != null) {
+            sc.setParameters("podId", podId);
+        }
+        if (clusterId != null) {
+            sc.setParameters("clusterId", clusterId);
+        }
+        return customSearch(sc, null);
+    }
+
+    @Override
+    public List<Long> listIdsByClusterId(Long clusterId) {
+        return listIdsBy(null, null, null, null, null, null, clusterId);
+    }
+
+    @Override
+    public List<Long> listIdsForUpRouting(Long zoneId, Long podId, Long clusterId) {
+        return listIdsBy(Type.Routing, Status.Up, null, null, zoneId, podId, clusterId);
+    }
+
+    @Override
+    public List<Long> listIdsByType(Type type) {
+        return listIdsBy(type, null, null, null, null, null, null);
+    }
+
+    @Override
+    public List<Long> listIdsForUpEnabledByZoneAndHypervisor(Long zoneId, HypervisorType hypervisorType) {
+        return listIdsBy(null, Status.Up, ResourceState.Enabled, hypervisorType, zoneId, null, null);
+    }
+
     @Override
     public List<HostVO> findByClusterIdAndEncryptionSupport(Long clusterId) {
         SearchBuilder<DetailVO> hostCapabilitySearch = _detailsDao.createSearchBuilder();
@@ -1244,6 +1354,15 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return listBy(sc);
     }
 
+    @Override
+    public HostVO findAnyStateHypervisorHostInCluster(long clusterId) {
+        SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
+        sc.setParameters("type", Host.Type.Routing);
+        sc.setParameters("cluster", clusterId);
+        List<HostVO> list = listBy(sc, new Filter(1));
+        return list.isEmpty() ? null : list.get(0);
+    }
+
     @Override
     public HostVO findOldestExistentHypervisorHostInCluster(long clusterId) {
         SearchCriteria<HostVO> sc = TypeClusterStatusSearch.create();
@@ -1254,7 +1373,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         Filter orderByFilter = new Filter(HostVO.class, "created", true, null, null);
 
         List<HostVO> hosts = search(sc, orderByFilter, null, false);
-        if (hosts != null && hosts.size() > 0) {
+        if (hosts != null && !hosts.isEmpty()) {
             return hosts.get(0);
         }
 
@@ -1263,9 +1382,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     @Override
     public List<Long> listAllHosts(long zoneId) {
-        SearchCriteria<Long> sc = HostIdSearch.create();
-        sc.addAnd("dataCenterId", SearchCriteria.Op.EQ, zoneId);
-        return customSearch(sc, null);
+        return listIdsBy(null, null, null, null, zoneId, null, null);
     }
 
     @Override
@@ -1299,19 +1416,19 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     }
 
     @Override
-    public List<Long> listClustersByHostTag(String computeOfferingTags) {
+    public List<Long> listClustersByHostTag(String hostTags) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        String sql = this.LIST_CLUSTERID_FOR_HOST_TAG;
-        PreparedStatement pstmt = null;
-        List<Long> result = new ArrayList();
-        List<String> tags = Arrays.asList(computeOfferingTags.split(this.SEPARATOR));
-        String subselect = getHostIdsByComputeTags(tags);
-        sql = String.format(sql, subselect);
+        String selectStmtToListClusterIdsByHostTags = LIST_CLUSTER_IDS_FOR_HOST_TAGS;
+        PreparedStatement pstmt;
+        List<Long> result = new ArrayList<>();
+        List<String> tags = Arrays.asList(hostTags.split(SEPARATOR));
+        String selectStmtToListHostIdsByHostTags = getSelectStmtToListHostIdsByHostTags(tags);
+        selectStmtToListClusterIdsByHostTags = String.format(selectStmtToListClusterIdsByHostTags, selectStmtToListHostIdsByHostTags);
 
         try {
-            pstmt = txn.prepareStatement(sql);
+            pstmt = txn.prepareStatement(selectStmtToListClusterIdsByHostTags);
 
-            for(int i = 0; i < tags.size(); i++){
+            for (int i = 0; i < tags.size(); i++){
                 pstmt.setString(i+1, tags.get(i));
             }
 
@@ -1322,20 +1439,20 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
             pstmt.close();
             return result;
         } catch (SQLException e) {
-            throw new CloudRuntimeException("DB Exception on: " + sql, e);
+            throw new CloudRuntimeException("DB Exception on: " + selectStmtToListClusterIdsByHostTags, e);
         }
     }
 
-    private List<Long> findHostByComputeOfferings(String computeOfferingTags){
+    private List<Long> findHostIdsByHostTags(String hostTags){
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        PreparedStatement pstmt = null;
-        List<Long> result = new ArrayList();
-        List<String> tags = Arrays.asList(computeOfferingTags.split(this.SEPARATOR));
-        String select = getHostIdsByComputeTags(tags);
+        PreparedStatement pstmt;
+        List<Long> result = new ArrayList<>();
+        List<String> tags = Arrays.asList(hostTags.split(SEPARATOR));
+        String selectStmtToListHostIdsByHostTags = getSelectStmtToListHostIdsByHostTags(tags);
         try {
-            pstmt = txn.prepareStatement(select);
+            pstmt = txn.prepareStatement(selectStmtToListHostIdsByHostTags);
 
-            for(int i = 0; i < tags.size(); i++){
+            for (int i = 0; i < tags.size(); i++){
                 pstmt.setString(i+1, tags.get(i));
             }
 
@@ -1346,7 +1463,7 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
             pstmt.close();
             return result;
         } catch (SQLException e) {
-            throw new CloudRuntimeException("DB Exception on: " + select, e);
+            throw new CloudRuntimeException("DB Exception on: " + selectStmtToListHostIdsByHostTags, e);
         }
     }
 
@@ -1396,16 +1513,16 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return result;
     }
 
-    private String getHostIdsByComputeTags(List<String> offeringTags){
-        List<String> questionMarks = new ArrayList();
-        offeringTags.forEach((tag) -> { questionMarks.add("?"); });
-        return String.format(this.LIST_HOST_IDS_BY_COMPUTETAGS, String.join(",", questionMarks),questionMarks.size());
+    private String getSelectStmtToListHostIdsByHostTags(List<String> hostTags){
+        List<String> questionMarks = new ArrayList<>();
+        hostTags.forEach((tag) -> questionMarks.add("?"));
+        return String.format(LIST_HOST_IDS_BY_HOST_TAGS, String.join(SEPARATOR, questionMarks), questionMarks.size());
     }
 
     @Override
     public List<HostVO> listHostsWithActiveVMs(long offeringId) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        PreparedStatement pstmt = null;
+        PreparedStatement pstmt;
         List<HostVO> result = new ArrayList<>();
         StringBuilder sql = new StringBuilder(GET_HOSTS_OF_ACTIVE_VMS);
         try {
@@ -1424,15 +1541,37 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     }
 
     @Override
-    public int countByMs(long msid) {
-        SearchCriteria<HostVO> sc = ResponsibleMsCountSearch.create();
-        sc.setParameters("managementServerId", msid);
+    public List<HostVO> listHostsByMsAndDc(long msId, long dcId) {
+        SearchCriteria<HostVO> sc = ResponsibleMsDcSearch.create();
+        sc.setParameters("managementServerId", msId);
+        sc.setParameters("dcId", dcId);
+        return listBy(sc);
+    }
+
+    @Override
+    public List<HostVO> listHostsByMs(long msId) {
+        SearchCriteria<HostVO> sc = ResponsibleMsSearch.create();
+        sc.setParameters("managementServerId", msId);
+        return listBy(sc);
+    }
+
+    @Override
+    public int countByMs(long msId) {
+        SearchCriteria<HostVO> sc = ResponsibleMsSearch.create();
+        sc.setParameters("managementServerId", msId);
         return getCount(sc);
     }
 
+    @Override
+    public List<String> listByMs(long msId) {
+        SearchCriteria<String> sc = ResponsibleMsIdSearch.create();
+        sc.addAnd("managementServerId", SearchCriteria.Op.EQ, msId);
+        return customSearch(sc, null);
+    }
+
     @Override
     public List<String> listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType) {
-        PreparedStatement pstmt = null;
+        PreparedStatement pstmt;
         List<String> result = new ArrayList<>();
         try {
             TransactionLegacy txn = TransactionLegacy.currentTxn();
@@ -1449,15 +1588,6 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return result;
     }
 
-    @Override
-    public List<HostVO> listAllHostsByType(Host.Type type) {
-        SearchCriteria<HostVO> sc = TypeSearch.create();
-        sc.setParameters("type", type);
-        sc.setParameters("resourceState", ResourceState.Enabled);
-
-        return listBy(sc);
-    }
-
     @Override
     public List<HostVO> listByType(Host.Type type) {
         SearchCriteria<HostVO> sc = TypeSearch.create();
@@ -1602,4 +1732,71 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         }
         return String.format(sqlFindHostInZoneToExecuteCommand, hostResourceStatus);
     }
+
+    @Override
+    public boolean isHostUp(long hostId) {
+        GenericSearchBuilder<HostVO, Status> sb = createSearchBuilder(Status.class);
+        sb.and("id", sb.entity().getId(), Op.EQ);
+        sb.selectFields(sb.entity().getStatus());
+        SearchCriteria<Status> sc = sb.create();
+        sc.setParameters("id", hostId);
+        List<Status> statuses = customSearch(sc, null);
+        return CollectionUtils.isNotEmpty(statuses) && Status.Up.equals(statuses.get(0));
+    }
+
+    @Override
+    public List<Long> findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(final Long zoneId, final Long clusterId,
+                final List<ResourceState> resourceStates, final List<Type> types,
+                final List<Hypervisor.HypervisorType> hypervisorTypes) {
+        GenericSearchBuilder<HostVO, Long> sb = createSearchBuilder(Long.class);
+        sb.selectFields(sb.entity().getId());
+        sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.and("resourceState", sb.entity().getResourceState(), SearchCriteria.Op.IN);
+        sb.and("type", sb.entity().getType(), SearchCriteria.Op.IN);
+        if (CollectionUtils.isNotEmpty(hypervisorTypes)) {
+            sb.and().op(sb.entity().getHypervisorType(), SearchCriteria.Op.NULL);
+            sb.or("hypervisorTypes", sb.entity().getHypervisorType(), SearchCriteria.Op.IN);
+            sb.cp();
+        }
+        sb.done();
+        SearchCriteria<Long> sc = sb.create();
+        if (zoneId != null) {
+            sc.setParameters("zoneId", zoneId);
+        }
+        if (clusterId != null) {
+            sc.setParameters("clusterId", clusterId);
+        }
+        if (CollectionUtils.isNotEmpty(hypervisorTypes)) {
+            sc.setParameters("hypervisorTypes", hypervisorTypes.toArray());
+        }
+        sc.setParameters("resourceState", resourceStates.toArray());
+        sc.setParameters("type", types.toArray());
+        return customSearch(sc, null);
+    }
+
+    @Override
+    public List<HypervisorType> listDistinctHypervisorTypes(final Long zoneId) {
+        GenericSearchBuilder<HostVO, HypervisorType> sb = createSearchBuilder(HypervisorType.class);
+        sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
+        sb.and("type", sb.entity().getType(), SearchCriteria.Op.EQ);
+        sb.select(null, Func.DISTINCT, sb.entity().getHypervisorType());
+        sb.done();
+        SearchCriteria<HypervisorType> sc = sb.create();
+        if (zoneId != null) {
+            sc.setParameters("zoneId", zoneId);
+        }
+        sc.setParameters("type", Type.Routing);
+        return customSearch(sc, null);
+    }
+
+    @Override
+    public List<HostVO> listByIds(List<Long> ids) {
+        if (CollectionUtils.isEmpty(ids)) {
+            return new ArrayList<>();
+        }
+        SearchCriteria<HostVO> sc = IdsSearch.create();
+        sc.setParameters("id", ids.toArray());
+        return search(sc, null);
+    }
 }

engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java

@@ -421,7 +421,7 @@ public class IPAddressDaoImpl extends GenericDaoBase<IPAddressVO, Long> implemen
     public long countFreeIpsInVlan(long vlanDbId) {
         SearchCriteria<IPAddressVO> sc = VlanDbIdSearchUnallocated.create();
         sc.setParameters("vlanDbId", vlanDbId);
-        return listBy(sc).size();
+        return getCount(sc);
     }
 
     @Override

engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java

@@ -415,8 +415,7 @@ public class NetworkDaoImpl extends GenericDaoBase<NetworkVO, Long>implements Ne
         sc.setParameters("broadcastUri", broadcastURI);
         sc.setParameters("guestType", guestTypes);
         sc.setJoinParameters("persistent", "persistent", isPersistent);
-        List<NetworkVO> persistentNetworks = search(sc, null);
-        return persistentNetworks.size();
+        return getCount(sc);
     }
 
     @Override

engine/schema/src/main/java/com/cloud/secstorage/CommandExecLogDaoImpl.java

@@ -55,8 +55,7 @@ public class CommandExecLogDaoImpl extends GenericDaoBase<CommandExecLogVO, Long
         SearchCriteria<CommandExecLogVO> sc = CommandSearch.create();
         sc.setParameters("host_id", id);
         sc.setParameters("command_name", "CopyCommand");
-        List<CommandExecLogVO> copyCmds = customSearch(sc, null);
-        return copyCmds.size();
+        return getCount(sc);
     }
 
     @Override

engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDao.java

@@ -54,7 +54,7 @@ public interface ServiceOfferingDao extends GenericDao<ServiceOfferingVO, Long>
 
     List<ServiceOfferingVO> listPublicByCpuAndMemory(Integer cpus, Integer memory);
 
-    List<ServiceOfferingVO> listByHostTag(String tag);
-
     ServiceOfferingVO findServiceOfferingByComputeOnlyDiskOffering(long diskOfferingId, boolean includingRemoved);
+
+    List<Long> listIdsByHostTag(String tag);
 }

engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDaoImpl.java

@@ -34,6 +34,7 @@ import com.cloud.service.ServiceOfferingVO;
 import com.cloud.storage.Storage.ProvisioningType;
 import com.cloud.utils.db.DB;
 import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -293,8 +294,9 @@ public class ServiceOfferingDaoImpl extends GenericDaoBase<ServiceOfferingVO, Lo
     }
 
     @Override
-    public List<ServiceOfferingVO> listByHostTag(String tag) {
-        SearchBuilder<ServiceOfferingVO> sb = createSearchBuilder();
+    public List<Long> listIdsByHostTag(String tag) {
+        GenericSearchBuilder<ServiceOfferingVO, Long> sb = createSearchBuilder(Long.class);
+        sb.selectFields(sb.entity().getId());
         sb.and("tagNotNull", sb.entity().getHostTag(), SearchCriteria.Op.NNULL);
         sb.and().op("tagEq", sb.entity().getHostTag(), SearchCriteria.Op.EQ);
         sb.or("tagStartLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE);
@@ -302,11 +304,12 @@ public class ServiceOfferingDaoImpl extends GenericDaoBase<ServiceOfferingVO, Lo
         sb.or("tagEndLike", sb.entity().getHostTag(), SearchCriteria.Op.LIKE);
         sb.cp();
         sb.done();
-        SearchCriteria<ServiceOfferingVO> sc = sb.create();
+        SearchCriteria<Long> sc = sb.create();
+
         sc.setParameters("tagEq", tag);
         sc.setParameters("tagStartLike", tag + ",%");
         sc.setParameters("tagMidLike", "%," + tag + ",%");
         sc.setParameters("tagEndLike",   "%," + tag);
-        return listBy(sc);
+        return customSearch(sc, null);
     }
 }

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDao.java

@@ -34,7 +34,7 @@ public interface StoragePoolHostDao extends GenericDao<StoragePoolHostVO, Long>
 
     List<Long> findHostsConnectedToPools(List<Long> poolIds);
 
-    List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly);
+    boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly);
 
     public void deletePrimaryRecordsForHost(long hostId);
 

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java

@@ -55,11 +55,11 @@ public class StoragePoolHostDaoImpl extends GenericDaoBase<StoragePoolHostVO, Lo
 
     protected static final String HOSTS_FOR_POOLS_SEARCH = "SELECT DISTINCT(ph.host_id) FROM storage_pool_host_ref ph, host h WHERE ph.host_id = h.id AND h.status = 'Up' AND resource_state = 'Enabled' AND ph.pool_id IN (?)";
 
-    protected static final String STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id,  count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
-        + " WHERE p.id = ph.pool_id AND p.data_center_id = ? " + " GROUP by p.data_center_id";
+    protected static final String STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph WHERE " +
+            "ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ?";
 
-    protected static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id,  count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
-        + " WHERE p.id = ph.pool_id AND p.data_center_id = ? " + " AND p.pool_type NOT IN ('LVM', 'Filesystem')" + " GROUP by p.data_center_id";
+    protected static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph " +
+            "WHERE ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ? AND p.pool_type NOT IN ('LVM', 'Filesystem')";
 
     protected static final String DELETE_PRIMARY_RECORDS = "DELETE " + "FROM storage_pool_host_ref " + "WHERE host_id = ?";
 
@@ -169,23 +169,23 @@ public class StoragePoolHostDaoImpl extends GenericDaoBase<StoragePoolHostVO, Lo
     }
 
     @Override
-    public List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly) {
-        ArrayList<Pair<Long, Integer>> l = new ArrayList<Pair<Long, Integer>>();
+    public boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly) {
+        Long poolCount = 0L;
         String sql = sharedOnly ? SHARED_STORAGE_POOL_HOST_INFO : STORAGE_POOL_HOST_INFO;
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        PreparedStatement pstmt = null;
-        try {
-            pstmt = txn.prepareAutoCloseStatement(sql);
+        try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) {
             pstmt.setLong(1, dcId);
-
             ResultSet rs = pstmt.executeQuery();
             while (rs.next()) {
-                l.add(new Pair<Long, Integer>(rs.getLong(1), rs.getInt(2)));
+                poolCount = rs.getLong(1);
+                if (poolCount > 0) {
+                    return true;
+                }
             }
         } catch (SQLException e) {
             logger.debug("SQLException: ", e);
         }
-        return l;
+        return false;
     }
 
     /**

engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java

@@ -67,6 +67,8 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long>, StateDao<
 
     public List<VMTemplateVO> userIsoSearch(boolean listRemoved);
 
+    List<VMTemplateVO> listAllReadySystemVMTemplates(Long zoneId);
+
     VMTemplateVO findSystemVMTemplate(long zoneId);
 
     VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType);
@@ -91,6 +93,5 @@ public interface VMTemplateDao extends GenericDao<VMTemplateVO, Long>, StateDao<
 
     List<VMTemplateVO> listByIds(List<Long> ids);
 
-    List<VMTemplateVO> listByTemplateTag(String tag);
-
+    List<Long> listIdsByTemplateTag(String tag);
 }

engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java

@@ -344,19 +344,12 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
         readySystemTemplateSearch = createSearchBuilder();
         readySystemTemplateSearch.and("state", readySystemTemplateSearch.entity().getState(), SearchCriteria.Op.EQ);
         readySystemTemplateSearch.and("templateType", readySystemTemplateSearch.entity().getTemplateType(), SearchCriteria.Op.EQ);
+        readySystemTemplateSearch.and("hypervisorType", readySystemTemplateSearch.entity().getHypervisorType(), SearchCriteria.Op.IN);
         SearchBuilder<TemplateDataStoreVO> templateDownloadSearch = _templateDataStoreDao.createSearchBuilder();
         templateDownloadSearch.and("downloadState", templateDownloadSearch.entity().getDownloadState(), SearchCriteria.Op.IN);
         readySystemTemplateSearch.join("vmTemplateJoinTemplateStoreRef", templateDownloadSearch, templateDownloadSearch.entity().getTemplateId(),
             readySystemTemplateSearch.entity().getId(), JoinBuilder.JoinType.INNER);
-        SearchBuilder<HostVO> hostHyperSearch2 = _hostDao.createSearchBuilder();
-        hostHyperSearch2.and("type", hostHyperSearch2.entity().getType(), SearchCriteria.Op.EQ);
-        hostHyperSearch2.and("zoneId", hostHyperSearch2.entity().getDataCenterId(), SearchCriteria.Op.EQ);
-        hostHyperSearch2.and("removed", hostHyperSearch2.entity().getRemoved(), SearchCriteria.Op.NULL);
-        hostHyperSearch2.groupBy(hostHyperSearch2.entity().getHypervisorType());
-
-        readySystemTemplateSearch.join("tmplHyper", hostHyperSearch2, hostHyperSearch2.entity().getHypervisorType(), readySystemTemplateSearch.entity()
-            .getHypervisorType(), JoinBuilder.JoinType.INNER);
-        hostHyperSearch2.done();
+        readySystemTemplateSearch.groupBy(readySystemTemplateSearch.entity().getId());
         readySystemTemplateSearch.done();
 
         tmpltTypeHyperSearch2 = createSearchBuilder();
@@ -556,29 +549,35 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
     }
 
     @Override
-    public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) {
+    public List<VMTemplateVO> listAllReadySystemVMTemplates(Long zoneId) {
+        List<HypervisorType> availableHypervisors = _hostDao.listDistinctHypervisorTypes(zoneId);
+        if (CollectionUtils.isEmpty(availableHypervisors)) {
+            return Collections.emptyList();
+        }
         SearchCriteria<VMTemplateVO> sc = readySystemTemplateSearch.create();
         sc.setParameters("templateType", Storage.TemplateType.SYSTEM);
         sc.setParameters("state", VirtualMachineTemplate.State.Active);
-        sc.setJoinParameters("tmplHyper", "type", Host.Type.Routing);
-        sc.setJoinParameters("tmplHyper", "zoneId", zoneId);
-        sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState", new VMTemplateStorageResourceAssoc.Status[] {VMTemplateStorageResourceAssoc.Status.DOWNLOADED, VMTemplateStorageResourceAssoc.Status.BYPASSED});
-
+        sc.setParameters("hypervisorType", availableHypervisors.toArray());
+        sc.setJoinParameters("vmTemplateJoinTemplateStoreRef", "downloadState",
+                List.of(VMTemplateStorageResourceAssoc.Status.DOWNLOADED,
+                        VMTemplateStorageResourceAssoc.Status.BYPASSED).toArray());
         // order by descending order of id
-        List<VMTemplateVO> tmplts = listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null));
-
-        if (tmplts.size() > 0) {
-            if (hypervisorType == HypervisorType.Any) {
-                return tmplts.get(0);
-            }
-            for (VMTemplateVO tmplt : tmplts) {
-                if (tmplt.getHypervisorType() == hypervisorType) {
-                    return tmplt;
-                }
-            }
+        return listBy(sc, new Filter(VMTemplateVO.class, "id", false, null, null));
+    }
 
+    @Override
+    public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) {
+        List<VMTemplateVO> templates = listAllReadySystemVMTemplates(zoneId);
+        if (CollectionUtils.isEmpty(templates)) {
+            return null;
         }
-        return null;
+        if (hypervisorType == HypervisorType.Any) {
+            return templates.get(0);
+        }
+        return templates.stream()
+                .filter(t -> t.getHypervisorType() == hypervisorType)
+                .findFirst()
+                .orElse(null);
     }
 
     @Override
@@ -687,13 +686,14 @@ public class VMTemplateDaoImpl extends GenericDaoBase<VMTemplateVO, Long> implem
     }
 
     @Override
-    public List<VMTemplateVO> listByTemplateTag(String tag) {
-        SearchBuilder<VMTemplateVO> sb = createSearchBuilder();
+    public List<Long> listIdsByTemplateTag(String tag) {
+        GenericSearchBuilder<VMTemplateVO, Long> sb = createSearchBuilder(Long.class);
+        sb.selectFields(sb.entity().getId());
         sb.and("tag", sb.entity().getTemplateTag(), SearchCriteria.Op.EQ);
         sb.done();
-        SearchCriteria<VMTemplateVO> sc = sb.create();
+        SearchCriteria<Long> sc = sb.create();
         sc.setParameters("tag", tag);
-        return listIncludingRemovedBy(sc);
+        return customSearchIncludingRemoved(sc, null);
     }
 
     @Override

engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java

@@ -571,14 +571,6 @@ public class VolumeDaoImpl extends GenericDaoBase<VolumeVO, Long> implements Vol
         }
     }
 
-    public static class SumCount {
-        public long sum;
-        public long count;
-
-        public SumCount() {
-        }
-    }
-
     @Override
     public List<VolumeVO> listVolumesToBeDestroyed() {
         SearchCriteria<VolumeVO> sc = AllFieldsSearch.create();

engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java

@@ -870,7 +870,7 @@ public class SystemVmTemplateRegistration {
             public void doInTransactionWithoutResult(final TransactionStatus status) {
                 Set<Hypervisor.HypervisorType> hypervisorsListInUse = new HashSet<Hypervisor.HypervisorType>();
                 try {
-                    hypervisorsListInUse = clusterDao.getDistictAvailableHypervisorsAcrossClusters();
+                    hypervisorsListInUse = clusterDao.getDistinctAvailableHypervisorsAcrossClusters();
 
                 } catch (final Exception e) {
                     LOGGER.error("updateSystemVmTemplates: Exception caught while getting hypervisor types from clusters: " + e.getMessage());

engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java

@@ -114,6 +114,17 @@ public class DatabaseAccessObject {
         }
     }
 
+    public void renameIndex(Connection conn, String tableName, String oldName, String newName) {
+        String stmt = String.format("ALTER TABLE %s RENAME INDEX %s TO %s", tableName, oldName, newName);
+        logger.debug("Statement: {}", stmt);
+        try (PreparedStatement pstmt = conn.prepareStatement(stmt)) {
+            pstmt.execute();
+            logger.debug("Renamed index {} to {}", oldName, newName);
+        } catch (SQLException e) {
+            logger.warn("Unable to rename index {} to {}", oldName, newName, e);
+        }
+    }
+
     protected void closePreparedStatement(PreparedStatement pstmt, String errorMessage) {
         try {
             if (pstmt != null) {

engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java

@@ -31,6 +31,12 @@ public class DbUpgradeUtils {
         }
     }
 
+    public static void renameIndexIfNeeded(Connection conn, String tableName, String oldName, String newName) {
+        if (!dao.indexExists(conn, tableName, oldName)) {
+            dao.renameIndex(conn, tableName, oldName, newName);
+        }
+    }
+
     public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) {
         dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName);
     }

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java

@@ -53,6 +53,7 @@ public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgr
 
     @Override
     public void performDataMigration(Connection conn) {
+        addIndexes(conn);
     }
 
     @Override
@@ -80,4 +81,42 @@ public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgr
             throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
         }
     }
+
+    private void addIndexes(Connection conn) {
+        DbUpgradeUtils.addIndexIfNeeded(conn, "host", "mgmt_server_id");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "host", "resource");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "host", "resource_state");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "host", "type");
+
+        DbUpgradeUtils.renameIndexIfNeeded(conn, "user_ip_address", "public_ip_address", "uk_public_ip_address");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "public_ip_address");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "data_center_id");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "vlan_db_id");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "user_ip_address", "removed");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "vlan", "vlan_type");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "vlan", "data_center_id");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "vlan", "removed");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "network_offering_details", "name");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "network_offering_details", "resource_id", "resource_type");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "cpu");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "speed");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "ram_size");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "op_host_planner_reservation", "resource_usage");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "storage_pool", "pool_type");
+        DbUpgradeUtils.addIndexIfNeeded(conn, "storage_pool", "data_center_id", "status", "scope", "hypervisor");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "router_network_ref", "guest_type");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "domain_router", "role");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "async_job", "instance_type", "job_status");
+
+        DbUpgradeUtils.addIndexIfNeeded(conn, "cluster", "managed_state");
+    }
 }

engine/schema/src/main/java/com/cloud/user/UserAccountVO.java

@@ -33,11 +33,11 @@ import javax.persistence.Table;
 import javax.persistence.Transient;
 
 import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.utils.db.Encrypt;
 import com.cloud.utils.db.GenericDao;
-import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
-import org.apache.commons.lang3.StringUtils;
 
 @Entity
 @Table(name = "user")
@@ -131,12 +131,6 @@ public class UserAccountVO implements UserAccount, InternalIdentity {
     public UserAccountVO() {
     }
 
-    @Override
-    public String toString() {
-        return String.format("UserAccount %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields
-                (this, "id", "uuid", "username", "accountName"));
-    }
-
     @Override
     public long getId() {
         return id;
@@ -379,4 +373,10 @@ public class UserAccountVO implements UserAccount, InternalIdentity {
     public void setDetails(Map<String, String> details) {
         this.details = details;
     }
+
+    @Override
+    public String toString() {
+        return String.format("UserAccount %s.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields
+                (this, "id", "uuid", "username", "accountName"));
+    }
 }

engine/schema/src/main/java/com/cloud/vm/dao/ConsoleProxyDao.java

@@ -45,7 +45,7 @@ public interface ConsoleProxyDao extends GenericDao<ConsoleProxyVO, Long> {
 
     public List<ConsoleProxyLoadInfo> getDatacenterSessionLoadMatrix();
 
-    public List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean countAllPoolTypes);
+    public boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly);
 
     public List<Pair<Long, Integer>> getProxyLoadMatrix();
 

engine/schema/src/main/java/com/cloud/vm/dao/ConsoleProxyDaoImpl.java

@@ -23,7 +23,6 @@ import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
-
 import org.springframework.stereotype.Component;
 
 import com.cloud.info.ConsoleProxyLoadInfo;
@@ -76,11 +75,11 @@ public class ConsoleProxyDaoImpl extends GenericDaoBase<ConsoleProxyVO, Long> im
 
     private static final String GET_PROXY_ACTIVE_LOAD = "SELECT active_session AS count" + " FROM console_proxy" + " WHERE id=?";
 
-    private static final String STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id,  count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
-        + " WHERE p.id = ph.pool_id AND p.data_center_id = ? " + " GROUP by p.data_center_id";
+    protected static final String STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph WHERE " +
+            "ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ?";
 
-    private static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT p.data_center_id,  count(ph.host_id) " + " FROM storage_pool p, storage_pool_host_ref ph "
-        + " WHERE p.pool_type <> 'LVM' AND p.id = ph.pool_id AND p.data_center_id = ? " + " GROUP by p.data_center_id";
+    protected static final String SHARED_STORAGE_POOL_HOST_INFO = "SELECT (SELECT id FROM storage_pool_host_ref ph " +
+            "WHERE ph.pool_id=p.id limit 1) AS sphr FROM storage_pool p WHERE p.data_center_id = ? AND p.pool_type NOT IN ('LVM', 'Filesystem')";
 
     protected SearchBuilder<ConsoleProxyVO> DataCenterStatusSearch;
     protected SearchBuilder<ConsoleProxyVO> StateSearch;
@@ -219,28 +218,23 @@ public class ConsoleProxyDaoImpl extends GenericDaoBase<ConsoleProxyVO, Long> im
     }
 
     @Override
-    public List<Pair<Long, Integer>> getDatacenterStoragePoolHostInfo(long dcId, boolean countAllPoolTypes) {
-        ArrayList<Pair<Long, Integer>> l = new ArrayList<Pair<Long, Integer>>();
-
+    public boolean hasDatacenterStoragePoolHostInfo(long dcId, boolean sharedOnly) {
+        Long poolCount = 0L;
+        String sql = sharedOnly ? SHARED_STORAGE_POOL_HOST_INFO : STORAGE_POOL_HOST_INFO;
         TransactionLegacy txn = TransactionLegacy.currentTxn();
-        ;
-        PreparedStatement pstmt = null;
-        try {
-            if (countAllPoolTypes) {
-                pstmt = txn.prepareAutoCloseStatement(STORAGE_POOL_HOST_INFO);
-            } else {
-                pstmt = txn.prepareAutoCloseStatement(SHARED_STORAGE_POOL_HOST_INFO);
-            }
+        try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql)) {
             pstmt.setLong(1, dcId);
-
             ResultSet rs = pstmt.executeQuery();
             while (rs.next()) {
-                l.add(new Pair<Long, Integer>(rs.getLong(1), rs.getInt(2)));
+                poolCount = rs.getLong(1);
+                if (poolCount > 0) {
+                    return true;
+                }
             }
         } catch (SQLException e) {
             logger.debug("Caught SQLException: ", e);
         }
-        return l;
+        return false;
     }
 
     @Override

engine/schema/src/main/java/com/cloud/vm/dao/NicIpAliasDaoImpl.java

@@ -170,8 +170,7 @@ public class NicIpAliasDaoImpl extends GenericDaoBase<NicIpAliasVO, Long> implem
     public Integer countAliasIps(long id) {
         SearchCriteria<NicIpAliasVO> sc = AllFieldsSearch.create();
         sc.setParameters("instanceId", id);
-        List<NicIpAliasVO> list = listBy(sc);
-        return list.size();
+        return getCount(sc);
     }
 
     @Override

engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.vm.dao;
 
+import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -81,7 +82,7 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
 
     List<VMInstanceVO> listByHostAndState(long hostId, State... states);
 
-    List<VMInstanceVO> listByTypes(VirtualMachine.Type... types);
+    int countByTypes(VirtualMachine.Type... types);
 
     VMInstanceVO findByIdTypes(long id, VirtualMachine.Type... types);
 
@@ -144,21 +145,28 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
      */
     List<String> listDistinctHostNames(long networkId, VirtualMachine.Type... types);
 
+    List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, Collection<Long> excludingIds, State... states);
+
     List<VMInstanceVO> findByHostInStates(Long hostId, State... states);
 
     List<VMInstanceVO> listStartingWithNoHostId();
 
     boolean updatePowerState(long instanceId, long powerHostId, VirtualMachine.PowerState powerState, Date wisdomEra);
 
+    Map<Long, VirtualMachine.PowerState> updatePowerState(Map<Long, VirtualMachine.PowerState> instancePowerStates,
+              long powerHostId, Date wisdomEra);
+
     void resetVmPowerStateTracking(long instanceId);
 
+    void resetVmPowerStateTracking(List<Long> instanceId);
+
     void resetHostPowerStateTracking(long hostId);
 
     HashMap<String, Long> countVgpuVMs(Long dcId, Long podId, Long clusterId);
 
     VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId);
 
-    boolean isPowerStateUpToDate(long instanceId);
+    boolean isPowerStateUpToDate(VMInstanceVO instance);
 
     List<VMInstanceVO> listNonMigratingVmsByHostEqualsLastHost(long hostId);
 
@@ -170,4 +178,13 @@ public interface VMInstanceDao extends GenericDao<VMInstanceVO, Long>, StateDao<
              List<Long> skippedVmIds);
 
     Pair<List<VMInstanceVO>, Integer> listByVmsNotInClusterUsingPool(long clusterId, long poolId);
+
+    List<VMInstanceVO> listIdServiceOfferingForUpVmsByHostId(Long hostId);
+
+    List<VMInstanceVO> listIdServiceOfferingForVmsMigratingFromHost(Long hostId);
+
+    Map<String, Long> getNameIdMapForVmInstanceNames(Collection<String> names);
+
+    Map<String, Long> getNameIdMapForVmIds(Collection<Long> ids);
+
 }

engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java

@@ -20,6 +20,7 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
@@ -75,6 +76,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
     protected SearchBuilder<VMInstanceVO> LHVMClusterSearch;
     protected SearchBuilder<VMInstanceVO> IdStatesSearch;
     protected SearchBuilder<VMInstanceVO> AllFieldsSearch;
+    protected SearchBuilder<VMInstanceVO> IdServiceOfferingIdSelectSearch;
     protected SearchBuilder<VMInstanceVO> ZoneTemplateNonExpungedSearch;
     protected SearchBuilder<VMInstanceVO> TemplateNonExpungedSearch;
     protected SearchBuilder<VMInstanceVO> NameLikeSearch;
@@ -101,6 +103,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
     protected SearchBuilder<VMInstanceVO> BackupSearch;
     protected SearchBuilder<VMInstanceVO> LastHostAndStatesSearch;
     protected SearchBuilder<VMInstanceVO> VmsNotInClusterUsingPool;
+    protected SearchBuilder<VMInstanceVO> IdsPowerStateSelectSearch;
 
     @Inject
     ResourceTagDao tagsDao;
@@ -175,6 +178,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), Op.EQ);
         AllFieldsSearch.done();
 
+        IdServiceOfferingIdSelectSearch = createSearchBuilder();
+        IdServiceOfferingIdSelectSearch.and("host", IdServiceOfferingIdSelectSearch.entity().getHostId(), Op.EQ);
+        IdServiceOfferingIdSelectSearch.and("lastHost", IdServiceOfferingIdSelectSearch.entity().getLastHostId(), Op.EQ);
+        IdServiceOfferingIdSelectSearch.and("state", IdServiceOfferingIdSelectSearch.entity().getState(), Op.EQ);
+        IdServiceOfferingIdSelectSearch.and("states", IdServiceOfferingIdSelectSearch.entity().getState(), Op.IN);
+        IdServiceOfferingIdSelectSearch.selectFields(IdServiceOfferingIdSelectSearch.entity().getId(), IdServiceOfferingIdSelectSearch.entity().getServiceOfferingId());
+        IdServiceOfferingIdSelectSearch.done();
+
         ZoneTemplateNonExpungedSearch = createSearchBuilder();
         ZoneTemplateNonExpungedSearch.and("zone", ZoneTemplateNonExpungedSearch.entity().getDataCenterId(), Op.EQ);
         ZoneTemplateNonExpungedSearch.and("template", ZoneTemplateNonExpungedSearch.entity().getTemplateId(), Op.EQ);
@@ -274,6 +285,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         HostAndStateSearch = createSearchBuilder();
         HostAndStateSearch.and("host", HostAndStateSearch.entity().getHostId(), Op.EQ);
         HostAndStateSearch.and("states", HostAndStateSearch.entity().getState(), Op.IN);
+        HostAndStateSearch.and("idsNotIn", HostAndStateSearch.entity().getId(), Op.NIN);
         HostAndStateSearch.done();
 
         StartingWithNoHostSearch = createSearchBuilder();
@@ -323,6 +335,15 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         VmsNotInClusterUsingPool.join("hostSearch2", hostSearch2, hostSearch2.entity().getId(), VmsNotInClusterUsingPool.entity().getHostId(), JoinType.INNER);
         VmsNotInClusterUsingPool.and("vmStates", VmsNotInClusterUsingPool.entity().getState(), Op.IN);
         VmsNotInClusterUsingPool.done();
+
+        IdsPowerStateSelectSearch = createSearchBuilder();
+        IdsPowerStateSelectSearch.and("id", IdsPowerStateSelectSearch.entity().getId(), Op.IN);
+        IdsPowerStateSelectSearch.selectFields(IdsPowerStateSelectSearch.entity().getId(),
+                IdsPowerStateSelectSearch.entity().getPowerHostId(),
+                IdsPowerStateSelectSearch.entity().getPowerState(),
+                IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(),
+                IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime());
+        IdsPowerStateSelectSearch.done();
     }
 
     @Override
@@ -458,10 +479,10 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
     }
 
     @Override
-    public List<VMInstanceVO> listByTypes(Type... types) {
+    public int countByTypes(Type... types) {
         SearchCriteria<VMInstanceVO> sc = TypesSearch.create();
         sc.setParameters("types", (Object[])types);
-        return listBy(sc);
+        return getCount(sc);
     }
 
     @Override
@@ -897,6 +918,17 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         return result;
     }
 
+    @Override
+    public List<VMInstanceVO> findByHostInStatesExcluding(Long hostId, Collection<Long> excludingIds, State... states) {
+        SearchCriteria<VMInstanceVO> sc = HostAndStateSearch.create();
+        sc.setParameters("host", hostId);
+        if (excludingIds != null && !excludingIds.isEmpty()) {
+            sc.setParameters("idsNotIn", excludingIds.toArray());
+        }
+        sc.setParameters("states", (Object[])states);
+        return listBy(sc);
+    }
+
     @Override
     public List<VMInstanceVO> findByHostInStates(Long hostId, State... states) {
         SearchCriteria<VMInstanceVO> sc = HostAndStateSearch.create();
@@ -912,42 +944,109 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         return listBy(sc);
     }
 
-    @Override
-    public boolean updatePowerState(final long instanceId, final long powerHostId, final VirtualMachine.PowerState powerState, Date wisdomEra) {
-        return Transaction.execute(new TransactionCallback<>() {
-            @Override
-            public Boolean doInTransaction(TransactionStatus status) {
-                boolean needToUpdate = false;
-                VMInstanceVO instance = findById(instanceId);
-                if (instance != null
-                        && (null == instance.getPowerStateUpdateTime()
-                        || instance.getPowerStateUpdateTime().before(wisdomEra))) {
-                    Long savedPowerHostId = instance.getPowerHostId();
-                    if (instance.getPowerState() != powerState
-                            || savedPowerHostId == null
-                            || savedPowerHostId != powerHostId
-                            || !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance)) {
-                        instance.setPowerState(powerState);
-                        instance.setPowerHostId(powerHostId);
-                        instance.setPowerStateUpdateCount(1);
-                        instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
-                        needToUpdate = true;
-                        update(instanceId, instance);
-                    } else {
-                        // to reduce DB updates, consecutive same state update for more than 3 times
-                        if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
-                            instance.setPowerStateUpdateCount(instance.getPowerStateUpdateCount() + 1);
-                            instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
-                            needToUpdate = true;
-                            update(instanceId, instance);
-                        }
-                    }
-                }
-                return needToUpdate;
+    protected List<VMInstanceVO> listSelectPowerStateByIds(final List<Long> ids) {
+        if (CollectionUtils.isEmpty(ids)) {
+            return new ArrayList<>();
+        }
+        SearchCriteria<VMInstanceVO> sc = IdsPowerStateSelectSearch.create();
+        sc.setParameters("id", ids.toArray());
+        return customSearch(sc, null);
+    }
+
+    protected Integer getPowerUpdateCount(final VMInstanceVO instance, final long powerHostId,
+              final VirtualMachine.PowerState powerState, Date wisdomEra) {
+        if (instance.getPowerStateUpdateTime() == null || instance.getPowerStateUpdateTime().before(wisdomEra)) {
+            Long savedPowerHostId = instance.getPowerHostId();
+            boolean isStateMismatch = instance.getPowerState() != powerState
+                    || savedPowerHostId == null
+                    || !savedPowerHostId.equals(powerHostId)
+                    || !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance);
+            if (isStateMismatch) {
+                return 1;
+            } else if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
+                return instance.getPowerStateUpdateCount() + 1;
             }
+        }
+        return null;
+    }
+
+    @Override
+    public boolean updatePowerState(final long instanceId, final long powerHostId,
+            final VirtualMachine.PowerState powerState, Date wisdomEra) {
+        return Transaction.execute((TransactionCallback<Boolean>) status -> {
+            VMInstanceVO instance = findById(instanceId);
+            if (instance == null) {
+                return false;
+            }
+            // Check if we need to update based on powerStateUpdateTime
+            if (instance.getPowerStateUpdateTime() == null || instance.getPowerStateUpdateTime().before(wisdomEra)) {
+                Long savedPowerHostId = instance.getPowerHostId();
+                boolean isStateMismatch = instance.getPowerState() != powerState
+                        || savedPowerHostId == null
+                        || !savedPowerHostId.equals(powerHostId)
+                        || !isPowerStateInSyncWithInstanceState(powerState, powerHostId, instance);
+
+                if (isStateMismatch) {
+                    instance.setPowerState(powerState);
+                    instance.setPowerHostId(powerHostId);
+                    instance.setPowerStateUpdateCount(1);
+                } else if (instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT) {
+                    instance.setPowerStateUpdateCount(instance.getPowerStateUpdateCount() + 1);
+                } else {
+                    // No need to update if power state is already in sync and count exceeded
+                    return false;
+                }
+                instance.setPowerStateUpdateTime(DateUtil.currentGMTTime());
+                update(instanceId, instance);
+                return true; // Return true since an update occurred
+            }
+            return false;
         });
     }
 
+    @Override
+    public Map<Long, VirtualMachine.PowerState> updatePowerState(
+            final Map<Long, VirtualMachine.PowerState> instancePowerStates, long powerHostId, Date wisdomEra) {
+        Map<Long, VirtualMachine.PowerState> notUpdated = new HashMap<>();
+        List<VMInstanceVO> instances = listSelectPowerStateByIds(new ArrayList<>(instancePowerStates.keySet()));
+        Map<Long, Integer> updateCounts = new HashMap<>();
+        for (VMInstanceVO instance : instances) {
+            VirtualMachine.PowerState powerState = instancePowerStates.get(instance.getId());
+            Integer count = getPowerUpdateCount(instance, powerHostId, powerState, wisdomEra);
+            if (count != null) {
+                updateCounts.put(instance.getId(), count);
+            } else {
+                notUpdated.put(instance.getId(), powerState);
+            }
+        }
+        if (updateCounts.isEmpty()) {
+            return notUpdated;
+        }
+        StringBuilder sql = new StringBuilder("UPDATE `cloud`.`vm_instance` SET " +
+                "`power_host` = ?, `power_state_update_time` = now(), `power_state` = CASE ");
+        updateCounts.keySet().forEach(key -> {
+            sql.append("WHEN id = ").append(key).append(" THEN '").append(instancePowerStates.get(key)).append("' ");
+        });
+        sql.append("END, `power_state_update_count` = CASE ");
+        StringBuilder idList = new StringBuilder();
+        updateCounts.forEach((key, value) -> {
+            sql.append("WHEN `id` = ").append(key).append(" THEN ").append(value).append(" ");
+            idList.append(key).append(",");
+        });
+        idList.setLength(idList.length() - 1);
+        sql.append("END WHERE `id` IN (").append(idList).append(")");
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) {
+            pstmt.setLong(1, powerHostId);
+            pstmt.executeUpdate();
+        } catch (SQLException e) {
+            logger.error("Unable to execute update power states SQL from VMs {} due to: {}",
+                    idList, e.getMessage(), e);
+            return instancePowerStates;
+        }
+        return notUpdated;
+    }
+
     private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) {
         State instanceState = instance.getState();
         if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running)
@@ -962,11 +1061,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
     }
 
     @Override
-    public boolean isPowerStateUpToDate(final long instanceId) {
-        VMInstanceVO instance = findById(instanceId);
-        if(instance == null) {
-            throw new CloudRuntimeException("checking power state update count on non existing instance " + instanceId);
-        }
+    public boolean isPowerStateUpToDate(final VMInstanceVO instance) {
         return instance.getPowerStateUpdateCount() < MAX_CONSECUTIVE_SAME_STATE_UPDATE_COUNT;
     }
 
@@ -985,6 +1080,25 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         });
     }
 
+    @Override
+    public void resetVmPowerStateTracking(List<Long> instanceIds) {
+        if (CollectionUtils.isEmpty(instanceIds)) {
+            return;
+        }
+        Transaction.execute(new TransactionCallbackNoReturn() {
+            @Override
+            public void doInTransactionWithoutResult(TransactionStatus status) {
+                SearchCriteria<VMInstanceVO> sc = IdsPowerStateSelectSearch.create();
+                sc.setParameters("id", instanceIds.toArray());
+                VMInstanceVO vm = createForUpdate();
+                vm.setPowerStateUpdateCount(0);
+                vm.setPowerStateUpdateTime(DateUtil.currentGMTTime());
+                UpdateBuilder ub = getUpdateBuilder(vm);
+                update(ub, sc, null);
+            }
+        });
+    }
+
     @Override @DB
     public void resetHostPowerStateTracking(final long hostId) {
         Transaction.execute(new TransactionCallbackNoReturn() {
@@ -1060,6 +1174,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         return searchIncludingRemoved(sc, filter, null, false);
     }
 
+    @Override
     public Pair<List<VMInstanceVO>, Integer> listByVmsNotInClusterUsingPool(long clusterId, long poolId) {
         SearchCriteria<VMInstanceVO> sc = VmsNotInClusterUsingPool.create();
         sc.setParameters("vmStates", State.Starting, State.Running, State.Stopping, State.Migrating, State.Restoring);
@@ -1069,4 +1184,44 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
         List<VMInstanceVO> uniqueVms = vms.stream().distinct().collect(Collectors.toList());
         return new Pair<>(uniqueVms, uniqueVms.size());
     }
+
+    @Override
+    public List<VMInstanceVO> listIdServiceOfferingForUpVmsByHostId(Long hostId) {
+        SearchCriteria<VMInstanceVO> sc = IdServiceOfferingIdSelectSearch.create();
+        sc.setParameters("host", hostId);
+        sc.setParameters("states", new Object[] {State.Starting, State.Running, State.Stopping, State.Migrating});
+        return customSearch(sc, null);
+    }
+
+    @Override
+    public List<VMInstanceVO> listIdServiceOfferingForVmsMigratingFromHost(Long hostId) {
+        SearchCriteria<VMInstanceVO> sc = IdServiceOfferingIdSelectSearch.create();
+        sc.setParameters("lastHost", hostId);
+        sc.setParameters("state", State.Migrating);
+        return customSearch(sc, null);
+    }
+
+    @Override
+    public Map<String, Long> getNameIdMapForVmInstanceNames(Collection<String> names) {
+        SearchBuilder<VMInstanceVO> sb = createSearchBuilder();
+        sb.and("name", sb.entity().getInstanceName(), Op.IN);
+        sb.selectFields(sb.entity().getId(), sb.entity().getInstanceName());
+        SearchCriteria<VMInstanceVO> sc = sb.create();
+        sc.setParameters("name", names.toArray());
+        List<VMInstanceVO> vms = customSearch(sc, null);
+        return vms.stream()
+                .collect(Collectors.toMap(VMInstanceVO::getInstanceName, VMInstanceVO::getId));
+    }
+
+    @Override
+    public Map<String, Long> getNameIdMapForVmIds(Collection<Long> ids) {
+        SearchBuilder<VMInstanceVO> sb = createSearchBuilder();
+        sb.and("id", sb.entity().getId(), Op.IN);
+        sb.selectFields(sb.entity().getId(), sb.entity().getInstanceName());
+        SearchCriteria<VMInstanceVO> sc = sb.create();
+        sc.setParameters("id", ids.toArray());
+        List<VMInstanceVO> vms = customSearch(sc, null);
+        return vms.stream()
+                .collect(Collectors.toMap(VMInstanceVO::getInstanceName, VMInstanceVO::getId));
+    }
 }

engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDao.java

@@ -40,6 +40,13 @@ public interface NetworkPermissionDao extends GenericDao<NetworkPermissionVO, Lo
      */
     void removeAllPermissions(long networkId);
 
+    /**
+     * Removes all network permissions associated with a given account.
+     *
+     * @param accountId The ID of the account from which all network permissions will be removed.
+     */
+    void removeAccountPermissions(long accountId);
+
     /**
      * Find a Network permission by networkId, accountName, and domainId
      *

engine/schema/src/main/java/org/apache/cloudstack/network/dao/NetworkPermissionDaoImpl.java

@@ -33,6 +33,7 @@ public class NetworkPermissionDaoImpl extends GenericDaoBase<NetworkPermissionVO
 
     private SearchBuilder<NetworkPermissionVO> NetworkAndAccountSearch;
     private SearchBuilder<NetworkPermissionVO> NetworkIdSearch;
+    private SearchBuilder<NetworkPermissionVO> accountSearch;
     private GenericSearchBuilder<NetworkPermissionVO, Long> FindNetworkIdsByAccount;
 
     protected NetworkPermissionDaoImpl() {
@@ -45,6 +46,10 @@ public class NetworkPermissionDaoImpl extends GenericDaoBase<NetworkPermissionVO
         NetworkIdSearch.and("networkId", NetworkIdSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
         NetworkIdSearch.done();
 
+        accountSearch = createSearchBuilder();
+        accountSearch.and("accountId", accountSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+        accountSearch.done();
+
         FindNetworkIdsByAccount = createSearchBuilder(Long.class);
         FindNetworkIdsByAccount.select(null, SearchCriteria.Func.DISTINCT, FindNetworkIdsByAccount.entity().getNetworkId());
         FindNetworkIdsByAccount.and("account", FindNetworkIdsByAccount.entity().getAccountId(), SearchCriteria.Op.IN);
@@ -69,6 +74,16 @@ public class NetworkPermissionDaoImpl extends GenericDaoBase<NetworkPermissionVO
         expunge(sc);
     }
 
+    @Override
+    public void removeAccountPermissions(long accountId) {
+        SearchCriteria<NetworkPermissionVO> sc = accountSearch.create();
+        sc.setParameters("accountId", accountId);
+        int networkPermissionRemoved = expunge(sc);
+        if (networkPermissionRemoved > 0) {
+            logger.debug(String.format("Removed [%s] network permission(s) for the account with Id [%s]", networkPermissionRemoved, accountId));
+        }
+    }
+
     @Override
     public NetworkPermissionVO findByNetworkAndAccount(long networkId, long accountId) {
         SearchCriteria<NetworkPermissionVO> sc = NetworkAndAccountSearch.create();

engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDao.java

@@ -88,6 +88,8 @@ public interface ResourceDetailsDao<R extends ResourceDetail> extends GenericDao
 
     public Map<String, String> listDetailsKeyPairs(long resourceId);
 
+    Map<String, String> listDetailsKeyPairs(long resourceId, List<String> keys);
+
     public Map<String, String> listDetailsKeyPairs(long resourceId, boolean forDisplay);
 
     Map<String, Boolean> listDetailsVisibility(long resourceId);

engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java

@@ -19,6 +19,7 @@ package org.apache.cloudstack.resourcedetail;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.apache.cloudstack.api.ResourceDetail;
 import org.apache.commons.collections.CollectionUtils;
@@ -91,6 +92,20 @@ public abstract class ResourceDetailsDaoBase<R extends ResourceDetail> extends G
         return details;
     }
 
+    @Override
+    public Map<String, String> listDetailsKeyPairs(long resourceId, List<String> keys) {
+        SearchBuilder<R> sb = createSearchBuilder();
+        sb.and("resourceId", sb.entity().getResourceId(), SearchCriteria.Op.EQ);
+        sb.and("name", sb.entity().getName(), SearchCriteria.Op.IN);
+        sb.done();
+        SearchCriteria<R> sc = sb.create();
+        sc.setParameters("resourceId", resourceId);
+        sc.setParameters("name", keys.toArray());
+
+        List<R> results = search(sc, null);
+        return results.stream().collect(Collectors.toMap(R::getName, R::getValue));
+    }
+
     public Map<String, Boolean> listDetailsVisibility(long resourceId) {
         SearchCriteria<R> sc = AllFieldsSearch.create();
         sc.setParameters("resourceId", resourceId);

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -28,20 +28,20 @@ import java.util.stream.Collectors;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.storage.Storage;
-import com.cloud.utils.Pair;
-import com.cloud.utils.db.Filter;
 import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.host.Status;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.storage.ScopeType;
+import com.cloud.storage.Storage;
 import com.cloud.storage.StoragePoolHostVO;
 import com.cloud.storage.StoragePoolStatus;
 import com.cloud.storage.StoragePoolTagVO;
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.StoragePoolTagsDao;
+import com.cloud.utils.Pair;
 import com.cloud.utils.db.DB;
+import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.JoinBuilder;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_update_api_permission.sql

@@ -0,0 +1,52 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION` (
+    IN role VARCHAR(255),
+    IN rule VARCHAR(255),
+    IN permission VARCHAR(255)
+)
+BEGIN
+     DECLARE role_id BIGINT(20) UNSIGNED
+;    DECLARE max_sort_order BIGINT(20) UNSIGNED
+
+;   SELECT `r`.`id` INTO role_id
+    FROM `cloud`.`roles` `r`
+    WHERE `r`.`name` = role
+        AND `r`.`is_default` = 1
+
+;   SELECT MAX(`rp`.`sort_order`) INTO max_sort_order
+    FROM `cloud`.`role_permissions` `rp`
+    WHERE `rp`.`role_id` = role_id
+
+;   IF NOT EXISTS (
+        SELECT * FROM `cloud`.`role_permissions` `rp`
+        WHERE `rp`.`role_id` = role_id
+            AND `rp`.`rule` = rule
+    ) THEN
+        UPDATE `cloud`.`role_permissions` `rp`
+        SET `rp`.`sort_order` = max_sort_order + 1
+        WHERE `rp`.`sort_order` = max_sort_order
+            AND `rp`.`role_id` = role_id
+
+;       INSERT INTO `cloud`.`role_permissions`
+            (uuid, role_id, rule, permission, sort_order)
+        VALUES (uuid(), role_id, rule, permission, max_sort_order)
+;   END IF
+;END;

engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql

@@ -21,3 +21,25 @@
 
 -- Add last_id to the volumes table
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'last_id', 'bigint(20) unsigned DEFAULT NULL');
+
+-- Grant access to 2FA APIs for the "Read-Only User - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only User - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only User - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only User - Default', 'listUserTwoFactorAuthenticatorProviders', 'ALLOW');
+
+-- Grant access to 2FA APIs for the "Support User - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support User - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support User - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support User - Default', 'listUserTwoFactorAuthenticatorProviders', 'ALLOW');
+
+-- Grant access to 2FA APIs for the "Read-Only Admin - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only Admin - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Read-Only Admin - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');
+
+-- Grant access to 2FA APIs for the "Support Admin - Default" role
+
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'setupUserTwoFactorAuthentication', 'ALLOW');
+CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Support Admin - Default', 'validateUserTwoFactorAuthenticationCode', 'ALLOW');

engine/schema/src/main/resources/META-INF/db/schema-42000to42010-cleanup.sql

@@ -18,3 +18,6 @@
 --;
 -- Schema upgrade cleanup from 4.20.0.0 to 4.20.1.0
 --;
+
+-- Delete `project_account` entries for users that were removed
+DELETE FROM `cloud`.`project_account` WHERE `user_id` IN (SELECT `id` FROM `cloud`.`user` WHERE `removed`);

engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql

@@ -22,6 +22,7 @@
 -- Add column api_key_access to user and account tables
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the user" AFTER `secret_key`');
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" ');
+CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" ');
 
 -- Modify index for mshost_peer
 DELETE FROM `cloud`.`mshost_peer`;
@@ -35,3 +36,6 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'last_id', 'bigint(20) uns
 
 -- Add used_iops column to support IOPS data in storage stats
 CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.storage_pool', 'used_iops', 'bigint unsigned DEFAULT NULL COMMENT "IOPS currently in use for this storage pool" ');
+
+-- Add reason column for op_ha_work
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.op_ha_work', 'reason', 'varchar(32) DEFAULT NULL COMMENT "Reason for the HA work"');

engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql

@@ -32,6 +32,9 @@ FROM `cloud`.`role_permissions` rp
 WHERE rp.rule = 'quotaStatement'
   AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList');
 
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host', 'last_mgmt_server_id', 'bigint unsigned DEFAULT NULL COMMENT "last management server this host is connected to" AFTER `mgmt_server_id`');
+
+
 -----------------------------------------------------------
 -- CKS Enhancements:
 -----------------------------------------------------------

engine/schema/src/main/resources/META-INF/db/views/cloud.network_offering_view.sql

@@ -76,13 +76,9 @@ SELECT
 FROM
     `cloud`.`network_offerings`
         LEFT JOIN
-    `cloud`.`network_offering_details` AS `domain_details` ON `domain_details`.`network_offering_id` = `network_offerings`.`id` AND `domain_details`.`name`='domainid'
+    `cloud`.`domain` AS `domain` ON `domain`.id IN (SELECT value from `network_offering_details` where `name` = 'domainid' and `network_offering_id` = `network_offerings`.`id`)
         LEFT JOIN
-    `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`)
-        LEFT JOIN
-    `cloud`.`network_offering_details` AS `zone_details` ON `zone_details`.`network_offering_id` = `network_offerings`.`id` AND `zone_details`.`name`='zoneid'
-        LEFT JOIN
-    `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`)
+    `cloud`.`data_center` AS `zone` ON `zone`.`id` IN (SELECT value from `network_offering_details` where `name` = 'zoneid' and `network_offering_id` = `network_offerings`.`id`)
         LEFT JOIN
     `cloud`.`network_offering_details` AS `offering_details` ON `offering_details`.`network_offering_id` = `network_offerings`.`id` AND `offering_details`.`name`='internetProtocol'
 GROUP BY