Merge branch 'main' into cks-enhancements-upstream

2025-01-02 08:27:10 -03:00 · 2025-01-02 08:27:10 -03:00 · b0eb65f31a
parent f8fd8e0377 fd24509ba7
commit b0eb65f31a
535 changed files with 20033 additions and 7943 deletions
--- a/.github/linters/.markdown-lint.yml
+++ b/.github/linters/.markdown-lint.yml
@ -27,27 +27,15 @@ MD004: false
 # MD007/ul-indent Unordered list indentation
 MD007: false

-# MD009/no-trailing-spaces Trailing spaces
-MD009: false
-
 # MD010/no-hard-tabs Hard tabs
 MD010: false

-# MD012/no-multiple-blanks Multiple consecutive blank lines
-MD012: false
-
 # MD013/line-length Line length
 MD013: false

 # MD014/commands-show-output Dollar signs used before commands without showing output
 MD014: false

-# MD018/no-missing-space-atx No space after hash on atx style heading
-MD018: false
-
-# MD019/no-multiple-space-atx Multiple spaces after hash on atx style heading
-MD019: false
-
 # MD022/blanks-around-headings Headings should be surrounded by blank lines
 MD022: false

--- a/.gitignore
+++ b/.gitignore
@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -34,7 +34,7 @@ repos:
      - id: check-merge-conflict
      - id: check-symlinks
      - id: check-vcs-permalinks
-      #- id: check-yaml
+      - id: check-yaml
      - id: destroyed-symlinks
      - id: detect-aws-credentials
        args: [--allow-missing-credentials]
@ -59,7 +59,7 @@ repos:
      - id: mixed-line-ending
        exclude: \.(cs|xml)$
      - id: trailing-whitespace
-        files: \.(header|in|java|md|properties|py|rb|sh|sql|txt|vue|xml|yaml|yml)$
+        files: \.(cs|bat|css|gitignore|header|in|install|java|md|properties|py|rb|sh|sql|txt|vue|xml|xsl|yaml|yml)$
        args: [--markdown-linebreak-ext=md]
        exclude: ^services/console-proxy/rdpconsole/src/test/doc/freerdp-debug-log\.txt$
  - repo: https://github.com/codespell-project/codespell
@ -87,7 +87,7 @@ repos:
        ^setup/bindir/cloud-setup-encryption\.in$|
        ^venv/.*$
  - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.40.0
+    rev: v0.43.0
    hooks:
      - id: markdownlint
        name: run markdownlint
--- a/CHANGES.md
+++ b/CHANGES.md
@ -242,7 +242,6 @@ Bug ID | Description
 [CLOUDSTACK-7722](https://issues.apache.org/jira/browse/CLOUDSTACK-7722) | add.label: Add button for tags show the label not "Add" text...
 [CLOUDSTACK-7246](https://issues.apache.org/jira/browse/CLOUDSTACK-7246) | VM deployment failed due to wrong in  script name createipalias.sh...

-
 Version 4.4.1
 -------------

@ -276,7 +275,6 @@ Bug ID | Description
 [CLOUDSTACK-1632](https://issues.apache.org/jira/browse/CLOUDSTACK-1632) | Mistakes in authorizeSecurityGroup* API docs...
 [CLOUDSTACK-401](https://issues.apache.org/jira/browse/CLOUDSTACK-401)   | Storage options missing from table...

-
 Version 4.4.0
 -------------

@ -930,7 +928,6 @@ Security Fixes:

 * CVE-2012-4501: Apache CloudStack configuration vulnerability

-
 Version 4.0.2
 ------------------------

@ -979,7 +976,6 @@ Issues fixed in this release:
 * CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1.
 * CLOUDSTACK-2091: Error in API documentation for 4.0.x.

-
 Version 4.0.1-incubating
 ------------------------

@ -1023,7 +1019,6 @@ Bugs fixed in this release:
 * CLOUDSTACK-961: Installation docs don't detail dependencies for building RPMs
 * CLOUDSTACK-995: Not able to add the KVM host

-
 Version 4.0.0-incubating
 ------------------------

@ -1056,7 +1051,6 @@ Security Fixes:

 * CVE-2012-4501: Apache CloudStack configuration vulnerability

-
 Updating this file
 ------------------

--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -51,11 +51,9 @@ $ git fetch upstream
 $ git rebase upstream/main
 ```

-
 Making changes
 --------------

-
 It is important that you create a new branch to make changes on and that you do not change the `main` branch (other than to rebase in changes from `upstream/main`).  In this example I will assume you will be making your changes to a branch called `feature_x`.  This `feature_x` branch will be created on your local repository and will be pushed to your forked repository on GitHub.  Once this branch is on your fork you will create a Pull Request for the changes to be added to the ACS project.

 It is best practice to create a new branch each time you want to contribute to the project and only track the changes for that pull request in this branch.
@ -70,7 +68,6 @@ $ git commit -a -m "descriptive commit message for your changes"

 > The `-b` specifies that you want to create a new branch called `feature_x`.  You only specify `-b` the first time you checkout because you are creating a new branch.  Once the `feature_x` branch exists, you can later switch to it with only `git checkout feature_x`.

-
 Rebase `feature_x` to include updates from `upstream/main`
 ------------------------------------------------------------

@ -92,7 +89,6 @@ $ git rebase main

 > Now your `feature_x` branch is up-to-date with all the code in `upstream/main`.

-
 Make a GitHub Pull Request to contribute your changes
 -----------------------------------------------------

@ -107,7 +103,7 @@ $ git push origin main
 $ git push origin feature_x
 ```

-Now that the `feature_x` branch has been pushed to your GitHub repository, you can initiate the pull request.  
+Now that the `feature_x` branch has been pushed to your GitHub repository, you can initiate the pull request.

 To initiate the pull request, do the following:

@ -118,7 +114,6 @@ To initiate the pull request, do the following:

 If you are requested to make modifications to your proposed changes, make the changes locally on your `feature_x` branch, re-push the `feature_x` branch to your fork.  The existing pull request should automatically pick up the change and update accordingly.

-
 Cleaning up after a successful pull request
 -------------------------------------------

--- a/ISSUE_TEMPLATE.md
+++ b/ISSUE_TEMPLATE.md
@ -35,17 +35,14 @@ New line separated list of affected versions, commit ID for issues on main branc
 Information about the configuration if relevant, e.g. basic network, advanced networking, etc.  N/A otherwise
 -->

-
 ##### OS / ENVIRONMENT
 <!--
 Information about the environment if relevant, N/A otherwise
 -->

-
 ##### SUMMARY
 <!-- Explain the problem/feature briefly -->

-
 ##### STEPS TO REPRODUCE
 <!--
 For bugs, show exactly how to reproduce the problem, using a minimal test-case. Use Screenshots if accurate.
--- a/PULL_REQUEST_TEMPLATE.md
+++ b/PULL_REQUEST_TEMPLATE.md
@ -40,10 +40,8 @@ This PR...
 - [ ] Minor
 - [ ] Trivial

-
 ### Screenshots (if appropriate):

-
 ### How Has This Been Tested?

 <!-- Please describe in detail how you tested your changes. -->
@ -53,5 +51,4 @@ This PR...

 <!-- see how your change affects other areas of the code, etc. -->

-
 <!-- Please read the [CONTRIBUTING](https://github.com/apache/cloudstack/blob/main/CONTRIBUTING.md) document -->
--- a/README.md
+++ b/README.md
@ -1,6 +1,6 @@
 # Apache CloudStack [![Build Status](https://github.com/apache/cloudstack/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/build.yml) [![UI Build](https://github.com/apache/cloudstack/actions/workflows/ui.yml/badge.svg)](https://github.com/apache/cloudstack/actions/workflows/ui.yml) [![License Check](https://github.com/apache/cloudstack/actions/workflows/rat.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/rat.yml) [![Simulator CI](https://github.com/apache/cloudstack/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/ci.yml) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache_cloudstack&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache_cloudstack) [![codecov](https://codecov.io/gh/apache/cloudstack/branch/main/graph/badge.svg)](https://codecov.io/gh/apache/cloudstack)

-[![Apache CloudStack](tools/logo/apache_cloudstack.png)](https://cloudstack.apache.org/)
+[![Apache CloudStack](tools/logo/acsxmas.jpg)](https://cloudstack.apache.org/)

 Apache CloudStack is open source software designed to deploy and manage large
 networks of virtual machines, as a highly available, highly scalable
--- a/agent/pom.xml
+++ b/agent/pom.xml
@ -24,7 +24,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
    </parent>
    <dependencies>
        <dependency>
--- a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java
+++ b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java
@ -397,9 +397,8 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe
    }

    public String authenticateConsoleAccess(String host, String port, String vmId, String sid, String ticket,
-                                            Boolean isReauthentication, String sessionToken) {
-
-        ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken);
+                                            Boolean isReauthentication, String sessionToken, String clientAddress) {
+        ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken, clientAddress);
        cmd.setReauthenticating(isReauthentication);

        ConsoleProxyAuthenticationResult result = new ConsoleProxyAuthenticationResult();
--- a/api/pom.xml
+++ b/api/pom.xml
@ -24,7 +24,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
    </parent>
    <dependencies>
        <dependency>
--- a/api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java
+++ b/api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java
@ -19,6 +19,9 @@ package com.cloud.agent.api.to;
 import com.cloud.network.rules.FirewallRule;
 import com.cloud.network.rules.PortForwardingRule;
 import com.cloud.utils.net.NetUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;

 /**
 * PortForwardingRuleTO specifies one port forwarding rule.
@ -29,6 +32,8 @@ public class PortForwardingRuleTO extends FirewallRuleTO {
    String dstIp;
    int[] dstPortRange;

+    List<String> sourceCidrList;
+
    protected PortForwardingRuleTO() {
        super();
    }
@ -37,6 +42,7 @@ public class PortForwardingRuleTO extends FirewallRuleTO {
        super(rule, srcVlanTag, srcIp);
        this.dstIp = rule.getDestinationIpAddress().addr();
        this.dstPortRange = new int[] {rule.getDestinationPortStart(), rule.getDestinationPortEnd()};
+        this.sourceCidrList = rule.getSourceCidrList();
    }

    public PortForwardingRuleTO(long id, String srcIp, int srcPortStart, int srcPortEnd, String dstIp, int dstPortStart, int dstPortEnd, String protocol,
@ -58,4 +64,11 @@ public class PortForwardingRuleTO extends FirewallRuleTO {
        return NetUtils.portRangeToString(dstPortRange);
    }

+    public String getSourceCidrListAsString() {
+        if (sourceCidrList != null) {
+            return StringUtils.join(sourceCidrList, ",");
+        }
+        return null;
+    }
+
 }
--- a/api/src/main/java/com/cloud/bgp/BGPService.java
+++ b/api/src/main/java/com/cloud/bgp/BGPService.java
@ -21,6 +21,7 @@ import com.cloud.network.Network;
 import com.cloud.network.vpc.Vpc;
 import com.cloud.utils.Pair;
 import org.apache.cloudstack.api.command.user.bgp.ListASNumbersCmd;
+import org.apache.cloudstack.network.BgpPeer;

 import java.util.List;

@ -36,4 +37,8 @@ public interface BGPService {
    boolean applyBgpPeers(Network network, boolean continueOnError) throws ResourceUnavailableException;

    boolean applyBgpPeers(Vpc vpc, boolean continueOnError) throws ResourceUnavailableException;
+
+    List<? extends BgpPeer> getBgpPeersForNetwork(Network network);
+
+    List<? extends BgpPeer> getBgpPeersForVpc(Vpc vpc);
 }
--- a/api/src/main/java/com/cloud/event/EventTypes.java
+++ b/api/src/main/java/com/cloud/event/EventTypes.java
@ -292,6 +292,7 @@ public class EventTypes {

    //register for user API and secret keys
    public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY";
+    public static final String API_KEY_ACCESS_UPDATE = "API.KEY.ACCESS.UPDATE";

    // Template Events
    public static final String EVENT_TEMPLATE_CREATE = "TEMPLATE.CREATE";
--- a/api/src/main/java/com/cloud/hypervisor/Hypervisor.java
+++ b/api/src/main/java/com/cloud/hypervisor/Hypervisor.java
@ -20,37 +20,57 @@ import com.cloud.storage.Storage.ImageFormat;
 import org.apache.commons.lang3.StringUtils;

 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.EnumSet;
+import java.util.stream.Collectors;
+
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate;
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.RootDiskSizeOverride;
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.VmStorageMigration;

 public class Hypervisor {
    public static class HypervisorType {
+        public enum Functionality {
+            DirectDownloadTemplate,
+            RootDiskSizeOverride,
+            VmStorageMigration
+        }
+
        private static final Map<String, HypervisorType> hypervisorTypeMap = new LinkedHashMap<>();
        public static final HypervisorType None = new HypervisorType("None"); //for storage hosts
-        public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD);
-        public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2);
-        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA);
+        public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
+        public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2, EnumSet.of(DirectDownloadTemplate, RootDiskSizeOverride, VmStorageMigration));
+        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
        public static final HypervisorType Hyperv = new HypervisorType("Hyperv");
        public static final HypervisorType VirtualBox = new HypervisorType("VirtualBox");
        public static final HypervisorType Parralels = new HypervisorType("Parralels");
        public static final HypervisorType BareMetal = new HypervisorType("BareMetal");
-        public static final HypervisorType Simulator = new HypervisorType("Simulator");
+        public static final HypervisorType Simulator = new HypervisorType("Simulator", null, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
        public static final HypervisorType Ovm = new HypervisorType("Ovm", ImageFormat.RAW);
        public static final HypervisorType Ovm3 = new HypervisorType("Ovm3", ImageFormat.RAW);
        public static final HypervisorType LXC = new HypervisorType("LXC");
-        public static final HypervisorType Custom = new HypervisorType("Custom");
+        public static final HypervisorType Custom = new HypervisorType("Custom", null, EnumSet.of(RootDiskSizeOverride));
        public static final HypervisorType Any = new HypervisorType("Any"); /*If you don't care about the hypervisor type*/
        private final String name;
        private final ImageFormat imageFormat;
+        private final Set<Functionality> supportedFunctionalities;

        public HypervisorType(String name) {
-            this(name, null);
+            this(name, null, EnumSet.noneOf(Functionality.class));
        }

        public HypervisorType(String name, ImageFormat imageFormat) {
+            this(name, imageFormat, EnumSet.noneOf(Functionality.class));
+        }
+
+        public HypervisorType(String name, ImageFormat imageFormat, Set<Functionality> supportedFunctionalities) {
            this.name = name;
            this.imageFormat = imageFormat;
+            this.supportedFunctionalities = supportedFunctionalities;
            if (name.equals("Parralels")){ // typo in the original code
                hypervisorTypeMap.put("parallels", this);
            } else {
@ -81,6 +101,12 @@ public class Hypervisor {
            return hypervisorType;
        }

+        public static List<HypervisorType> getListOfHypervisorsSupportingFunctionality(Functionality functionality) {
+            return hypervisorTypeMap.values().stream()
+                    .filter(hypervisor -> hypervisor.supportedFunctionalities.contains(functionality))
+                    .collect(Collectors.toList());
+        }
+
        /**
         * Returns the display name of a hypervisor type in case the custom hypervisor is used,
         * using the 'hypervisor.custom.display.name' setting. Otherwise, returns hypervisor name
@ -102,6 +128,15 @@ public class Hypervisor {
            return name;
        }

+        /**
+         * Make this method to be part of the properties of the hypervisor type itself.
+         *
+         * @return true if the hypervisor plugin support the specified functionality
+         */
+        public boolean isFunctionalitySupported(Functionality functionality) {
+            return supportedFunctionalities.contains(functionality);
+        }
+
        @Override
        public int hashCode() {
            return Objects.hash(name);
--- a/api/src/main/java/com/cloud/network/rules/RulesService.java
+++ b/api/src/main/java/com/cloud/network/rules/RulesService.java
@ -26,6 +26,7 @@ import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.user.Account;
 import com.cloud.utils.Pair;
 import com.cloud.utils.net.Ip;
+import org.apache.cloudstack.api.command.user.firewall.UpdatePortForwardingRuleCmd;

 public interface RulesService {
    Pair<List<? extends FirewallRule>, Integer> searchStaticNatRules(Long ipId, Long id, Long vmId, Long start, Long size, String accountName, Long domainId,
@ -81,6 +82,8 @@ public interface RulesService {

    boolean disableStaticNat(long ipId) throws ResourceUnavailableException, NetworkRuleConflictException, InsufficientAddressCapacityException;

-    PortForwardingRule updatePortForwardingRule(long id, Integer privatePort, Integer privateEndPort, Long virtualMachineId, Ip vmGuestIp, String customId, Boolean forDisplay);
+    PortForwardingRule updatePortForwardingRule(UpdatePortForwardingRuleCmd cmd);
+
+    void  validatePortForwardingSourceCidrList(List<String> sourceCidrList);

 }
--- a/api/src/main/java/com/cloud/server/ManagementServerHostStats.java
+++ b/api/src/main/java/com/cloud/server/ManagementServerHostStats.java
@ -32,6 +32,8 @@ public interface ManagementServerHostStats {

    String getManagementServerHostUuid();

+    long getManagementServerRunId();
+
    long getSessions();

    double getCpuUtilization();
--- a/api/src/main/java/com/cloud/user/Account.java
+++ b/api/src/main/java/com/cloud/user/Account.java
@ -93,4 +93,8 @@ public interface Account extends ControlledEntity, InternalIdentity, Identity {

    boolean isDefault();

+    public void setApiKeyAccess(Boolean apiKeyAccess);
+
+    public Boolean getApiKeyAccess();
+
 }
--- a/api/src/main/java/com/cloud/user/AccountService.java
+++ b/api/src/main/java/com/cloud/user/AccountService.java
@ -19,6 +19,7 @@ package com.cloud.user;
 import java.util.List;
 import java.util.Map;

+import com.cloud.utils.Pair;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@ -116,6 +117,8 @@ public interface AccountService {

    void checkAccess(Account account, AccessType accessType, boolean sameOwner, String apiName, ControlledEntity... entities) throws PermissionDeniedException;

+    void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource);
+
    Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly);

    /**
@ -125,9 +128,9 @@ public interface AccountService {
     */
    UserAccount getUserAccountById(Long userId);

-    public Map<String, String> getKeys(GetUserKeysCmd cmd);
+    public Pair<Boolean, Map<String, String>> getKeys(GetUserKeysCmd cmd);

-    public Map<String, String> getKeys(Long userId);
+    public Pair<Boolean, Map<String, String>> getKeys(Long userId);

    /**
     * Lists user two-factor authentication provider plugins
--- a/api/src/main/java/com/cloud/user/User.java
+++ b/api/src/main/java/com/cloud/user/User.java
@ -94,4 +94,9 @@ public interface User extends OwnedBy, InternalIdentity {
    public boolean isUser2faEnabled();

    public String getKeyFor2fa();
+
+    public void setApiKeyAccess(Boolean apiKeyAccess);
+
+    public Boolean getApiKeyAccess();
+
 }
--- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java
+++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java
@ -74,6 +74,7 @@ public interface VmDetailConstants {
    String ENCRYPTED_PASSWORD = "Encrypted.Password";

    String CONFIG_DRIVE_LOCATION = "configDriveLocation";
+    String LAST_CONFIG_DRIVE_LOCATION = "lastConfigDriveLocation";

    String SKIP_DRS = "skipFromDRS";

--- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
@ -35,6 +35,7 @@ public class ApiConstants {
    public static final String ALLOW_USER_FORCE_STOP_VM = "allowuserforcestopvm";
    public static final String ANNOTATION = "annotation";
    public static final String API_KEY = "apikey";
+    public static final String API_KEY_ACCESS = "apikeyaccess";
    public static final String ARCHIVED = "archived";
    public static final String ARCH = "arch";
    public static final String AS_NUMBER = "asnumber";
@ -244,6 +245,7 @@ public class ApiConstants {
    public static final String ICMP_TYPE = "icmptype";
    public static final String ID = "id";
    public static final String IDS = "ids";
+    public static final String IMPORT_INSTANCE_HOST_ID = "importinstancehostid";
    public static final String INDEX = "index";
    public static final String INSTANCES_DISKS_STATS_RETENTION_ENABLED = "instancesdisksstatsretentionenabled";
    public static final String INSTANCES_DISKS_STATS_RETENTION_TIME = "instancesdisksstatsretentiontime";
@ -385,6 +387,14 @@ public class ApiConstants {
    public static final String PATH = "path";
    public static final String PAYLOAD = "payload";
    public static final String PAYLOAD_URL = "payloadurl";
+    public static final String PEERS = "peers";
+    public static final String PEER_ID = "peerid";
+    public static final String PEER_NAME = "peername";
+    public static final String PEER_MSID = "peermsid";
+    public static final String PEER_RUNID = "peerrunid";
+    public static final String PEER_SERVICE_IP = "peerserviceip";
+    public static final String PEER_SERVICE_PORT = "peerserviceport";
+    public static final String PEER_STATE = "peerstate";
    public static final String POD_ID = "podid";
    public static final String POD_NAME = "podname";
    public static final String POD_IDS = "podids";
@ -459,6 +469,7 @@ public class ApiConstants {
    public static final String SNAPSHOT_POLICY_ID = "snapshotpolicyid";
    public static final String SNAPSHOT_TYPE = "snapshottype";
    public static final String SNAPSHOT_QUIESCEVM = "quiescevm";
+    public static final String SOURCE_CIDR_LIST = "sourcecidrlist";
    public static final String SOURCE_ZONE_ID = "sourcezoneid";
    public static final String SSL_VERIFICATION = "sslverification";
    public static final String START_ASN = "startasn";
@ -997,6 +1008,7 @@ public class ApiConstants {
    public static final String ACL_NAME = "aclname";
    public static final String NUMBER = "number";
    public static final String IS_DYNAMICALLY_SCALABLE = "isdynamicallyscalable";
+    public static final String ROUTED_MODE_ENABLED = "routedmodeenabled";
    public static final String ROUTING = "isrouting";
    public static final String ROUTING_MODE = "routingmode";
    public static final String MAX_CONNECTIONS = "maxconnections";
@ -1255,4 +1267,30 @@ public class ApiConstants {
    public enum DomainDetails {
        all, resource, min;
    }
+
+    public enum ApiKeyAccess {
+        DISABLED(false),
+        ENABLED(true),
+        INHERIT(null);
+
+        Boolean apiKeyAccess;
+
+        ApiKeyAccess(Boolean keyAccess) {
+            apiKeyAccess = keyAccess;
+        }
+
+        public Boolean toBoolean() {
+            return apiKeyAccess;
+        }
+
+        public static ApiKeyAccess fromBoolean(Boolean value) {
+            if (value == null) {
+                return INHERIT;
+            } else if (value) {
+                return ENABLED;
+            } else {
+                return DISABLED;
+            }
+        }
+    }
 }
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/account/UpdateAccountCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/account/UpdateAccountCmd.java
@ -21,7 +21,9 @@ import java.util.Map;

 import javax.inject.Inject;

+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.RoleResponse;

 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@ -40,8 +42,8 @@ import org.apache.cloudstack.region.RegionService;
 import com.cloud.user.Account;

@APICommand(name = "updateAccount", description = "Updates account information for the authenticated user", responseObject = AccountResponse.class, entityType = {Account.class},
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
-public class UpdateAccountCmd extends BaseCmd {
+        responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+public class UpdateAccountCmd extends BaseCmd implements UserCmd {

    /////////////////////////////////////////////////////
    //////////////// API parameters /////////////////////
@ -70,6 +72,9 @@ public class UpdateAccountCmd extends BaseCmd {
    @Parameter(name = ApiConstants.ACCOUNT_DETAILS, type = CommandType.MAP, description = "Details for the account used to store specific parameters")
    private Map details;

+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "Determines if Api key access for this user is enabled, disabled or inherits the value from its parent, the domain level setting api.key.access", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
    @Inject
    RegionService _regionService;

@ -109,6 +114,10 @@ public class UpdateAccountCmd extends BaseCmd {
        return params;
    }

+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
    /////////////////////////////////////////////////////
    /////////////// API Implementation///////////////////
    /////////////////////////////////////////////////////
@ -131,7 +140,7 @@ public class UpdateAccountCmd extends BaseCmd {
    public void execute() {
        Account result = _regionService.updateAccount(this);
        if (result != null){
-            AccountResponse response = _responseGenerator.createAccountResponse(ResponseView.Full, result);
+            AccountResponse response = _responseGenerator.createAccountResponse(getResponseView(), result);
            response.setResponseName(getCommandName());
            setResponseObject(response);
        } else {
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java
@ -23,6 +23,7 @@ import org.apache.cloudstack.api.BaseListCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ManagementServerResponse;
+import org.apache.commons.lang3.BooleanUtils;

@APICommand(name = "listManagementServers", description = "Lists management servers.", responseObject = ManagementServerResponse.class,
        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@ -39,6 +40,11 @@ public class ListMgmtsCmd extends BaseListCmd {
    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the management server")
    private String hostName;

+    @Parameter(name = ApiConstants.PEERS, type = CommandType.BOOLEAN,
+            description = "Whether to return the management server peers or not. By default, the management server peers will not be returned.",
+            since = "4.20.1.0")
+    private Boolean peers;
+
    /////////////////////////////////////////////////////
    /////////////////// Accessors ///////////////////////
    /////////////////////////////////////////////////////
@ -51,6 +57,10 @@ public class ListMgmtsCmd extends BaseListCmd {
        return hostName;
    }

+    public Boolean getPeers() {
+        return BooleanUtils.toBooleanDefaultIfNull(peers, false);
+    }
+
    /////////////////////////////////////////////////////
    /////////////// API Implementation///////////////////
    /////////////////////////////////////////////////////
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserKeysCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserKeysCmd.java
@ -20,6 +20,7 @@ package org.apache.cloudstack.api.command.admin.user;

 import com.cloud.user.Account;
 import com.cloud.user.User;
+import com.cloud.utils.Pair;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@ -54,11 +55,13 @@ public class GetUserKeysCmd extends BaseCmd{
        else return Account.ACCOUNT_ID_SYSTEM;
    }
    public void execute(){
-        Map<String, String> keys = _accountService.getKeys(this);
+        Pair<Boolean, Map<String, String>> keys = _accountService.getKeys(this);
+
        RegisterResponse response = new RegisterResponse();
        if(keys != null){
-            response.setApiKey(keys.get("apikey"));
-            response.setSecretKey(keys.get("secretkey"));
+            response.setApiKeyAccess(keys.first());
+            response.setApiKey(keys.second().get("apikey"));
+            response.setSecretKey(keys.second().get("secretkey"));
        }

        response.setObjectName("userkeys");
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/ListUsersCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/ListUsersCmd.java
@ -16,23 +16,29 @@
 // under the License.
 package org.apache.cloudstack.api.command.admin.user;

+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.server.ResourceIcon;
 import com.cloud.server.ResourceTag;
 import com.cloud.user.Account;
+import com.cloud.user.User;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.ResourceIconResponse;

 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseListAccountResourcesCmd;
 import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.UserResponse;
+import org.apache.commons.lang3.EnumUtils;

 import java.util.List;

@APICommand(name = "listUsers", description = "Lists user accounts", responseObject = UserResponse.class,
-        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
-public class ListUsersCmd extends BaseListAccountResourcesCmd {
+        responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
+public class ListUsersCmd extends BaseListAccountResourcesCmd implements UserCmd {


    /////////////////////////////////////////////////////
@ -53,10 +59,17 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd {
    @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "List user by the username")
    private String username;

+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "List users by the Api key access value", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
    @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, type = CommandType.BOOLEAN,
            description = "flag to display the resource icon for users")
    private Boolean showIcon;

+    @Parameter(name = ApiConstants.USER_SOURCE, type = CommandType.STRING, since = "4.21.0.0",
+            description = "List users by their authentication source. Valid values are: native, ldap, saml2 and saml2disabled.")
+    private String userSource;
+
    /////////////////////////////////////////////////////
    /////////////////// Accessors ///////////////////////
    /////////////////////////////////////////////////////
@ -77,17 +90,38 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd {
        return username;
    }

+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
    public Boolean getShowIcon() {
        return showIcon != null ? showIcon : false;
    }

+    public User.Source getUserSource() {
+        if (userSource == null) {
+            return null;
+        }
+
+        User.Source source = EnumUtils.getEnumIgnoreCase(User.Source.class, userSource);
+        if (source == null || List.of(User.Source.OAUTH2, User.Source.UNKNOWN).contains(source)) {
+            throw new InvalidParameterValueException(String.format("Invalid user source: %s. Valid values are: native, ldap, saml2 and saml2disabled.", userSource));
+        }
+
+        if (source == User.Source.NATIVE) {
+            return User.Source.UNKNOWN;
+        }
+
+        return source;
+    }
+
    /////////////////////////////////////////////////////
    /////////////// API Implementation///////////////////
    /////////////////////////////////////////////////////

    @Override
    public void execute() {
-        ListResponse<UserResponse> response = _queryService.searchForUsers(this);
+        ListResponse<UserResponse> response = _queryService.searchForUsers(getResponseView(), this);
        response.setResponseName(getCommandName());
        this.setResponseObject(response);
        if (response != null && response.getCount() > 0 && getShowIcon()) {
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java
@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.admin.user;

 import javax.inject.Inject;

+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@ -69,6 +70,9 @@ public class UpdateUserCmd extends BaseCmd {
    @Parameter(name = ApiConstants.USER_SECRET_KEY, type = CommandType.STRING, description = "The secret key for the user. Must be specified with userApiKey")
    private String secretKey;

+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "Determines if Api key access for this user is enabled, disabled or inherits the value from its parent, the owning account", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
    @Parameter(name = ApiConstants.TIMEZONE,
            type = CommandType.STRING,
            description = "Specifies a timezone for this command. For more information on the timezone parameter, see Time Zone Format.")
@ -120,6 +124,10 @@ public class UpdateUserCmd extends BaseCmd {
        return secretKey;
    }

+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
    public String getTimezone() {
        return timezone;
    }
--- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java
@ -144,15 +144,19 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd {
    private String clusterName;

    @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
-            description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.")
+            description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v conversion from VMware to KVM.")
    private Long convertInstanceHostId;

+    @Parameter(name = ApiConstants.IMPORT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, since = "4.19.2",
+            description = "(only for importing VMs from VMware to KVM) optional - the host to import the converted instance from VMware to KVM.")
+    private Long importInstanceHostId;
+
    @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class,
            description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.")
    private Long convertStoragePoolId;

    @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN,
-            description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to import VM file(s) to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.")
+            description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to export OVF from VMware to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.")
    private Boolean forceMsToImportVmFiles;

    /////////////////////////////////////////////////////
@ -199,6 +203,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd {
        return convertInstanceHostId;
    }

+    public Long getImportInstanceHostId() {
+        return importInstanceHostId;
+    }
+
    public Long getConvertStoragePoolId() {
        return convertStoragePoolId;
    }
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java
@ -20,6 +20,7 @@ import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;

+import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@ -70,6 +71,9 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC
               description = "comma separated list of account details requested, value can be a list of [ all, resource, min]")
    private List<String> viewDetails;

+    @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "List accounts by the Api key access value", since = "4.20.1.0", authorized = {RoleType.Admin})
+    private String apiKeyAccess;
+
    @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, type = CommandType.BOOLEAN,
            description = "flag to display the resource icon for accounts")
    private Boolean showIcon;
@ -120,6 +124,10 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC
        return dv;
    }

+    public String getApiKeyAccess() {
+        return apiKeyAccess;
+    }
+
    public boolean getShowIcon() {
        return showIcon != null ? showIcon : false;
    }
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java
@ -82,7 +82,7 @@ public class ListBackupScheduleCmd extends BaseCmd {
            List<BackupSchedule> schedules = backupManager.listBackupSchedule(getVmId());
            ListResponse<BackupScheduleResponse> response = new ListResponse<>();
            List<BackupScheduleResponse> scheduleResponses = new ArrayList<>();
-            if (CollectionUtils.isNullOrEmpty(schedules)) {
+            if (!CollectionUtils.isNullOrEmpty(schedules)) {
                for (BackupSchedule schedule : schedules) {
                    scheduleResponses.add(_responseGenerator.createBackupScheduleResponse(schedule));
                }
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java
@ -105,8 +105,12 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P
                description = "the ID of the virtual machine for the port forwarding rule")
    private Long virtualMachineId;

-    @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, description = "the cidr list to forward traffic from. Multiple entries must be separated by a single comma character (,). This parameter is deprecated. Do not use.")
-    private List<String> cidrlist;
+    @Parameter(name = ApiConstants.CIDR_LIST,
+            type = CommandType.LIST,
+            collectionType = CommandType.STRING,
+            description = " the source CIDR list to allow traffic from; all other CIDRs will be blocked. " +
+                    "Multiple entries must be separated by a single comma character (,). This param will be used only for VPC tiers. By default, all CIDRs are allowed.")
+    private List<String> sourceCidrList;

    @Parameter(name = ApiConstants.OPEN_FIREWALL, type = CommandType.BOOLEAN, description = "if true, firewall rule for source/end public port is automatically created; "
        + "if false - firewall rule has to be created explicitly. If not specified 1) defaulted to false when PF"
@ -155,11 +159,7 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P

    @Override
    public List<String> getSourceCidrList() {
-        if (cidrlist != null) {
-            throw new InvalidParameterValueException("Parameter cidrList is deprecated; if you need to open firewall "
-                + "rule for the specific cidr, please refer to createFirewallRule command");
-        }
-        return null;
+        return sourceCidrList;
    }

    public Boolean getOpenFirewall() {
@ -332,12 +332,6 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P

    @Override
    public void create() {
-        // cidr list parameter is deprecated
-        if (cidrlist != null) {
-            throw new InvalidParameterValueException(
-                "Parameter cidrList is deprecated; if you need to open firewall rule for the specific cidr, please refer to createFirewallRule command");
-        }
-
        Ip privateIp = getVmSecondaryIp();
        if (privateIp != null) {
            if (!NetUtils.isValidIp4(privateIp.toString())) {
@ -345,6 +339,8 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P
            }
        }

+        _rulesService.validatePortForwardingSourceCidrList(sourceCidrList);
+
        try {
            PortForwardingRule result = _rulesService.createPortForwardingRule(this, virtualMachineId, privateIp, getOpenFirewall(), isDisplay());
            setEntityId(result.getId());
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java
@ -32,6 +32,8 @@ import com.cloud.network.rules.PortForwardingRule;
 import com.cloud.user.Account;
 import com.cloud.utils.net.Ip;

+import java.util.List;
+
@APICommand(name = "updatePortForwardingRule",
            responseObject = FirewallRuleResponse.class,
        description = "Updates a port forwarding rule. Only the private port and the virtual machine can be updated.", entityType = {PortForwardingRule.class},
@ -63,6 +65,13 @@ public class UpdatePortForwardingRuleCmd extends BaseAsyncCustomIdCmd {
    @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "an optional field, whether to the display the rule to the end user or not", since = "4.4", authorized = {RoleType.Admin})
    private Boolean display;

+    @Parameter(name = ApiConstants.CIDR_LIST,
+            type = CommandType.LIST,
+            collectionType = CommandType.STRING,
+            description = " the source CIDR list to allow traffic from; all other CIDRs will be blocked. " +
+                    "Multiple entries must be separated by a single comma character (,). This param will be used only for VPC tiers. By default, all CIDRs are allowed.")
+    private List<String> sourceCidrList;
+
    /////////////////////////////////////////////////////
    /////////////////// Accessors ///////////////////////
    /////////////////////////////////////////////////////
@ -94,6 +103,10 @@ public class UpdatePortForwardingRuleCmd extends BaseAsyncCustomIdCmd {
        return display;
    }

+    public List<String> getSourceCidrList() {
+        return sourceCidrList;
+    }
+
    /////////////////////////////////////////////////////
    /////////////// API Implementation///////////////////
    /////////////////////////////////////////////////////
@ -130,7 +143,7 @@ public class UpdatePortForwardingRuleCmd extends BaseAsyncCustomIdCmd {

    @Override
    public void execute() {
-        PortForwardingRule rule = _rulesService.updatePortForwardingRule(getId(), getPrivatePort(), getPrivateEndPort(), getVirtualMachineId(), getVmGuestIp(), getCustomId(), getDisplay());
+        PortForwardingRule rule = _rulesService.updatePortForwardingRule(this);
        FirewallRuleResponse fwResponse = new FirewallRuleResponse();
        if (rule != null) {
            fwResponse = _responseGenerator.createPortForwardingRuleResponse(rule);
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java
@ -104,7 +104,7 @@ public class CreateLoadBalancerRuleCmd extends BaseAsyncCreateCmd /*implements L
    @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "the domain ID associated with the load balancer")
    private Long domainId;

-    @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, since = "4.18.0.0", description = "the CIDR list to allow traffic, "
+    @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, since = "4.18.0.0", description = "the source CIDR list to allow traffic from; "
            + "all other CIDRs will be blocked. Multiple entries must be separated by a single comma character (,). By default, all CIDRs are allowed.")
    private List<String> cidrlist;

--- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java
@ -371,7 +371,9 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
                    "Parameter zoneids cannot combine all zones (-1) option with other zones");

        String customHypervisor = HypervisorGuru.HypervisorCustomDisplayName.value();
-        if (isDirectDownload() && !(getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString())
+        if (isDirectDownload() &&
+                !(Hypervisor.HypervisorType.getType(getHypervisor())
+                        .isFunctionalitySupported(Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate)
                || getHypervisor().equalsIgnoreCase(customHypervisor))) {
            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Parameter directdownload " +
                    "is only allowed for KVM or %s templates", customHypervisor));
--- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java
+++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java
@ -73,6 +73,10 @@ public class ResizeVolumeCmd extends BaseAsyncCmd implements UserCmd {
               description = "new disk offering id")
    private Long newDiskOfferingId;

+    @Parameter(name = ApiConstants.AUTO_MIGRATE, type = CommandType.BOOLEAN, required = false,
+            description = "Flag to allow automatic migration of the volume to another suitable storage pool that accommodates the new size", since = "4.20.1")
+    private Boolean autoMigrate;
+
    /////////////////////////////////////////////////////
    /////////////////// Accessors ///////////////////////
    /////////////////////////////////////////////////////
@ -129,6 +133,10 @@ public class ResizeVolumeCmd extends BaseAsyncCmd implements UserCmd {
        return newDiskOfferingId;
    }

+    public boolean getAutoMigrate() {
+        return autoMigrate == null ? false : autoMigrate;
+    }
+
    /////////////////////////////////////////////////////
    /////////////// API Implementation///////////////////
    /////////////////////////////////////////////////////
--- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java
@ -271,6 +271,10 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou
    @Param(description = "The tagged resource limit and count for the account", since = "4.20.0")
    List<TaggedResourceLimitAndCountResponse> taggedResources;

+    @SerializedName(ApiConstants.API_KEY_ACCESS)
+    @Param(description = "whether api key access is Enabled, Disabled or set to Inherit (it inherits the value from the parent)", since = "4.20.1.0")
+    ApiConstants.ApiKeyAccess apiKeyAccess;
+
    @Override
    public String getObjectId() {
        return id;
@ -554,4 +558,8 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou
    public void setTaggedResourceLimitsAndCounts(List<TaggedResourceLimitAndCountResponse> taggedResourceLimitsAndCounts) {
        this.taggedResources = taggedResourceLimitsAndCounts;
    }
+
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = ApiConstants.ApiKeyAccess.fromBoolean(apiKeyAccess);
+    }
 }
--- a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java
@ -24,7 +24,9 @@ import org.apache.cloudstack.api.BaseResponse;
 import org.apache.cloudstack.api.EntityReference;
 import org.apache.cloudstack.management.ManagementServerHost.State;

+import java.util.ArrayList;
 import java.util.Date;
+import java.util.List;

@EntityReference(value = ManagementServerHost.class)
 public class ManagementServerResponse extends BaseResponse {
@ -76,6 +78,10 @@ public class ManagementServerResponse extends BaseResponse {
    @Param(description = "the IP Address for this Management Server")
    private String serviceIp;

+    @SerializedName(ApiConstants.PEERS)
+    @Param(description = "the Management Server Peers")
+    private List<PeerManagementServerNodeResponse> peers;
+
    public String getId() {
        return this.id;
    }
@ -171,4 +177,19 @@ public class ManagementServerResponse extends BaseResponse {
    public String getKernelVersion() {
        return kernelVersion;
    }
+
+    public List<PeerManagementServerNodeResponse> getPeers() {
+        return peers;
+    }
+
+    public void setPeers(List<PeerManagementServerNodeResponse> peers) {
+        this.peers = peers;
+    }
+
+    public void addPeer(PeerManagementServerNodeResponse peer) {
+        if (peers == null) {
+            peers = new ArrayList<>();
+        }
+        peers.add(peer);
+    }
 }
--- a/api/src/main/java/org/apache/cloudstack/api/response/PeerManagementServerNodeResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/PeerManagementServerNodeResponse.java
@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.management.ManagementServerHost.State;
+
+import java.util.Date;
+
+public class PeerManagementServerNodeResponse extends BaseResponse {
+
+    @SerializedName(ApiConstants.STATE)
+    @Param(description = "the state of the management server peer")
+    private State state;
+
+    @SerializedName(ApiConstants.LAST_UPDATED)
+    @Param(description = "the last updated time of the management server peer state")
+    private Date lastUpdated;
+
+    @SerializedName(ApiConstants.PEER_ID)
+    @Param(description = "the ID of the peer management server")
+    private String peerId;
+
+    @SerializedName(ApiConstants.PEER_NAME)
+    @Param(description = "the name of the peer management server")
+    private String peerName;
+
+    @SerializedName(ApiConstants.PEER_MSID)
+    @Param(description = "the management ID of the peer management server")
+    private String peerMsId;
+
+    @SerializedName(ApiConstants.PEER_RUNID)
+    @Param(description = "the run ID of the peer management server")
+    private String peerRunId;
+
+    @SerializedName(ApiConstants.PEER_STATE)
+    @Param(description = "the state of the peer management server")
+    private String peerState;
+
+    @SerializedName(ApiConstants.PEER_SERVICE_IP)
+    @Param(description = "the IP Address for the peer Management Server")
+    private String peerServiceIp;
+
+    @SerializedName(ApiConstants.PEER_SERVICE_PORT)
+    @Param(description = "the service port for the peer Management Server")
+    private String peerServicePort;
+
+    public void setState(State state) {
+        this.state = state;
+    }
+
+    public void setLastUpdated(Date lastUpdated) {
+        this.lastUpdated = lastUpdated;
+    }
+
+    public void setPeerId(String peerId) {
+        this.peerId = peerId;
+    }
+
+    public void setPeerName(String peerName) {
+        this.peerName = peerName;
+    }
+
+    public void setPeerMsId(String peerMsId) {
+        this.peerMsId = peerMsId;
+    }
+
+    public void setPeerRunId(String peerRunId) {
+        this.peerRunId = peerRunId;
+    }
+
+    public void setPeerState(String peerState) {
+        this.peerState = peerState;
+    }
+
+    public void setPeerServiceIp(String peerServiceIp) {
+        this.peerServiceIp = peerServiceIp;
+    }
+
+    public void setPeerServicePort(String peerServicePort) {
+        this.peerServicePort = peerServicePort;
+    }
+}
--- a/api/src/main/java/org/apache/cloudstack/api/response/RegisterResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/RegisterResponse.java
@ -18,19 +18,24 @@ package org.apache.cloudstack.api.response;

 import com.google.gson.annotations.SerializedName;

+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;

 import com.cloud.serializer.Param;

 public class RegisterResponse extends BaseResponse {
-    @SerializedName("apikey")
+    @SerializedName(ApiConstants.API_KEY)
    @Param(description = "the api key of the registered user", isSensitive = true)
    private String apiKey;

-    @SerializedName("secretkey")
+    @SerializedName(ApiConstants.SECRET_KEY)
    @Param(description = "the secret key of the registered user", isSensitive = true)
    private String secretKey;

+    @SerializedName(ApiConstants.API_KEY_ACCESS)
+    @Param(description = "whether api key access is allowed or not", isSensitive = true)
+    private Boolean apiKeyAccess;
+
    public String getApiKey() {
        return apiKey;
    }
@ -46,4 +51,8 @@ public class RegisterResponse extends BaseResponse {
    public void setSecretKey(String secretKey) {
        this.secretKey = secretKey;
    }
+
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = apiKeyAccess;
+    }
 }
--- a/api/src/main/java/org/apache/cloudstack/api/response/UserResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/UserResponse.java
@ -67,7 +67,7 @@ public class UserResponse extends BaseResponse implements SetResourceIconRespons
    @Param(description = "the account type of the user")
    private Integer accountType;

-    @SerializedName("usersource")
+    @SerializedName(ApiConstants.USER_SOURCE)
    @Param(description = "the source type of the user in lowercase, such as native, ldap, saml2")
    private String userSource;

@ -128,6 +128,10 @@ public class UserResponse extends BaseResponse implements SetResourceIconRespons
    @Param(description = "true if user has two factor authentication is mandated", since = "4.18.0.0")
    private Boolean is2FAmandated;

+    @SerializedName(ApiConstants.API_KEY_ACCESS)
+    @Param(description = "whether api key access is Enabled, Disabled or set to Inherit (it inherits the value from the parent)", since = "4.20.1.0")
+    ApiConstants.ApiKeyAccess apiKeyAccess;
+
    @Override
    public String getObjectId() {
        return this.getId();
@ -309,4 +313,8 @@ public class UserResponse extends BaseResponse implements SetResourceIconRespons
    public void set2FAmandated(Boolean is2FAmandated) {
        this.is2FAmandated = is2FAmandated;
    }
+
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = ApiConstants.ApiKeyAccess.fromBoolean(apiKeyAccess);
+    }
 }
--- a/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java
+++ b/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java
@ -157,6 +157,11 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso
    @Param(description = "AS Number Range")
    private String asnRange;

+    @SerializedName(ApiConstants.ROUTED_MODE_ENABLED)
+    @Param(description = "true, if routed network/vpc is enabled", since = "4.20.1")
+    private boolean routedModeEnabled = false;
+
+
    public ZoneResponse() {
        tags = new LinkedHashSet<ResourceTagResponse>();
    }
@ -412,4 +417,12 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso
    public String getAsnRange() {
        return asnRange;
    }
+
+    public boolean isRoutedModeEnabled() {
+        return routedModeEnabled;
+    }
+
+    public void setRoutedModeEnabled(boolean routedModeEnabled) {
+        this.routedModeEnabled = routedModeEnabled;
+    }
 }
--- a/api/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManager.java
+++ b/api/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManager.java
@ -44,7 +44,7 @@ public interface ConsoleAccessManager extends Manager, Configurable {

    void removeSessions(String[] sessionUuids);

-    void acquireSession(String sessionUuid);
+    void acquireSession(String sessionUuid, String clientAddress);

    String genAccessTicket(String host, String port, String sid, String tag, String sessionUuid);
    String genAccessTicket(String host, String port, String sid, String tag, Date normalizedHashTime, String sessionUuid);
--- a/api/src/main/java/org/apache/cloudstack/network/RoutedIpv4Manager.java
+++ b/api/src/main/java/org/apache/cloudstack/network/RoutedIpv4Manager.java
@ -57,6 +57,13 @@ import java.util.List;

 public interface RoutedIpv4Manager extends PluggableService, Configurable {

+    ConfigKey<Boolean> RoutedNetworkVpcEnabled = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK, Boolean.class,
+            "routed.network.vpc.enabled",
+            "true",
+            "If true, the Routed network and VPC are enabled in the zone.",
+            true,
+            ConfigKey.Scope.Zone);
+
    ConfigKey<Integer> RoutedNetworkIPv4MaxCidrSize = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK, Integer.class,
            "routed.network.ipv4.max.cidr.size", "30", "The maximum value of the cidr size for isolated networks in ROUTED mode",
            true, ConfigKey.Scope.Account);
@ -196,4 +203,6 @@ public interface RoutedIpv4Manager extends PluggableService, Configurable {
    void removeBgpPeersByAccountId(long accountId);

    void removeBgpPeersByDomainId(long domainId);
+
+    Boolean isRoutedNetworkVpcEnabled(long zoneId);
 }
--- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java
+++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java
@ -19,6 +19,7 @@ package org.apache.cloudstack.query;
 import java.util.List;

 import org.apache.cloudstack.affinity.AffinityGroupResponse;
+import org.apache.cloudstack.api.ResponseObject;
 import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd;
 import org.apache.cloudstack.api.command.admin.host.ListHostTagsCmd;
 import org.apache.cloudstack.api.command.admin.host.ListHostsCmd;
@ -130,7 +131,7 @@ public interface QueryService {
    ConfigKey<Boolean> ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true",
            "Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global);

-    ListResponse<UserResponse> searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException;
+    ListResponse<UserResponse> searchForUsers(ResponseObject.ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException;

    ListResponse<UserResponse> searchForUsers(Long domainId, boolean recursive) throws PermissionDeniedException;

--- a/client/pom.xml
+++ b/client/pom.xml
@ -25,7 +25,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
    </parent>
    <dependencies>
        <dependency>
--- a/core/pom.xml
+++ b/core/pom.xml
@ -24,7 +24,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
    </parent>
    <dependencies>
        <dependency>
--- a/core/src/main/java/com/cloud/agent/api/ConsoleAccessAuthenticationCommand.java
+++ b/core/src/main/java/com/cloud/agent/api/ConsoleAccessAuthenticationCommand.java
@ -27,6 +27,7 @@ public class ConsoleAccessAuthenticationCommand extends AgentControlCommand {
    private String _sid;
    private String _ticket;
    private String sessionUuid;
+    private String clientAddress;

    private boolean _isReauthenticating;

@ -35,13 +36,14 @@ public class ConsoleAccessAuthenticationCommand extends AgentControlCommand {
    }

    public ConsoleAccessAuthenticationCommand(String host, String port, String vmId, String sid, String ticket,
-                                              String sessiontkn) {
+                                              String sessiontkn, String clientAddress) {
        _host = host;
        _port = port;
        _vmId = vmId;
        _sid = sid;
        _ticket = ticket;
        sessionUuid = sessiontkn;
+        this.clientAddress = clientAddress;
    }

    public String getHost() {
@ -79,4 +81,12 @@ public class ConsoleAccessAuthenticationCommand extends AgentControlCommand {
    public void setSessionUuid(String sessionUuid) {
        this.sessionUuid = sessionUuid;
    }
+
+    public String getClientAddress() {
+        return clientAddress;
+    }
+
+    public void setClientAddress(String clientAddress) {
+        this.clientAddress = clientAddress;
+    }
 }
--- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java
+++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java
@ -20,6 +20,8 @@ import org.apache.cloudstack.vm.UnmanagedInstanceTO;

 public class ConvertInstanceAnswer extends Answer {

+    private String temporaryConvertUuid;
+
    public ConvertInstanceAnswer() {
        super();
    }
@ -34,6 +36,15 @@ public class ConvertInstanceAnswer extends Answer {
        this.convertedInstance = convertedInstance;
    }

+    public ConvertInstanceAnswer(Command command, String temporaryConvertUuid) {
+        super(command, true, "");
+        this.temporaryConvertUuid = temporaryConvertUuid;
+    }
+
+    public String getTemporaryConvertUuid() {
+        return temporaryConvertUuid;
+    }
+
    public UnmanagedInstanceTO getConvertedInstance() {
        return convertedInstance;
    }
--- a/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java
+++ b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java
@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+import org.apache.cloudstack.vm.UnmanagedInstanceTO;
+
+public class ImportConvertedInstanceAnswer extends Answer {
+
+    public ImportConvertedInstanceAnswer() {
+        super();
+    }
+    private UnmanagedInstanceTO convertedInstance;
+
+    public ImportConvertedInstanceAnswer(Command command, boolean success, String details) {
+        super(command, success, details);
+    }
+
+    public ImportConvertedInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) {
+        super(command, true, "");
+        this.convertedInstance = convertedInstance;
+    }
+
+    public UnmanagedInstanceTO getConvertedInstance() {
+        return convertedInstance;
+    }
+}
--- a/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java
+++ b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java
@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.RemoteInstanceTO;
+
+import java.util.List;
+
+public class ImportConvertedInstanceCommand extends Command {
+
+    private RemoteInstanceTO sourceInstance;
+    private List<String> destinationStoragePools;
+    private DataStoreTO conversionTemporaryLocation;
+    private String temporaryConvertUuid;
+
+    public ImportConvertedInstanceCommand() {
+    }
+
+    public ImportConvertedInstanceCommand(RemoteInstanceTO sourceInstance,
+                                          List<String> destinationStoragePools,
+                                          DataStoreTO conversionTemporaryLocation, String temporaryConvertUuid) {
+        this.sourceInstance = sourceInstance;
+        this.destinationStoragePools = destinationStoragePools;
+        this.conversionTemporaryLocation = conversionTemporaryLocation;
+        this.temporaryConvertUuid = temporaryConvertUuid;
+    }
+
+    public RemoteInstanceTO getSourceInstance() {
+        return sourceInstance;
+    }
+
+    public List<String> getDestinationStoragePools() {
+        return destinationStoragePools;
+    }
+
+    public DataStoreTO getConversionTemporaryLocation() {
+        return conversionTemporaryLocation;
+    }
+
+    public String getTemporaryConvertUuid() {
+        return temporaryConvertUuid;
+    }
+
+    @Override
+    public boolean executeInSequence() {
+        return false;
+    }
+}
--- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetPortForwardingRulesConfigItem.java
+++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetPortForwardingRulesConfigItem.java
@ -41,7 +41,7 @@ public class SetPortForwardingRulesConfigItem extends AbstractConfigItemFacade {

        for (final PortForwardingRuleTO rule : command.getRules()) {
            final ForwardingRule fwdRule = new ForwardingRule(rule.revoked(), rule.getProtocol().toLowerCase(), rule.getSrcIp(), rule.getStringSrcPortRange(), rule.getDstIp(),
-                    rule.getStringDstPortRange());
+                    rule.getStringDstPortRange(), rule.getSourceCidrListAsString());
            rules.add(fwdRule);
        }

--- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/ForwardingRule.java
+++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/ForwardingRule.java
@ -26,18 +26,21 @@ public class ForwardingRule {
    private String sourcePortRange;
    private String destinationIpAddress;
    private String destinationPortRange;
+    private String sourceCidrList;

    public ForwardingRule() {
        // Empty constructor for (de)serialization
    }

-    public ForwardingRule(boolean revoke, String protocol, String sourceIpAddress, String sourcePortRange, String destinationIpAddress, String destinationPortRange) {
+    public ForwardingRule(boolean revoke, String protocol, String sourceIpAddress, String sourcePortRange, String destinationIpAddress, String destinationPortRange,
+                          String sourceCidrList) {
        this.revoke = revoke;
        this.protocol = protocol;
        this.sourceIpAddress = sourceIpAddress;
        this.sourcePortRange = sourcePortRange;
        this.destinationIpAddress = destinationIpAddress;
        this.destinationPortRange = destinationPortRange;
+        this.sourceCidrList = sourceCidrList;
    }

    public boolean isRevoke() {
@ -88,4 +91,8 @@ public class ForwardingRule {
        this.destinationPortRange = destinationPortRange;
    }

+    public String getSourceCidrList() {
+        return sourceCidrList;
+    }
+
 }
--- a/debian/changelog
+++ b/debian/changelog
@ -1,6 +1,12 @@
-cloudstack (4.20.0.0-SNAPSHOT) unstable; urgency=low
+cloudstack (4.21.0.0-SNAPSHOT) unstable; urgency=low

-  * Update the version to 4.20.0.0-SNAPSHOT
+  * Update the version to 4.21.0.0-SNAPSHOT
+
+ -- the Apache CloudStack project <dev@cloudstack.apache.org>  Tue, 19 Nov 2024 08:54:07 -0300
+
+cloudstack (4.21.0.0-SNAPSHOT-SNAPSHOT) unstable; urgency=low
+
+  * Update the version to 4.21.0.0-SNAPSHOT-SNAPSHOT

 -- the Apache CloudStack project <dev@cloudstack.apache.org>  Mon, 29 Jan 2024 10:21:52 +0530

--- a/debian/cloudstack-cli.install
+++ b/debian/cloudstack-cli.install
@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- a/debian/cloudstack-docs.install
+++ b/debian/cloudstack-docs.install
@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
--- a/debian/control
+++ b/debian/control
@ -24,7 +24,7 @@ Description: CloudStack server library

 Package: cloudstack-agent
 Architecture: all
-Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker
+Depends: ${python:Depends}, ${python3:Depends}, openjdk-17-jre-headless | java17-runtime-headless | java17-runtime | zulu-17, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, rsync, lsb-release, ufw, apparmor, cpu-checker, libvirt-daemon-driver-storage-rbd
 Recommends: init-system-helpers
 Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts
 Description: CloudStack agent
--- a/developer/pom.xml
+++ b/developer/pom.xml
@ -25,7 +25,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
    </parent>
    <dependencies>
        <dependency>
--- a/engine/api/pom.xml
+++ b/engine/api/pom.xml
@ -24,7 +24,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloud-engine</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
        <relativePath>../pom.xml</relativePath>
    </parent>
    <dependencies>
--- a/engine/components-api/pom.xml
+++ b/engine/components-api/pom.xml
@ -24,7 +24,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloud-engine</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
        <relativePath>../pom.xml</relativePath>
    </parent>
    <dependencies>
--- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
+++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java
@ -40,6 +40,7 @@ public interface CapacityManager {
    static final String StorageCapacityDisableThresholdCK = "pool.storage.capacity.disablethreshold";
    static final String StorageOverprovisioningFactorCK = "storage.overprovisioning.factor";
    static final String StorageAllocatedCapacityDisableThresholdCK = "pool.storage.allocated.capacity.disablethreshold";
+    static final String StorageAllocatedCapacityDisableThresholdForVolumeResizeCK = "pool.storage.allocated.resize.capacity.disablethreshold";

    static final ConfigKey<Float> CpuOverprovisioningFactor =
            new ConfigKey<>(
@ -118,6 +119,17 @@ public interface CapacityManager {
                    "Percentage (as a value between 0 and 1) of secondary storage capacity threshold.",
                    true);

+    static final ConfigKey<Double> StorageAllocatedCapacityDisableThresholdForVolumeSize =
+            new ConfigKey<>(
+                    ConfigKey.CATEGORY_ALERT,
+                    Double.class,
+                    StorageAllocatedCapacityDisableThresholdForVolumeResizeCK,
+                    "0.90",
+                    "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " +
+                            "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.",
+                    true,
+                    ConfigKey.Scope.Zone);
+
    public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId);

    void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost);
--- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java
+++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java
@ -32,7 +32,7 @@ import java.util.List;
 */
 public interface HighAvailabilityManager extends Manager {

-    public ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false",
+    ConfigKey<Boolean> ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false",
        "Force High-Availability to happen even if the VM says no.", true, Cluster);

    ConfigKey<Integer> HAWorkers = new ConfigKey<>("Advanced", Integer.class, "ha.workers", "5",
@ -112,7 +112,7 @@ public interface HighAvailabilityManager extends Manager {

    void cancelDestroy(VMInstanceVO vm, Long hostId);

-    void scheduleDestroy(VMInstanceVO vm, long hostId);
+    boolean scheduleDestroy(VMInstanceVO vm, long hostId);

    /**
     * Schedule restarts for all vms running on the host.
@ -143,7 +143,7 @@ public interface HighAvailabilityManager extends Manager {
     * @param host host the virtual machine is on.
     * @param type which type of stop is requested.
     */
-    void scheduleStop(VMInstanceVO vm, long hostId, WorkType type);
+    boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type);

    void cancelScheduledMigrations(HostVO host);

--- a/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java
+++ b/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java
@ -37,8 +37,26 @@ import com.cloud.network.PhysicalNetwork;
 import com.cloud.network.addr.PublicIp;
 import com.cloud.offering.NetworkOffering;
 import com.cloud.user.Account;
+import org.apache.cloudstack.framework.config.ConfigKey;

 public interface VpcManager {
+    ConfigKey<Boolean> VpcTierNamePrepend = new ConfigKey<>(Boolean.class,
+            "vpc.tier.name.prepend",
+            ConfigKey.CATEGORY_NETWORK,
+            "false",
+            "Whether to prepend the VPC name to the VPC tier network name",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+    ConfigKey<String> VpcTierNamePrependDelimiter = new ConfigKey<>(String.class,
+            "vpc.tier.name.prepend.delimiter",
+            ConfigKey.CATEGORY_NETWORK,
+            " ",
+            "Delimiter string to use between the VPC and the VPC tier name",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
    /**
     * Returns all the Guest networks that are part of VPC
     *
--- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
+++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
@ -209,6 +209,11 @@ public interface StorageManager extends StorageService {
    ConfigKey<Long> HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000",
            "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true);

+    ConfigKey<Boolean> AllowVolumeReSizeBeyondAllocation = new ConfigKey<Boolean>("Advanced", Boolean.class, "volume.resize.allowed.beyond.allocation", "false",
+            "Determines whether volume size can exceed the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) " +
+                    "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)",
+            true, ConfigKey.Scope.Zone);
+
    /**
     * should we execute in sequence not involving any storages?
     * @return tru if commands should execute in sequence
--- a/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java
+++ b/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java
@ -31,7 +31,7 @@ public interface VMSnapshotManager extends VMSnapshotService, Manager {
    static final ConfigKey<Integer> VMSnapshotExpireInterval = new ConfigKey<Integer>("Advanced", Integer.class, "vmsnapshot.expire.interval", "-1",
            "VM Snapshot expire interval in hours", true, ConfigKey.Scope.Account);

-    public static final int VMSNAPSHOTMAX = 10;
+    ConfigKey<Integer> VMSnapshotMax = new ConfigKey<Integer>("Advanced", Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a single vm", true, ConfigKey.Scope.Global);

    /**
     * Delete all VM snapshots belonging to one VM
--- a/engine/orchestration/pom.xml
+++ b/engine/orchestration/pom.xml
@ -24,7 +24,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloud-engine</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
        <relativePath>../pom.xml</relativePath>
    </parent>
    <dependencies>
--- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java
+++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java
@ -47,6 +47,7 @@ import com.cloud.agent.api.CleanupNetworkRulesCmd;
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.CreateStoragePoolCommand;
 import com.cloud.agent.api.DeleteStoragePoolCommand;
+import com.cloud.agent.api.HandleConfigDriveIsoCommand;
 import com.cloud.agent.api.MaintainCommand;
 import com.cloud.agent.api.MigrateCommand;
 import com.cloud.agent.api.ModifySshKeysCommand;
@ -122,11 +123,10 @@ public abstract class AgentAttache {

    public final static String[] s_commandsAllowedInMaintenanceMode = new String[] { MaintainCommand.class.toString(), MigrateCommand.class.toString(),
        StopCommand.class.toString(), CheckVirtualMachineCommand.class.toString(), PingTestCommand.class.toString(), CheckHealthCommand.class.toString(),
-        ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(),
-        CleanupNetworkRulesCmd.class.toString(), CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(),
-        ModifyTargetsCommand.class.toString(), ModifySshKeysCommand.class.toString(),
-        CreateStoragePoolCommand.class.toString(), DeleteStoragePoolCommand.class.toString(), ModifyStoragePoolCommand.class.toString(),
-        SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(), CleanupPersistentNetworkResourceCommand.class.toString()};
+        ReadyCommand.class.toString(), ShutdownCommand.class.toString(), SetupCommand.class.toString(), CleanupNetworkRulesCmd.class.toString(),
+        CheckNetworkCommand.class.toString(), PvlanSetupCommand.class.toString(), CheckOnHostCommand.class.toString(), ModifyTargetsCommand.class.toString(),
+        ModifySshKeysCommand.class.toString(), CreateStoragePoolCommand.class.toString(), DeleteStoragePoolCommand.class.toString(), ModifyStoragePoolCommand.class.toString(),
+        SetupMSListCommand.class.toString(), RollingMaintenanceCommand.class.toString(), CleanupPersistentNetworkResourceCommand.class.toString(), HandleConfigDriveIsoCommand.class.toString()};
    protected final static String[] s_commandsNotAllowedInConnectingMode = new String[] { StartCommand.class.toString(), CreateCommand.class.toString() };
    static {
        Arrays.sort(s_commandsAllowedInMaintenanceMode);
--- a/engine/pom.xml
+++ b/engine/pom.xml
@ -25,7 +25,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
        <relativePath>../pom.xml</relativePath>
    </parent>
    <build>
--- a/engine/schema/pom.xml
+++ b/engine/schema/pom.xml
@ -24,7 +24,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloud-engine</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
        <relativePath>../pom.xml</relativePath>
    </parent>
    <dependencies>
--- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDao.java
+++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDao.java
@ -29,4 +29,6 @@ public interface FirewallRulesCidrsDao extends GenericDao<FirewallRulesCidrsVO,

    @DB
    List<FirewallRulesCidrsVO> listByFirewallRuleId(long firewallRuleId);
+
+    void updateSourceCidrsForRule(Long firewallRuleId, List<String> sourceCidrList);
 }
--- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java
@ -20,6 +20,7 @@ import java.util.ArrayList;
 import java.util.List;


+import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.stereotype.Component;

 import com.cloud.utils.db.DB;
@ -45,7 +46,7 @@ public class FirewallRulesCidrsDaoImpl extends GenericDaoBase<FirewallRulesCidrs
        sc.setParameters("firewallRuleId", firewallRuleId);

        List<FirewallRulesCidrsVO> results = search(sc, null);
-        List<String> cidrs = new ArrayList<String>(results.size());
+        List<String> cidrs = new ArrayList<>(results.size());
        for (FirewallRulesCidrsVO result : results) {
            cidrs.add(result.getCidr());
        }
@ -63,9 +64,27 @@ public class FirewallRulesCidrsDaoImpl extends GenericDaoBase<FirewallRulesCidrs
        return results;
    }

+    @Override
+    public void updateSourceCidrsForRule(Long firewallRuleId, List<String> sourceCidrList) {
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        txn.start();
+
+        SearchCriteria<FirewallRulesCidrsVO> sc = CidrsSearch.create();
+        sc.setParameters("firewallRuleId", firewallRuleId);
+        remove(sc);
+
+        persist(firewallRuleId, sourceCidrList);
+
+        txn.commit();
+    }
+
    @Override
    @DB
    public void persist(long firewallRuleId, List<String> sourceCidrs) {
+        if (CollectionUtils.isEmpty(sourceCidrs)) {
+            return;
+        }
+
        TransactionLegacy txn = TransactionLegacy.currentTxn();

        txn.start();
--- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java
@ -258,9 +258,6 @@ public class FirewallRulesDaoImpl extends GenericDaoBase<FirewallRuleVO, Long> i
    }

    public void saveSourceCidrs(FirewallRuleVO firewallRule, List<String> cidrList) {
-        if (cidrList == null) {
-            return;
-        }
        _firewallRulesCidrsDao.persist(firewallRule.getId(), cidrList);
    }

--- a/engine/schema/src/main/java/com/cloud/network/rules/PortForwardingRuleVO.java
+++ b/engine/schema/src/main/java/com/cloud/network/rules/PortForwardingRuleVO.java
@ -25,6 +25,7 @@ import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
 import javax.persistence.PrimaryKeyJoinColumn;
 import javax.persistence.Table;
+import javax.persistence.Transient;

 import com.cloud.utils.net.Ip;

@ -47,21 +48,30 @@ public class PortForwardingRuleVO extends FirewallRuleVO implements PortForwardi
    @Column(name = "instance_id")
    private long virtualMachineId;

+    @Transient
+    List<String> sourceCidrs;
+
    public PortForwardingRuleVO() {
    }

    public PortForwardingRuleVO(String xId, long srcIpId, int srcPortStart, int srcPortEnd, Ip dstIp, int dstPortStart, int dstPortEnd, String protocol, long networkId,
-            long accountId, long domainId, long instanceId) {
-        super(xId, srcIpId, srcPortStart, srcPortEnd, protocol, networkId, accountId, domainId, Purpose.PortForwarding, null, null, null, null, null);
+                                long accountId, long domainId, long instanceId, List<String> sourceCidrs) {
+        super(xId, srcIpId, srcPortStart, srcPortEnd, protocol, networkId, accountId, domainId, Purpose.PortForwarding, sourceCidrs, null, null, null, null);
        this.destinationIpAddress = dstIp;
        this.virtualMachineId = instanceId;
        this.destinationPortStart = dstPortStart;
        this.destinationPortEnd = dstPortEnd;
+        this.sourceCidrs = sourceCidrs;
    }

-    public PortForwardingRuleVO(String xId, long srcIpId, int srcPort, Ip dstIp, int dstPort, String protocol, List<String> sourceCidrs, long networkId, long accountId,
-            long domainId, long instanceId) {
-        this(xId, srcIpId, srcPort, srcPort, dstIp, dstPort, dstPort, protocol.toLowerCase(), networkId, accountId, domainId, instanceId);
+    public PortForwardingRuleVO(String xId, long srcIpId, int srcPortStart, int srcPortEnd, Ip dstIp, int dstPortStart, int dstPortEnd, String protocol, long networkId,
+                                long accountId, long domainId, long instanceId) {
+        this(xId, srcIpId, srcPortStart, srcPortEnd, dstIp, dstPortStart, dstPortEnd, protocol.toLowerCase(), networkId, accountId, domainId, instanceId, null);
+    }
+
+    public PortForwardingRuleVO(String xId, long srcIpId, int srcPort, Ip dstIp, int dstPort, String protocol, long networkId, long accountId,
+                                long domainId, long instanceId) {
+        this(xId, srcIpId, srcPort, srcPort, dstIp, dstPort, dstPort, protocol.toLowerCase(), networkId, accountId, domainId, instanceId, null);
    }

    @Override
@ -106,4 +116,13 @@ public class PortForwardingRuleVO extends FirewallRuleVO implements PortForwardi
        return null;
    }

+    public void setSourceCidrList(List<String> sourceCidrs) {
+        this.sourceCidrs = sourceCidrs;
+    }
+
+    @Override
+    public List<String> getSourceCidrList() {
+        return sourceCidrs;
+    }
+
 }
--- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java
@ -31,6 +31,9 @@ import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.SearchCriteria.Op;
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallback;
+import com.cloud.utils.db.TransactionLegacy;

@Component
 public class PortForwardingRulesDaoImpl extends GenericDaoBase<PortForwardingRuleVO, Long> implements PortForwardingRulesDao {
@ -42,7 +45,7 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase<PortForwardingRul
    protected final SearchBuilder<PortForwardingRuleVO> ActiveRulesSearchByAccount;

    @Inject
-    protected FirewallRulesCidrsDao _portForwardingRulesCidrsDao;
+    protected FirewallRulesCidrsDao portForwardingRulesCidrsDao;

    protected PortForwardingRulesDaoImpl() {
        super();
@ -194,4 +197,43 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase<PortForwardingRul
        sc.setParameters("vmIds", vmIds.toArray());
        return batchExpunge(sc, batchSize);
    }
+
+    public PortForwardingRuleVO persist(PortForwardingRuleVO portForwardingRule) {
+        return Transaction.execute((TransactionCallback<PortForwardingRuleVO>) transactionStatus -> {
+            PortForwardingRuleVO dbPfRule = super.persist(portForwardingRule);
+
+            portForwardingRulesCidrsDao.persist(portForwardingRule.getId(), portForwardingRule.getSourceCidrList());
+            List<String> cidrList = portForwardingRulesCidrsDao.getSourceCidrs(portForwardingRule.getId());
+            portForwardingRule.setSourceCidrList(cidrList);
+
+            return dbPfRule;
+        });
+
+    }
+
+    @Override
+    public boolean update(Long id, PortForwardingRuleVO entity) {
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        txn.start();
+
+        boolean success = super.update(id, entity);
+        if (!success) {
+            return false;
+        }
+
+        portForwardingRulesCidrsDao.updateSourceCidrsForRule(entity.getId(), entity.getSourceCidrList());
+        txn.commit();
+
+        return true;
+    }
+
+    @Override
+    public PortForwardingRuleVO findById(Long id) {
+        PortForwardingRuleVO rule = super.findById(id);
+
+        List<String> sourceCidrList = portForwardingRulesCidrsDao.getSourceCidrs(id);
+        rule.setSourceCidrList(sourceCidrList);
+
+        return rule;
+    }
 }
--- a/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java
+++ b/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java
@ -47,6 +47,8 @@ public interface ProjectAccountDao extends GenericDao<ProjectAccountVO, Long> {

    void removeAccountFromProjects(long accountId);

+    void removeUserFromProjects(long userId);
+
    boolean canUserModifyProject(long projectId, long accountId, long userId);

    List<ProjectAccountVO> listUsersOrAccountsByRole(long id);
--- a/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java
@ -192,6 +192,17 @@ public class ProjectAccountDaoImpl extends GenericDaoBase<ProjectAccountVO, Long
        }
    }

+    @Override
+    public void removeUserFromProjects(long userId) {
+        SearchCriteria<ProjectAccountVO> sc = AllFieldsSearch.create();
+        sc.setParameters("userId", userId);
+
+        int removedCount = remove(sc);
+        if (removedCount > 0) {
+            logger.debug(String.format("Removed user [%s] from %s project(s).", userId, removedCount));
+        }
+    }
+
    @Override
    public boolean canUserModifyProject(long projectId, long accountId, long userId) {
        SearchCriteria<ProjectAccountVO> sc = AllFieldsSearch.create();
--- a/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java
+++ b/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java
@ -48,31 +48,34 @@ public class VolumeVO implements Volume {
    @TableGenerator(name = "volume_sq", table = "sequence", pkColumnName = "name", valueColumnName = "value", pkColumnValue = "volume_seq", allocationSize = 1)
    @GeneratedValue(strategy = GenerationType.TABLE)
    @Column(name = "id")
-    long id;
+    private long id;
+
+    @Column(name = "last_id")
+    private long lastId;

    @Column(name = "name")
-    String name;
+    private String name;

    @Column(name = "pool_id")
-    Long poolId;
+    private Long poolId;

    @Column(name = "last_pool_id")
-    Long lastPoolId;
+    private Long lastPoolId;

    @Column(name = "account_id")
-    long accountId;
+    private long accountId;

    @Column(name = "domain_id")
-    long domainId;
+    private long domainId;

    @Column(name = "instance_id")
-    Long instanceId = null;
+    private Long instanceId = null;

    @Column(name = "device_id")
-    Long deviceId = null;
+    private Long deviceId = null;

    @Column(name = "size")
-    Long size;
+    private Long size;

    @Column(name = "min_iops")
    private Long minIops;
@ -81,50 +84,50 @@ public class VolumeVO implements Volume {
    private Long maxIops;

    @Column(name = "folder")
-    String folder;
+    private String folder;

    @Column(name = "path")
-    String path;
+    private String path;

    @Column(name = "pod_id")
-    Long podId;
+    private Long podId;

    @Column(name = "created")
-    Date created;
+    private Date created;

    @Column(name = "attached")
    @Temporal(value = TemporalType.TIMESTAMP)
-    Date attached;
+    private Date attached;

    @Column(name = "data_center_id")
-    long dataCenterId;
+    private long dataCenterId;

    @Column(name = "host_ip")
-    String hostip;
+    private String hostIp;

    @Column(name = "disk_offering_id")
-    long diskOfferingId;
+    private long diskOfferingId;

    @Column(name = "template_id")
-    Long templateId;
+    private Long templateId;

    @Column(name = "first_snapshot_backup_uuid")
-    String firstSnapshotBackupUuid;
+    private String firstSnapshotBackupUuid;

    @Column(name = "volume_type")
    @Enumerated(EnumType.STRING)
-    Type volumeType = Volume.Type.UNKNOWN;
+    private Type volumeType = Volume.Type.UNKNOWN;

    @Column(name = "pool_type")
    @Convert(converter = StoragePoolTypeConverter.class)
-    StoragePoolType poolType;
+    private StoragePoolType poolType;

    @Column(name = GenericDao.REMOVED_COLUMN)
-    Date removed;
+    private Date removed;

    @Column(name = "updated")
    @Temporal(value = TemporalType.TIMESTAMP)
-    Date updated;
+    private Date updated;

    @Column(name = "update_count", updatable = true, nullable = false)
    protected long updatedCount; // This field should be updated everytime the
@ -133,17 +136,17 @@ public class VolumeVO implements Volume {
                                 // dao code.

    @Column(name = "recreatable")
-    boolean recreatable;
+    private boolean recreatable;

    @Column(name = "state")
    @Enumerated(value = EnumType.STRING)
    private State state;

    @Column(name = "chain_info", length = 65535)
-    String chainInfo;
+    private String chainInfo;

    @Column(name = "uuid")
-    String uuid;
+    private String uuid;

    @Column(name = "format")
    private Storage.ImageFormat format;
@ -168,7 +171,7 @@ public class VolumeVO implements Volume {

    @Transient
    // @Column(name="reservation")
-    String reservationId;
+    private String reservationId;

    @Column(name = "hv_ss_reserve")
    private Integer hypervisorSnapshotReserve;
@ -428,11 +431,11 @@ public class VolumeVO implements Volume {
    }

    public String getHostIp() {
-        return hostip;
+        return hostIp;
    }

    public void setHostIp(String hostip) {
-        this.hostip = hostip;
+        this.hostIp = hostip;
    }

    public void setPodId(Long podId) {
@ -690,4 +693,12 @@ public class VolumeVO implements Volume {
    public void setDeleteProtection(boolean deleteProtection) {
        this.deleteProtection = deleteProtection;
    }
+
+    public long getLastId() {
+        return lastId;
+    }
+
+    public void setLastId(long lastId) {
+        this.lastId = lastId;
+    }
 }
--- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeStatsDaoImpl.java
@ -126,14 +126,7 @@ public class VolumeStatsDaoImpl extends GenericDaoBase<VolumeStatsVO, Long> impl

        logger.debug(String.format("Starting to remove all volume_stats rows older than [%s].", limitDate));

-        long totalRemoved = 0;
-        long removed;
-
-        do {
-            removed = expunge(sc, limitPerQuery);
-            totalRemoved += removed;
-            logger.trace(String.format("Removed [%s] volume_stats rows on the last update and a sum of [%s] volume_stats rows older than [%s] until now.", removed, totalRemoved, limitDate));
-        } while (limitPerQuery > 0 && removed >= limitPerQuery);
+        long totalRemoved = batchExpunge(sc, limitPerQuery);

        logger.info(String.format("Removed a total of [%s] volume_stats rows older than [%s].", totalRemoved, limitDate));
    }
--- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java
+++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java
@ -88,6 +88,8 @@ import com.cloud.upgrade.dao.Upgrade41800to41810;
 import com.cloud.upgrade.dao.Upgrade41810to41900;
 import com.cloud.upgrade.dao.Upgrade41900to41910;
 import com.cloud.upgrade.dao.Upgrade41910to42000;
+import com.cloud.upgrade.dao.Upgrade42000to42010;
+import com.cloud.upgrade.dao.Upgrade42010to42100;
 import com.cloud.upgrade.dao.Upgrade420to421;
 import com.cloud.upgrade.dao.Upgrade421to430;
 import com.cloud.upgrade.dao.Upgrade430to440;
@ -230,6 +232,8 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
                .next("4.18.1.0", new Upgrade41810to41900())
                .next("4.19.0.0", new Upgrade41900to41910())
                .next("4.19.1.0", new Upgrade41910to42000())
+                .next("4.20.0.0", new Upgrade42000to42010())
+                .next("4.20.1.0", new Upgrade42010to42100())
                .build();
    }

--- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to41920.java
+++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to41920.java
@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+public class Upgrade41910to41920 implements DbUpgrade {
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[]{"4.19.1.0", "4.19.2.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.19.2.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-41910to41920.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-41910to41920-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[]{script};
+    }
+}
--- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java
+++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java
@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+import com.cloud.upgrade.SystemVmTemplateRegistration;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+    private SystemVmTemplateRegistration systemVmTemplateRegistration;
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[] {"4.20.0.0", "4.20.1.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.20.1.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-42000to42010.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-42000to42010-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    private void initSystemVmTemplateRegistration() {
+        systemVmTemplateRegistration = new SystemVmTemplateRegistration("");
+    }
+
+    @Override
+    public void updateSystemVmTemplates(Connection conn) {
+        logger.debug("Updating System Vm template IDs");
+        initSystemVmTemplateRegistration();
+        try {
+            systemVmTemplateRegistration.updateSystemVmTemplates(conn);
+        } catch (Exception e) {
+            throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
+        }
+    }
+}
--- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java
+++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java
@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import com.cloud.upgrade.SystemVmTemplateRegistration;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+    private SystemVmTemplateRegistration systemVmTemplateRegistration;
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[] {"4.20.1.0", "4.21.0.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.21.0.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-42010to42100.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        final String scriptFile = "META-INF/db/schema-42010to42100-cleanup.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    private void initSystemVmTemplateRegistration() {
+        systemVmTemplateRegistration = new SystemVmTemplateRegistration("");
+    }
+
+    @Override
+    public void updateSystemVmTemplates(Connection conn) {
+        logger.debug("Updating System Vm template IDs");
+        initSystemVmTemplateRegistration();
+        try {
+            systemVmTemplateRegistration.updateSystemVmTemplates(conn);
+        } catch (Exception e) {
+            throw new CloudRuntimeException("Failed to find / register SystemVM template(s)");
+        }
+    }
+}
--- a/engine/schema/src/main/java/com/cloud/user/AccountVO.java
+++ b/engine/schema/src/main/java/com/cloud/user/AccountVO.java
@ -77,6 +77,9 @@ public class AccountVO implements Account {
    @Column(name = "default")
    boolean isDefault;

+    @Column(name = "api_key_access")
+    private Boolean apiKeyAccess;
+
    public AccountVO() {
        uuid = UUID.randomUUID().toString();
    }
@ -229,4 +232,14 @@ public class AccountVO implements Account {
    public String reflectionToString() {
        return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "accountName", "domainId");
    }
+
+    @Override
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = apiKeyAccess;
+    }
+
+    @Override
+    public Boolean getApiKeyAccess() {
+        return apiKeyAccess;
+    }
 }
--- a/engine/schema/src/main/java/com/cloud/user/UserVO.java
+++ b/engine/schema/src/main/java/com/cloud/user/UserVO.java
@ -115,6 +115,9 @@ public class UserVO implements User, Identity, InternalIdentity {
    @Column(name = "key_for_2fa")
    private String keyFor2fa;

+    @Column(name = "api_key_access")
+    private Boolean apiKeyAccess;
+
    public UserVO() {
        this.uuid = UUID.randomUUID().toString();
    }
@ -350,4 +353,13 @@ public class UserVO implements User, Identity, InternalIdentity {
        this.user2faProvider = user2faProvider;
    }

+    @Override
+    public void setApiKeyAccess(Boolean apiKeyAccess) {
+        this.apiKeyAccess = apiKeyAccess;
+    }
+
+    @Override
+    public Boolean getApiKeyAccess() {
+        return apiKeyAccess;
+    }
 }
--- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
@ -41,8 +41,8 @@ import java.util.List;

@Component
 public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements AccountDao {
-    private static final String FIND_USER_ACCOUNT_BY_API_KEY = "SELECT u.id, u.username, u.account_id, u.secret_key, u.state, "
-        + "a.id, a.account_name, a.type, a.role_id, a.domain_id, a.state " + "FROM `cloud`.`user` u, `cloud`.`account` a "
+    private static final String FIND_USER_ACCOUNT_BY_API_KEY = "SELECT u.id, u.username, u.account_id, u.secret_key, u.state, u.api_key_access, "
+        + "a.id, a.account_name, a.type, a.role_id, a.domain_id, a.state, a.api_key_access " + "FROM `cloud`.`user` u, `cloud`.`account` a "
        + "WHERE u.account_id = a.id AND u.api_key = ? and u.removed IS NULL";

    protected final SearchBuilder<AccountVO> AllFieldsSearch;
@ -148,13 +148,25 @@ public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements A
                u.setAccountId(rs.getLong(3));
                u.setSecretKey(DBEncryptionUtil.decrypt(rs.getString(4)));
                u.setState(State.getValueOf(rs.getString(5)));
+                boolean apiKeyAccess = rs.getBoolean(6);
+                if (rs.wasNull()) {
+                    u.setApiKeyAccess(null);
+                } else {
+                    u.setApiKeyAccess(apiKeyAccess);
+                }

-                AccountVO a = new AccountVO(rs.getLong(6));
-                a.setAccountName(rs.getString(7));
-                a.setType(Account.Type.getFromValue(rs.getInt(8)));
-                a.setRoleId(rs.getLong(9));
-                a.setDomainId(rs.getLong(10));
-                a.setState(State.getValueOf(rs.getString(11)));
+                AccountVO a = new AccountVO(rs.getLong(7));
+                a.setAccountName(rs.getString(8));
+                a.setType(Account.Type.getFromValue(rs.getInt(9)));
+                a.setRoleId(rs.getLong(10));
+                a.setDomainId(rs.getLong(11));
+                a.setState(State.getValueOf(rs.getString(12)));
+                apiKeyAccess = rs.getBoolean(13);
+                if (rs.wasNull()) {
+                    a.setApiKeyAccess(null);
+                } else {
+                    a.setApiKeyAccess(apiKeyAccess);
+                }

                userAcctPair = new Pair<User, Account>(u, a);
            }
--- a/engine/schema/src/main/java/com/cloud/vm/ConsoleSessionVO.java
+++ b/engine/schema/src/main/java/com/cloud/vm/ConsoleSessionVO.java
@ -64,6 +64,12 @@ public class ConsoleSessionVO {
    @Column(name = "removed")
    private Date removed;

+    @Column(name = "console_endpoint_creator_address")
+    private String consoleEndpointCreatorAddress;
+
+    @Column(name = "client_address")
+    private String clientAddress;
+
    public long getId() {
        return id;
    }
@ -135,4 +141,20 @@ public class ConsoleSessionVO {
    public void setAcquired(Date acquired) {
        this.acquired = acquired;
    }
+
+    public String getConsoleEndpointCreatorAddress() {
+        return consoleEndpointCreatorAddress;
+    }
+
+    public void setConsoleEndpointCreatorAddress(String consoleEndpointCreatorAddress) {
+        this.consoleEndpointCreatorAddress = consoleEndpointCreatorAddress;
+    }
+
+    public String getClientAddress() {
+        return clientAddress;
+    }
+
+    public void setClientAddress(String clientAddress) {
+        this.clientAddress = clientAddress;
+    }
 }
--- a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java
@ -33,7 +33,7 @@ public interface ConsoleSessionDao extends GenericDao<ConsoleSessionVO, Long> {

    int expungeSessionsOlderThanDate(Date date);

-    void acquireSession(String sessionUuid);
+    void acquireSession(String sessionUuid, String clientAddress);

    int expungeByVmList(List<Long> vmIds, Long batchSize);
 }
--- a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java
@ -62,9 +62,10 @@ public class ConsoleSessionDaoImpl extends GenericDaoBase<ConsoleSessionVO, Long
    }

    @Override
-    public void acquireSession(String sessionUuid) {
+    public void acquireSession(String sessionUuid, String clientAddress) {
        ConsoleSessionVO consoleSessionVO = findByUuid(sessionUuid);
        consoleSessionVO.setAcquired(new Date());
+        consoleSessionVO.setClientAddress(clientAddress);
        update(consoleSessionVO.getId(), consoleSessionVO);
    }

--- a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java
@ -123,14 +123,7 @@ public class VmStatsDaoImpl extends GenericDaoBase<VmStatsVO, Long> implements V

        logger.debug(String.format("Starting to remove all vm_stats rows older than [%s].", limitDate));

-        long totalRemoved = 0;
-        long removed;
-
-        do {
-            removed = expunge(sc, limitPerQuery);
-            totalRemoved += removed;
-            logger.trace(String.format("Removed [%s] vm_stats rows on the last update and a sum of [%s] vm_stats rows older than [%s] until now.", removed, totalRemoved, limitDate));
-        } while (limitPerQuery > 0 && removed >= limitPerQuery);
+        long totalRemoved = batchExpunge(sc, limitPerQuery);

        logger.info(String.format("Removed a total of [%s] vm_stats rows older than [%s].", totalRemoved, limitDate));
    }
--- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml
+++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml
@ -117,6 +117,7 @@
  <bean id="loadBalancerCertMapDaoImpl" class="com.cloud.network.dao.LoadBalancerCertMapDaoImpl" />
  <bean id="managementServerHostDaoImpl" class="com.cloud.cluster.dao.ManagementServerHostDaoImpl" />
  <bean id="managementServerHostPeerDaoImpl" class="com.cloud.cluster.dao.ManagementServerHostPeerDaoImpl" />
+  <bean id="managementServerHostPeerJoinDaoImpl" class="com.cloud.cluster.dao.ManagementServerHostPeerJoinDaoImpl" />
  <bean id="managementServerStatusDaoImpl" class="com.cloud.cluster.dao.ManagementServerStatusDaoImpl" />
  <bean id="networkAccountDaoImpl" class="com.cloud.network.dao.NetworkAccountDaoImpl" />
  <bean id="networkACLDaoImpl" class="com.cloud.network.vpc.dao.NetworkACLDaoImpl" />
--- a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_foreign_key.sql
+++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_foreign_key.sql
@ -0,0 +1,28 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`;
+
+CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY` (
+	IN in_table_name VARCHAR(200)
+    , IN in_key_name VARCHAR(200)
+    , IN in_foreign_key VARCHAR(200)
+    , IN in_references VARCHAR(1000)
+)
+BEGIN
+
+    DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT_WS(' ', 'ALTER TABLE ', in_table_name, ' ADD CONSTRAINT ', in_key_name, ' FOREIGN KEY ', in_foreign_key, ' REFERENCES ', in_references, ' ON DELETE CASCADE'); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END;
--- a/engine/schema/src/main/resources/META-INF/db/schema-41910to41920-cleanup.sql
+++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to41920-cleanup.sql
@ -0,0 +1,23 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.19.1.0 to 4.19.2.0
+--;
+
+-- Delete `project_account` entries for users that were removed
+DELETE FROM `cloud`.`project_account` WHERE `user_id` IN (SELECT `id` FROM `cloud`.`user` WHERE `removed`);
--- a/engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql
+++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql
@ -0,0 +1,23 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.19.1.0 to 4.19.2.0
+--;
+
+-- Add last_id to the volumes table
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'last_id', 'bigint(20) unsigned DEFAULT NULL');
--- a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010-cleanup.sql
+++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010-cleanup.sql
@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.20.0.0 to 4.20.1.0
+--;
--- a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql
+++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql
@ -0,0 +1,34 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.20.0.0 to 4.20.1.0
+--;
+
+-- Add column api_key_access to user and account tables
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the user" AFTER `secret_key`');
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" ');
+
+-- Modify index for mshost_peer
+DELETE FROM `cloud`.`mshost_peer`;
+CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost');
+CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer');
+CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)');
+CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)');
+
+-- Add last_id to the volumes table
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'last_id', 'bigint(20) unsigned DEFAULT NULL');
--- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql
+++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql
@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade cleanup from 4.20.1.0 to 4.21.0.0
+--;
--- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql
+++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql
@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.20.1.0 to 4.21.0.0
+--;
+
+-- Add console_endpoint_creator_address column to cloud.console_session table
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_creator_address', 'VARCHAR(45)');
+
+-- Add client_address column to cloud.console_session table
+CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'client_address', 'VARCHAR(45)');
--- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql
+++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql
@ -31,6 +31,7 @@ select
    `account`.`cleanup_needed` AS `cleanup_needed`,
    `account`.`network_domain` AS `network_domain` ,
    `account`.`default` AS `default`,
+    `account`.`api_key_access` AS `api_key_access`,
    `domain`.`id` AS `domain_id`,
    `domain`.`uuid` AS `domain_uuid`,
    `domain`.`name` AS `domain_name`,
--- a/engine/schema/src/main/resources/META-INF/db/views/cloud.mshost_peer_view.sql
+++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.mshost_peer_view.sql
@ -0,0 +1,44 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+
+DROP VIEW IF EXISTS `cloud`.`mshost_peer_view`;
+
+CREATE VIEW `cloud`.`mshost_peer_view` AS
+SELECT
+    `mshost_peer`.`id` AS `id`,
+    `mshost_peer`.`peer_state` AS `peer_state`,
+    `mshost_peer`.`last_update` AS `last_update`,
+    `owner_mshost`.`id` AS `owner_mshost_id`,
+    `owner_mshost`.`msid` AS `owner_mshost_msid`,
+    `owner_mshost`.`runid` AS `owner_mshost_runid`,
+    `owner_mshost`.`name` AS `owner_mshost_name`,
+    `owner_mshost`.`uuid` AS `owner_mshost_uuid`,
+    `owner_mshost`.`state` AS `owner_mshost_state`,
+    `owner_mshost`.`service_ip` AS `owner_mshost_service_ip`,
+    `owner_mshost`.`service_port` AS `owner_mshost_service_port`,
+    `peer_mshost`.`id` AS `peer_mshost_id`,
+    `peer_mshost`.`msid` AS `peer_mshost_msid`,
+    `peer_mshost`.`runid` AS `peer_mshost_runid`,
+    `peer_mshost`.`name` AS `peer_mshost_name`,
+    `peer_mshost`.`uuid` AS `peer_mshost_uuid`,
+    `peer_mshost`.`state` AS `peer_mshost_state`,
+    `peer_mshost`.`service_ip` AS `peer_mshost_service_ip`,
+    `peer_mshost`.`service_port` AS `peer_mshost_service_port`
+FROM `cloud`.`mshost_peer`
+LEFT JOIN `cloud`.`mshost` AS owner_mshost on `mshost_peer`.`owner_mshost` = `owner_mshost`.`id`
+LEFT JOIN `cloud`.`mshost` AS peer_mshost on `mshost_peer`.`peer_mshost` = `peer_mshost`.`id`;
--- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql
+++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql
@ -39,6 +39,7 @@ select
    user.incorrect_login_attempts,
    user.source,
    user.default,
+    user.api_key_access,
    account.id account_id,
    account.uuid account_uuid,
    account.account_name account_name,
--- a/engine/service/pom.xml
+++ b/engine/service/pom.xml
@ -22,7 +22,7 @@
    <parent>
        <groupId>org.apache.cloudstack</groupId>
        <artifactId>cloud-engine</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
    </parent>
    <artifactId>cloud-engine-service</artifactId>
    <packaging>war</packaging>
--- a/Show More
+++ b/Show More