Merge branch 'main' of https://github.com/apache/cloudstack into nsx-integration

.asf.yaml

@@ -53,11 +53,11 @@ github:
     - rajujith
     - alexandremattioli
     - vishesh92
-    - soreana
     - GaOrtiga
     - acs-robot
     - BryanMLima
     - SadiJr
     - JoaoJandre
+    - winterhazel
 
   protected_branches: ~

.github/workflows/ci.yml

@@ -59,7 +59,8 @@ jobs:
                   smoke/test_domain_network_offerings
                   smoke/test_domain_service_offerings
                   smoke/test_domain_vpc_offerings",
-                "smoke/test_dynamicroles
+                "smoke/test_cluster_drs
+                  smoke/test_dynamicroles
                   smoke/test_enable_account_settings_for_domain
                   smoke/test_enable_role_based_users_in_projects
                   smoke/test_events_resource

api/pom.xml

@@ -36,6 +36,11 @@
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-math3</artifactId>
+            <version>${cs.commons-math3.version}</version>
+        </dependency>
         <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>

api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java

@@ -807,9 +807,11 @@ public class OVFHelper {
                 String eulaLicense = null;
                 for (int i = 0; i < eulaChildNodes.getLength(); i++) {
                     Node eulaItem = eulaChildNodes.item(i);
-                    if (eulaItem.getNodeName().equalsIgnoreCase("Info")) {
+                    if (eulaItem.getNodeName().equalsIgnoreCase("Info")
+                            || eulaItem.getNodeName().endsWith(":Info")) {
                         eulaInfo = eulaItem.getTextContent();
-                    } else if (eulaItem.getNodeName().equalsIgnoreCase("License")) {
+                    } else if (eulaItem.getNodeName().equalsIgnoreCase("License")
+                            || eulaItem.getNodeName().endsWith(":License")) {
                         eulaLicense = eulaItem.getTextContent();
                     }
                 }

api/src/main/java/com/cloud/event/EventTypes.java

@@ -678,6 +678,11 @@ public class EventTypes {
     //Usage related events
     public static final String EVENT_USAGE_REMOVE_USAGE_RECORDS = "USAGE.REMOVE.USAGE.RECORDS";
 
+    // DRS Events
+    public static final String EVENT_CLUSTER_DRS = "CLUSTER.DRS";
+    public static final String EVENT_CLUSTER_DRS_GENERATE = "CLUSTER.DRS.GENERATE";
+
+
     // Netscaler Service Package events
     public static final String EVENT_NETSCALER_SERVICEPACKAGE_ADD = "NETSCALER.SERVICEPACKAGE.ADD";
     public static final String EVENT_NETSCALER_SERVICEPACKAGE_DELETE = "NETSCALER.SERVICEPACKAGE.DELETE";

api/src/main/java/com/cloud/network/VNF.java

@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class VNF {
+
+    public enum AccessMethod {
+        SSH_WITH_PASSWORD("ssh-password"),
+        SSH_WITH_KEY("ssh-key"),
+        HTTP("http"),
+        HTTPS("https"),
+        CONSOLE("console");
+
+        String _method;
+
+        AccessMethod(String method) {
+            _method = method;
+        }
+
+        @Override
+        public String toString() {
+            return _method;
+        }
+
+        public static AccessMethod fromValue(String method) {
+            if (StringUtils.isBlank(method)) {
+                return null;
+            } else {
+                for (AccessMethod accessMethod : AccessMethod.values()) {
+                    if (accessMethod.toString().equalsIgnoreCase(method)) {
+                        return accessMethod;
+                    }
+                }
+            }
+            return null;
+        }
+    }
+
+    public enum AccessDetail {
+        ACCESS_METHODS,
+        USERNAME,
+        PASSWORD,
+        SSH_USER,
+        SSH_PASSWORD,
+        SSH_PORT,
+        WEB_USER,
+        WEB_PASSWORD,
+        HTTP_PATH,
+        HTTP_PORT,
+        HTTPS_PATH,
+        HTTPS_PORT
+    }
+
+    public enum VnfDetail {
+        ICON,
+        VERSION,
+        VENDOR,
+        MAINTAINER
+    }
+
+    public static class VnfNic {
+        long deviceId;
+        String name;
+        boolean required;
+        boolean management;
+        String description;
+
+        public VnfNic(long deviceId, String nicName, boolean required, boolean management, String nicDescription) {
+            this.deviceId = deviceId;
+            this.name = nicName;
+            this.required = required;
+            this.management = management;
+            this.description = nicDescription;
+        }
+
+        public long getDeviceId() {
+            return deviceId;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public boolean isRequired() {
+            return required;
+        }
+
+        public boolean isManagement() {
+            return management;
+        }
+
+        public String getDescription() {
+            return description;
+        }
+    }
+}

api/src/main/java/com/cloud/server/ManagementService.java

@@ -441,6 +441,8 @@ public interface ManagementService {
      */
     Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> listHostsForMigrationOfVM(Long vmId, Long startIndex, Long pageSize, String keyword);
 
+    Ternary<Pair<List<? extends Host>, Integer>, List<? extends Host>, Map<Host, Boolean>> listHostsForMigrationOfVM(VirtualMachine vm, Long startIndex, Long pageSize, String keyword, List<VirtualMachine> vmList);
+
     /**
      * List storage pools for live migrating of a volume. The API returns list of all pools in the cluster to which the
      * volume can be migrated. Current pool is not included in the list. In case of vSphere datastore cluster storage pools,

api/src/main/java/com/cloud/server/ResourceTag.java

@@ -30,6 +30,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit
     public enum ResourceObjectType {
         UserVm(true, true, true),
         Template(true, true, true),
+        VnfTemplate(true, true, true),
         ISO(true, false, true),
         Volume(true, true),
         Snapshot(true, false),

api/src/main/java/com/cloud/storage/Snapshot.java

@@ -26,7 +26,7 @@ import java.util.Date;
 
 public interface Snapshot extends ControlledEntity, Identity, InternalIdentity, StateObject<Snapshot.State> {
     public enum Type {
-        MANUAL, RECURRING, TEMPLATE, HOURLY, DAILY, WEEKLY, MONTHLY, GROUP;
+        MANUAL, RECURRING, TEMPLATE, HOURLY, DAILY, WEEKLY, MONTHLY, GROUP, FROM_GROUP;
         private int max = 8;
 
         public void setMax(int max) {

api/src/main/java/com/cloud/storage/Storage.java

@@ -125,6 +125,7 @@ public class Storage {
         BUILTIN, /* buildin template */
         PERHOST, /* every host has this template, don't need to install it in secondary storage */
         USER, /* User supplied template/iso */
+        VNF,    /* VNFs (virtual network functions) template */
         DATADISK, /* Template corresponding to a datadisk(non root disk) present in an OVA */
         ISODISK /* Template corresponding to a iso (non root disk) present in an OVA */
     }

api/src/main/java/com/cloud/vm/VirtualMachine.java

@@ -315,6 +315,9 @@ public interface VirtualMachine extends RunningOn, ControlledEntity, Partition,
     @Override
     Long getHostId();
 
+
+    void setHostId(Long hostId);
+
     /**
      * @return should HA be enabled for this machine?
      */

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -73,6 +73,8 @@ public interface VmDetailConstants {
 
     String CONFIG_DRIVE_LOCATION = "configDriveLocation";
 
+    String SKIP_DRS = "skipFromDRS";
+
     // VM import with nic, disk and custom params for custom compute offering
     String NIC = "nic";
     String NETWORK = "network";

api/src/main/java/org/apache/cloudstack/affinity/AffinityGroupProcessor.java

@@ -21,8 +21,11 @@ import com.cloud.deploy.DeploymentPlan;
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.exception.AffinityConflictException;
 import com.cloud.utils.component.Adapter;
+import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
 
+import java.util.List;
+
 public interface AffinityGroupProcessor extends Adapter {
 
     /**
@@ -35,7 +38,12 @@ public interface AffinityGroupProcessor extends Adapter {
      *            deployment plan that tells you where it's being deployed to.
      * @param avoid
      *            avoid these data centers, pods, clusters, or hosts.
+     * @param vmList
+     *            list of virtual machines objects according to which the affinity group should be processed.
+     *            This can be used to process a theoretical state in some cases like generating DRS plans
      */
+    void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid, List<VirtualMachine> vmList) throws AffinityConflictException;
+
     void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid) throws AffinityConflictException;
 
     /**

api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java

@@ -21,14 +21,23 @@ import com.cloud.deploy.DeploymentPlan;
 import com.cloud.deploy.DeploymentPlanner.ExcludeList;
 import com.cloud.exception.AffinityConflictException;
 import com.cloud.utils.component.AdapterBase;
+import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachineProfile;
 
+import java.util.Collections;
+import java.util.List;
+
 public class AffinityProcessorBase extends AdapterBase implements AffinityGroupProcessor {
 
     protected String _type;
 
     @Override
     public void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid) throws AffinityConflictException {
+        process(vm, plan, avoid, Collections.emptyList());
+    }
+
+    @Override
+    public void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid, List<VirtualMachine> vmList) throws AffinityConflictException {
 
     }
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -292,6 +292,7 @@ public class ApiConstants {
     public static final String MIN_CPU_NUMBER = "mincpunumber";
     public static final String MIN_MEMORY = "minmemory";
     public static final String MIGRATION_TYPE = "migrationtype";
+    public static final String MIGRATIONS = "migrations";
     public static final String MEMORY = "memory";
     public static final String MODE = "mode";
     public static final String NSX_MODE = "nsxmode";
@@ -436,6 +437,7 @@ public class ApiConstants {
     public static final String TEMPLATE_ID = "templateid";
     public static final String TEMPLATE_IDS = "templateids";
     public static final String TEMPLATE_NAME = "templatename";
+    public static final String TEMPLATE_TYPE = "templatetype";
     public static final String TIMEOUT = "timeout";
     public static final String TIMEZONE = "timezone";
     public static final String TIMEZONEOFFSET = "timezoneoffset";
@@ -662,6 +664,7 @@ public class ApiConstants {
     public static final String SPECIFY_IP_RANGES = "specifyipranges";
     public static final String IS_SOURCE_NAT = "issourcenat";
     public static final String IS_STATIC_NAT = "isstaticnat";
+    public static final String ITERATIONS = "iterations";
     public static final String SORT_BY = "sortby";
     public static final String CHANGE_CIDR = "changecidr";
     public static final String PURPOSE = "purpose";
@@ -1024,7 +1027,6 @@ public class ApiConstants {
     public static final String DEPLOY_AS_IS = "deployasis";
     public static final String DEPLOY_AS_IS_DETAILS = "deployasisdetails";
     public static final String CROSS_ZONES = "crossZones";
-    public static final String TEMPLATETYPE = "templatetype";
     public static final String SOURCETEMPLATEID = "sourcetemplateid";
     public static final String DYNAMIC_SCALING_ENABLED = "dynamicscalingenabled";
     public static final String IOTHREADS_ENABLED = "iothreadsenabled";
@@ -1057,8 +1059,15 @@ public class ApiConstants {
     public static final String SOURCE_NAT_IP = "sourcenatipaddress";
     public static final String SOURCE_NAT_IP_ID = "sourcenatipaddressid";
     public static final String HAS_RULES = "hasrules";
-
     public static final String NSX_DETAIL_KEY = "forNsx";
+    public static final String MANAGEMENT = "management";
+    public static final String IS_VNF = "isvnf";
+    public static final String VNF_NICS = "vnfnics";
+    public static final String VNF_DETAILS = "vnfdetails";
+    public static final String CLEAN_UP_VNF_DETAILS = "cleanupvnfdetails";
+    public static final String CLEAN_UP_VNF_NICS = "cleanupvnfnics";
+    public static final String VNF_CONFIGURE_MANAGEMENT = "vnfconfiguremanagement";
+    public static final String VNF_CIDR_LIST = "vnfcidrlist";
 
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
@@ -1105,7 +1114,7 @@ public class ApiConstants {
     }
 
     public enum VMDetails {
-        all, group, nics, stats, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp;
+        all, group, nics, stats, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp, vnfnics;
     }
 
     public enum DomainDetails {

api/src/main/java/org/apache/cloudstack/api/BaseCmd.java

@@ -43,6 +43,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerService;
 import org.apache.cloudstack.network.lb.InternalLoadBalancerVMService;
 import org.apache.cloudstack.query.QueryService;
 import org.apache.cloudstack.storage.ImageStoreService;
+import org.apache.cloudstack.storage.template.VnfTemplateManager;
 import org.apache.cloudstack.usage.UsageService;
 import org.apache.commons.collections.MapUtils;
 import org.apache.log4j.Logger;
@@ -213,6 +214,8 @@ public abstract class BaseCmd {
     public ResourceIconManager resourceIconManager;
     @Inject
     public Ipv6Service ipv6Service;
+    @Inject
+    public VnfTemplateManager vnfTemplateManager;
 
     public abstract void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
         ResourceAllocationException, NetworkRuleConflictException;

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ExecuteClusterDrsPlanCmd.java

@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.api.command.admin.cluster;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.host.Host;
+import com.cloud.user.Account;
+import com.cloud.utils.UuidUtils;
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.ClusterDrsPlanResponse;
+import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.cluster.ClusterDrsService;
+import org.apache.commons.collections.MapUtils;
+
+import javax.inject.Inject;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+@APICommand(name = "executeClusterDrsPlan",
+            description = "Execute DRS for a cluster. If there is another plan in progress for the same cluster, " +
+                    "this command will fail.",
+            responseObject = ClusterDrsPlanResponse.class, since = "4.19.0", requestHasSensitiveInfo = false,
+            responseHasSensitiveInfo = false)
+public class ExecuteClusterDrsPlanCmd extends BaseAsyncCmd {
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ClusterResponse.class, required = true,
+               description = "ID of cluster")
+    private Long id;
+
+    @Parameter(
+            name = ApiConstants.MIGRATE_TO,
+            type = CommandType.MAP,
+            description = "Virtual Machine to destination host mapping. This parameter specifies the mapping between " +
+                    "a vm and a host to migrate that VM. clusterid is required if this parameter is set." +
+                    "Format of this parameter: migrateto[vm-index].vm=<uuid>&migrateto[vm-index].host=<uuid> " +
+                    "Where, [vm-index] indicates the index to identify the vm that you want to migrate, " +
+                    "vm=<uuid> indicates the UUID of the vm that you want to migrate, and " +
+                    "host=<uuid> indicates the UUID of the host where you want to migrate the vm. " +
+                    "Example: migrateto[0].vm=<71f43cd6-69b0-4d3b-9fbc-67f50963d60b>" +
+                    "&migrateto[0].host=<a382f181-3d2b-4413-b92d-b8931befa7e1>" +
+                    "&migrateto[1].vm=<88de0173-55c0-4c1c-a269-83d0279eeedf>" +
+                    "&migrateto[1].host=<95d6e97c-6766-4d67-9a30-c449c15011d1>" +
+                    "&migrateto[2].vm=<1b331390-59f2-4796-9993-bf11c6e76225>" +
+                    "&migrateto[2].host=<41fdb564-9d3b-447d-88ed-7628f7640cbc>")
+    private Map<String, String> migrateVmTo;
+
+    @Inject
+    private ClusterDrsService clusterDrsService;
+
+    public Map<VirtualMachine, Host> getVmToHostMap() {
+        Map<VirtualMachine, Host> vmToHostMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(migrateVmTo)) {
+            Collection<?> allValues = migrateVmTo.values();
+            Iterator<?> iter = allValues.iterator();
+            while (iter.hasNext()) {
+                HashMap<String, String> vmToHost = (HashMap<String, String>) iter.next();
+
+                String vmId = vmToHost.get("vm");
+                String hostId = vmToHost.get("host");
+
+                VirtualMachine vm;
+                Host host;
+                if (UuidUtils.isUuid(vmId)) {
+                    vm = _entityMgr.findByUuid(VirtualMachine.class, vmId);
+                } else {
+                    vm = _entityMgr.findById(VirtualMachine.class, Long.parseLong(vmId));
+                }
+
+                if (UuidUtils.isUuid(hostId)) {
+                    host = _entityMgr.findByUuid(Host.class, hostId);
+                } else {
+                    host = _entityMgr.findById(Host.class, Long.parseLong(hostId));
+                }
+
+                if (vm == null || host == null) {
+                    throw new InvalidParameterValueException(
+                            String.format("Unable to find the vm/host for vmId=%s, destHostId=%s", vmId, hostId));
+                }
+
+                vmToHostMap.put(vm, host);
+            }
+        }
+        return vmToHostMap;
+    }
+
+    @Override
+    public void execute() {
+        ClusterDrsPlanResponse response = clusterDrsService.executeDrsPlan(this);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+    @Override
+    public Long getApiResourceId() {
+        return getId();
+    }
+
+    public Long getId() {
+        return id;
+    }
+
+    @Override
+    public ApiCommandResourceType getApiResourceType() {
+        return ApiCommandResourceType.Cluster;
+    }
+
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_CLUSTER_DRS;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return String.format("Executing DRS plan for cluster: %d", getId());
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/GenerateClusterDrsPlanCmd.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.api.command.admin.cluster;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.ClusterDrsPlanResponse;
+import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.cluster.ClusterDrsService;
+
+import javax.inject.Inject;
+
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsMaxMigrations;
+
+@APICommand(name = "generateClusterDrsPlan", description = "Generate DRS plan for a cluster",
+            responseObject = ClusterDrsPlanResponse.class, since = "4.19.0", requestHasSensitiveInfo = false,
+            responseHasSensitiveInfo = false)
+public class GenerateClusterDrsPlanCmd extends BaseCmd {
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ClusterResponse.class, required = true,
+               description = "the ID of the Cluster")
+    private Long id;
+
+    @Parameter(name = ApiConstants.MIGRATIONS, type = CommandType.INTEGER,
+               description = "Maximum number of VMs to migrate for a DRS execution. Defaults to value of cluster's drs.vm.migrations setting")
+    private Integer migrations;
+
+    @Inject
+    private ClusterDrsService clusterDrsService;
+
+    public Integer getMaxMigrations() {
+        if (migrations == null) {
+            return ClusterDrsMaxMigrations.valueIn(getId());
+        }
+        return migrations;
+    }
+
+    public Long getId() {
+        return id;
+    }
+
+    @Override
+    public void execute() {
+        final ClusterDrsPlanResponse response = clusterDrsService.generateDrsPlan(this);
+        response.setResponseName(getCommandName());
+        response.setObjectName(getCommandName());
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+    @Override
+    public Long getApiResourceId() {
+        return getId();
+    }
+
+    @Override
+    public ApiCommandResourceType getApiResourceType() {
+        return ApiCommandResourceType.Cluster;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClusterDrsPlanCmd.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.api.command.admin.cluster;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.ClusterDrsPlanResponse;
+import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.cluster.ClusterDrsService;
+
+import javax.inject.Inject;
+
+@APICommand(name = "listClusterDrsPlan", description = "List DRS plans for a clusters",
+            responseObject = ClusterDrsPlanResponse.class, since = "4.19.0", requestHasSensitiveInfo = false)
+public class ListClusterDrsPlanCmd extends BaseListCmd {
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ClusterDrsPlanResponse.class,
+               description = "ID of the drs plan")
+    private Long id;
+
+    @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class,
+               description = "ID of the cluster")
+    private Long clusterId;
+
+    @Inject
+    private ClusterDrsService clusterDrsService;
+
+    public Long getId() {
+        return id;
+    }
+
+    public Long getClusterId() {
+        return clusterId;
+    }
+
+    @Override
+    public void execute() {
+        ListResponse<ClusterDrsPlanResponse> response = clusterDrsService.listDrsPlan(this);
+        response.setResponseName(getCommandName());
+        response.setObjectName(getCommandName());
+        setResponseObject(response);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/template/ListVnfTemplatesCmdByAdmin.java

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.template;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.command.user.template.ListVnfTemplatesCmd;
+import org.apache.cloudstack.api.response.TemplateResponse;
+
+import com.cloud.template.VirtualMachineTemplate;
+
+@APICommand(name = "listVnfTemplates", description = "List all public, private, and privileged VNF templates.",
+        responseObject = TemplateResponse.class, entityType = {VirtualMachineTemplate.class}, responseView = ResponseView.Full,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.19.0")
+public class ListVnfTemplatesCmdByAdmin extends ListVnfTemplatesCmd implements AdminCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/template/RegisterTemplateCmdByAdmin.java

@@ -18,9 +18,11 @@ package org.apache.cloudstack.api.command.admin.template;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
 import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd;
 import org.apache.cloudstack.api.response.TemplateResponse;
 
 @APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud.", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd {}
+public class RegisterTemplateCmdByAdmin extends RegisterTemplateCmd implements AdminCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/template/RegisterVnfTemplateCmdByAdmin.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.template;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd;
+import org.apache.cloudstack.api.response.TemplateResponse;
+
+@APICommand(name = "registerVnfTemplate",
+        description = "Registers an existing VNF template into the CloudStack cloud. ",
+        responseObject = TemplateResponse.class, responseView = ResponseView.Full,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.19.0")
+public class RegisterVnfTemplateCmdByAdmin extends RegisterVnfTemplateCmd implements AdminCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/template/UpdateTemplateCmdByAdmin.java

@@ -24,4 +24,5 @@ import org.apache.cloudstack.api.response.TemplateResponse;
 
 @APICommand(name = "updateTemplate", description = "Updates attributes of a template.", responseObject = TemplateResponse.class, responseView = ResponseView.Full,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class UpdateTemplateCmdByAdmin extends UpdateTemplateCmd implements AdminCmd {}
+public class UpdateTemplateCmdByAdmin extends UpdateTemplateCmd implements AdminCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/template/UpdateVnfTemplateCmdByAdmin.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.template;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd;
+import org.apache.cloudstack.api.response.TemplateResponse;
+
+@APICommand(name = "updateVnfTemplate",
+        description = "Updates a template to VNF template or attributes of a VNF template.",
+        responseObject = TemplateResponse.class, responseView = ResponseView.Full,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.19.0")
+public class UpdateVnfTemplateCmdByAdmin extends UpdateVnfTemplateCmd implements AdminCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/DeployVnfApplianceCmdByAdmin.java

@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.vm;
+
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ResponseObject;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd;
+import org.apache.cloudstack.api.response.UserVmResponse;
+
+@APICommand(name = "deployVnfAppliance",
+        description = "Creates and automatically starts a VNF appliance based on a service offering, disk offering, and template.",
+        responseObject = UserVmResponse.class,
+        responseView = ResponseObject.ResponseView.Full,
+        entityType = {VirtualMachine.class},
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true,
+        since = "4.19.0")
+public class DeployVnfApplianceCmdByAdmin extends DeployVnfApplianceCmd implements AdminCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/user/template/DeleteVnfTemplateCmd.java

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.template;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.response.SuccessResponse;
+
+import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.user.Account;
+
+@APICommand(name = "deleteVnfTemplate",
+        responseObject = SuccessResponse.class,
+        description = "Deletes a VNF template from the system. All virtual machines using the deleted template will not be affected.",
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User},
+        since = "4.19.0")
+public class DeleteVnfTemplateCmd extends DeleteTemplateCmd {
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, getId());
+        if (template != null) {
+            return template.getAccountId();
+        }
+
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java

@@ -96,6 +96,15 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User
             description = "comma separated list of template details requested, value can be a list of [ all, min]")
     private List<String> viewDetails;
 
+    @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING,
+            description = "the type of the template", since = "4.19.0")
+    private String templateType;
+
+    @Parameter(name = ApiConstants.IS_VNF, type = CommandType.BOOLEAN,
+            description = "flag to list VNF templates or not; true if need to list VNF templates, false otherwise.",
+            since = "4.19.0")
+    private Boolean isVnf;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -151,6 +160,10 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User
         return parentTemplateId;
     }
 
+    public String getTemplateType() {
+        return templateType;
+    }
+
     public boolean listInReadyState() {
 
         Account account = CallContext.current().getCallingAccount();
@@ -175,6 +188,10 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User
         return  showIcon != null ? showIcon : false;
     }
 
+    public Boolean getVnf() {
+        return isVnf;
+    }
+
     @Override
     public String getCommandName() {
         return s_name;

api/src/main/java/org/apache/cloudstack/api/command/user/template/ListVnfTemplatesCmd.java

@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.template;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.user.UserCmd;
+import org.apache.cloudstack.api.response.TemplateResponse;
+
+import com.cloud.template.VirtualMachineTemplate;
+
+@APICommand(name = "listVnfTemplates", description = "List all public, private, and privileged VNF templates.",
+        responseObject = TemplateResponse.class, entityType = {VirtualMachineTemplate.class}, responseView = ResponseView.Restricted,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User},
+        since = "4.19.0")
+public class ListVnfTemplatesCmd extends ListTemplatesCmd implements UserCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java

@@ -143,6 +143,7 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
                description = "true if template contains XS/VMWare tools inorder to support dynamic scaling of VM cpu/memory")
     protected Boolean isDynamicallyScalable;
 
+    @Deprecated
     @Parameter(name = ApiConstants.ROUTING, type = CommandType.BOOLEAN, description = "true if the template type is routing i.e., if template is used to deploy router")
     protected Boolean isRoutingType;
 
@@ -168,6 +169,11 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
             description = "(VMware only) true if VM deployments should preserve all the configurations defined for this template", since = "4.15.1")
     protected Boolean deployAsIs;
 
+    @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING,
+            description = "the type of the template. Valid options are: USER/VNF (for all users) and SYSTEM/ROUTING/BUILTIN (for admins only).",
+            since = "4.19.0")
+    private String templateType;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -285,6 +291,10 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
                 Boolean.TRUE.equals(deployAsIs);
     }
 
+    public String getTemplateType() {
+        return templateType;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -315,7 +325,7 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
 
             VirtualMachineTemplate template = _templateService.registerTemplate(this);
             if (template != null) {
-                ListResponse<TemplateResponse> response = new ListResponse<TemplateResponse>();
+                ListResponse<TemplateResponse> response = new ListResponse<>();
                 List<TemplateResponse> templateResponses = _responseGenerator.createTemplateResponses(getResponseView(),
                         template, getZoneIds(), false);
                 response.setResponses(templateResponses);

api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterVnfTemplateCmd.java

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.template;
+
+import java.util.List;
+import java.util.Map;
+
+import com.cloud.network.VNF;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.user.UserCmd;
+import org.apache.cloudstack.api.response.TemplateResponse;
+import org.apache.cloudstack.storage.template.VnfTemplateUtils;
+
+@APICommand(name = "registerVnfTemplate",
+        description = "Registers an existing VNF template into the CloudStack cloud. ",
+        responseObject = TemplateResponse.class, responseView = ResponseView.Restricted,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User},
+        since = "4.19.0")
+public class RegisterVnfTemplateCmd extends RegisterTemplateCmd implements UserCmd {
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.VNF_NICS,
+            type = CommandType.MAP,
+            description = "VNF nics in key/value pairs using format vnfnics[i].keyname=keyvalue. "
+                    + " Example: vnfnics[0].deviceid=0&&vnfnics[0].name=FirstNIC&&vnfnics[0].required=true"
+                    + "&&vnfnics[1].deviceid=1&&vnfnics[1].name=SecondNIC")
+    protected Map vnfNics;
+
+    @Parameter(name = ApiConstants.VNF_DETAILS,
+            type = CommandType.MAP,
+            description = "VNF details in key/value pairs using format vnfdetails[i].keyname=keyvalue. "
+                    + "Example: vnfdetails[0].vendor=xxx&&vnfdetails[0].version=2.0")
+    protected Map vnfDetails;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+    public List<VNF.VnfNic> getVnfNics() {
+        return VnfTemplateUtils.getVnfNicsList(this.vnfNics);
+    }
+
+    public Map<String, String> getVnfDetails() {
+        return convertDetailsToMap(vnfDetails);
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    protected void validateParameters() {
+        super.validateParameters();
+
+        VnfTemplateUtils.validateApiCommandParams(this, null);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java

@@ -16,10 +16,11 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.template;
 
-import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiCommandResourceType;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiErrorCode;
 import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd;
 import org.apache.cloudstack.api.Parameter;
@@ -41,7 +42,8 @@ public class UpdateTemplateCmd extends BaseUpdateTemplateOrIsoCmd implements Use
     //////////////// API parameters /////////////////////
     /////////////////////////////////////////////////////
 
-    @Parameter(name = "templatetype", type = CommandType.STRING, description = "the type of the template")
+    @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING,
+            description = "the type of the template. Valid options are: USER/VNF (for all users) and SYSTEM/ROUTING/BUILTIN (for admins only).")
     private String templateType;
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateVnfTemplateCmd.java

@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.template;
+
+import com.cloud.network.VNF;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.user.UserCmd;
+import org.apache.cloudstack.api.response.TemplateResponse;
+import org.apache.cloudstack.storage.template.VnfTemplateUtils;
+
+import java.util.List;
+import java.util.Map;
+
+@APICommand(name = "updateVnfTemplate", description = "Updates a template to VNF template or attributes of a VNF template.",
+        responseObject = TemplateResponse.class, responseView = ResponseView.Restricted,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User},
+        since = "4.19.0")
+public class UpdateVnfTemplateCmd extends UpdateTemplateCmd implements UserCmd {
+
+    /////////////////////////////////////////////////////
+    //////////////// API parameters /////////////////////
+    /////////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.VNF_NICS,
+            type = CommandType.MAP,
+            description = "VNF nics in key/value pairs using format vnfnics[i].keyname=keyvalue. "
+                    + " Example: vnfnics[0].deviceid=0&&vnfnics[0].name=FirstNIC&&vnfnics[0].required=true"
+                    + "&&vnfnics[1].deviceid=1&&vnfnics[1].name=SecondNIC")
+    protected Map vnfNics;
+
+    @Parameter(name = ApiConstants.VNF_DETAILS,
+            type = CommandType.MAP,
+            description = "VNF details in key/value pairs using format vnfdetails[i].keyname=keyvalue. "
+                    + "Example: vnfdetails[0].vendor=xxx&&vnfdetails[0].version=2.0")
+    protected Map vnfDetails;
+
+    @Parameter(name = ApiConstants.CLEAN_UP_VNF_DETAILS,
+            type = CommandType.BOOLEAN,
+            description = "optional boolean field, which indicates if VNF details will be cleaned up or not")
+    private Boolean cleanupVnfDetails = null;
+
+    @Parameter(name = ApiConstants.CLEAN_UP_VNF_NICS,
+            type = CommandType.BOOLEAN,
+            description = "optional boolean field, which indicates if VNF nics will be cleaned up or not")
+    private Boolean cleanupVnfNics = null;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+
+    public List<VNF.VnfNic> getVnfNics() {
+        return VnfTemplateUtils.getVnfNicsList(this.vnfNics);
+    }
+
+    public Map<String, String> getVnfDetails() {
+        return convertDetailsToMap(vnfDetails);
+    }
+
+    public boolean isCleanupVnfDetails(){
+        return cleanupVnfDetails == null ? false : cleanupVnfDetails.booleanValue();
+    }
+
+    public boolean isCleanupVnfNics(){
+        return cleanupVnfNics == null ? false : cleanupVnfNics.booleanValue();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java

@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.vm;
+
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.utils.net.NetUtils;
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ResponseObject;
+import org.apache.cloudstack.api.command.user.UserCmd;
+import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.storage.template.VnfTemplateUtils;
+import org.apache.commons.collections.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+@APICommand(name = "deployVnfAppliance",
+        description = "Creates and automatically starts a VNF appliance based on a service offering, disk offering, and template.",
+        responseObject = UserVmResponse.class,
+        responseView = ResponseObject.ResponseView.Restricted,
+        entityType = {VirtualMachine.class},
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = true,
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User},
+        since = "4.19.0")
+public class DeployVnfApplianceCmd extends DeployVMCmd implements UserCmd {
+
+    @Parameter(name = ApiConstants.VNF_CONFIGURE_MANAGEMENT, type = CommandType.BOOLEAN, required = false,
+            description = "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. " +
+                    "Network rules are configured if management network is an isolated network or shared network with security groups.")
+    private Boolean vnfConfigureManagement;
+
+    @Parameter(name = ApiConstants.VNF_CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING,
+            description = "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.")
+    private List<String> vnfCidrlist;
+
+    public Boolean getVnfConfigureManagement() {
+        return vnfConfigureManagement != null && vnfConfigureManagement;
+    }
+
+    public List<String> getVnfCidrlist() {
+        if (CollectionUtils.isNotEmpty(vnfCidrlist)) {
+            return vnfCidrlist;
+        } else {
+            List<String> defaultCidrList = new ArrayList<String>();
+            defaultCidrList.add(NetUtils.ALL_IP4_CIDRS);
+            return defaultCidrList;
+        }
+    }
+
+    @Override
+    public void create() throws ResourceAllocationException {
+        VnfTemplateUtils.validateVnfCidrList(this.getVnfCidrlist());
+
+        super.create();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java

@@ -189,6 +189,10 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
         return zoneId;
     }
 
+    @Parameter(name = ApiConstants.IS_VNF, type = CommandType.BOOLEAN,
+            description = "flag to list vms created from VNF templates (as known as VNF appliances) or not; true if need to list VNF appliances, false otherwise.",
+            since = "4.19.0")
+    private Boolean isVnf;
 
     public Long getNetworkId() {
         return networkId;
@@ -266,6 +270,10 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
         return accumulate;
     }
 
+    public Boolean getVnf() {
+        return isVnf;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/response/ChildTemplateResponse.java

@@ -39,7 +39,7 @@ public class ChildTemplateResponse extends BaseResponse {
     @Param(description = "the size of the template")
     private Integer size;
 
-    @SerializedName("templatetype")
+    @SerializedName(ApiConstants.TEMPLATE_TYPE)
     @Param(description = "the type of the template")
     private String templateType;
 

api/src/main/java/org/apache/cloudstack/api/response/ClusterDrsPlanMigrationResponse.java

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.api.response;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.jobs.JobInfo;
+
+public class ClusterDrsPlanMigrationResponse extends BaseResponse {
+    @SerializedName(ApiConstants.VIRTUAL_MACHINE_ID)
+    @Param(description = "VM to migrate")
+    String vmId;
+
+    @SerializedName(ApiConstants.VIRTUAL_MACHINE_NAME)
+    @Param(description = "VM to migrate")
+    String vmName;
+
+    @SerializedName("sourcehostid")
+    @Param(description = "Original host for VM migration")
+    String srcHostId;
+
+    @SerializedName("sourcehostname")
+    @Param(description = "Original host for VM migration")
+    String srcHostName;
+
+    @SerializedName("destinationhostid")
+    @Param(description = "Destination host for VM migration")
+    String destHostId;
+
+    @SerializedName("destinationhostname")
+    @Param(description = "Destination host for VM migration")
+    String destHostName;
+
+    @SerializedName(ApiConstants.JOB_ID)
+    @Param(description = "id of VM migration async job")
+    private Long jobId;
+
+    @SerializedName(ApiConstants.JOB_STATUS)
+    @Param(description = "Job status of VM migration async job")
+    private JobInfo.Status jobStatus;
+
+
+    public ClusterDrsPlanMigrationResponse(String vmId, String vmName, String srcHostId, String srcHostName,
+                                           String destHostId, String destHostName, Long jobId,
+                                           JobInfo.Status jobStatus) {
+        this.vmId = vmId;
+        this.vmName = vmName;
+        this.srcHostId = srcHostId;
+        this.srcHostName = srcHostName;
+        this.destHostId = destHostId;
+        this.destHostName = destHostName;
+        this.jobId = jobId;
+        this.jobStatus = jobStatus;
+        this.setObjectName(ApiConstants.MIGRATIONS);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/response/ClusterDrsPlanResponse.java

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.api.response;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.api.EntityReference;
+import org.apache.cloudstack.cluster.ClusterDrsPlan;
+
+import java.util.Date;
+import java.util.List;
+
+@EntityReference(value = ClusterDrsPlan.class)
+public class ClusterDrsPlanResponse extends BaseResponse {
+    @SerializedName(ApiConstants.MIGRATIONS)
+    @Param(description = "List of migrations")
+    List<ClusterDrsPlanMigrationResponse> migrationPlans;
+
+    @SerializedName(ApiConstants.ID)
+    @Param(description = "unique ID of the drs plan for cluster")
+    private String id;
+
+    @SerializedName(ApiConstants.CLUSTER_ID)
+    @Param(description = "Id of the cluster")
+    private String clusterId;
+
+    @SerializedName("eventid")
+    @Param(description = "Start event Id of the DRS Plan")
+    private String eventId;
+
+    @SerializedName(ApiConstants.TYPE)
+    @Param(description = "Type of DRS Plan (Automated or Manual))")
+    private ClusterDrsPlan.Type type;
+
+    @SerializedName(ApiConstants.STATUS)
+    @Param(description = "Status of DRS Plan")
+    private ClusterDrsPlan.Status status;
+
+    @SerializedName(ApiConstants.CREATED)
+    private Date created;
+
+
+    public ClusterDrsPlanResponse(String clusterId, ClusterDrsPlan plan, String eventId,
+                                  List<ClusterDrsPlanMigrationResponse> migrationPlans) {
+        this.clusterId = clusterId;
+        this.eventId = eventId;
+        if (plan != null) {
+            this.id = plan.getUuid();
+            this.type = plan.getType();
+            this.status = plan.getStatus();
+            this.created = plan.getCreated();
+        }
+        this.migrationPlans = migrationPlans;
+        this.setObjectName("drsPlan");
+    }
+
+    public List<ClusterDrsPlanMigrationResponse> getMigrationPlans() {
+        return migrationPlans;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/response/GuestOSResponse.java

@@ -53,7 +53,7 @@ public class GuestOSResponse extends BaseResponse {
 
     @SerializedName(ApiConstants.IS_USER_DEFINED)
     @Param(description = "is the guest OS user defined")
-    private String isUserDefined;
+    private Boolean isUserDefined;
 
     @SerializedName(ApiConstants.FOR_DISPLAY)
     @Param(description = "is the guest OS visible for the users")
@@ -99,11 +99,11 @@ public class GuestOSResponse extends BaseResponse {
         this.description = description;
     }
 
-    public String getIsUserDefined() {
+    public Boolean getIsUserDefined() {
         return isUserDefined;
     }
 
-    public void setIsUserDefined(String isUserDefined) {
+    public void setIsUserDefined(Boolean isUserDefined) {
         this.isUserDefined = isUserDefined;
     }
 

api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java

@@ -138,6 +138,14 @@ public class NicResponse extends BaseResponse {
     @Param(description = "MTU configured on the NIC", since="4.18.0")
     private Integer mtu;
 
+    @SerializedName(ApiConstants.PUBLIC_IP_ID)
+    @Param(description = "public IP address id associated with this nic via Static nat rule")
+    private String publicIpId;
+
+    @SerializedName(ApiConstants.PUBLIC_IP)
+    @Param(description = "public IP address associated with this nic via Static nat rule")
+    private String publicIp;
+
     public void setVmId(String vmId) {
         this.vmId = vmId;
     }
@@ -400,4 +408,12 @@ public class NicResponse extends BaseResponse {
     public String getVpcId() {
         return vpcId;
     }
+
+    public void setPublicIpId(String publicIpId) {
+        this.publicIpId = publicIpId;
+    }
+
+    public void setPublicIp(String publicIp) {
+        this.publicIp = publicIp;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java

@@ -123,7 +123,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements
     @Param(description = "the physical size of the template")
     private Long physicalSize;
 
-    @SerializedName(ApiConstants.TEMPLATETYPE)
+    @SerializedName(ApiConstants.TEMPLATE_TYPE)
     @Param(description = "the type of the template")
     private String templateType;
 

api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java

@@ -16,9 +16,12 @@
 // under the License.
 package org.apache.cloudstack.api.response;
 
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.Date;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.TreeSet;
@@ -130,6 +133,10 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
     @Param(description = "the name of the template for the virtual machine")
     private String templateName;
 
+    @SerializedName(ApiConstants.TEMPLATE_TYPE)
+    @Param(description = "the type of the template for the virtual machine", since = "4.19.0")
+    private String templateType;
+
     @SerializedName("templatedisplaytext")
     @Param(description = " an alternate display text of the template for the virtual machine")
     private String templateDisplayText;
@@ -360,6 +367,14 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
     @SerializedName(ApiConstants.USER_DATA_DETAILS) @Param(description="list of variables and values for the variables declared in userdata", since = "4.18.0")
     private String userDataDetails;
 
+    @SerializedName(ApiConstants.VNF_NICS)
+    @Param(description = "NICs of the VNF appliance", since = "4.19.0")
+    private List<VnfNicResponse> vnfNics;
+
+    @SerializedName(ApiConstants.VNF_DETAILS)
+    @Param(description = "VNF details", since = "4.19.0")
+    private Map<String, String> vnfDetails;
+
     public UserVmResponse() {
         securityGroupList = new LinkedHashSet<>();
         nics = new TreeSet<>(Comparator.comparingInt(x -> Integer.parseInt(x.getDeviceId())));
@@ -1045,4 +1060,49 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
         this.userDataDetails = userDataDetails;
     }
 
+    public Long getBytesReceived() {
+        return bytesReceived;
+    }
+
+    public Long getBytesSent() {
+        return bytesSent;
+    }
+
+    public String getTemplateType() {
+        return templateType;
+    }
+
+    public void setTemplateType(String templateType) {
+        this.templateType = templateType;
+    }
+
+    public List<VnfNicResponse> getVnfNics() {
+        return vnfNics;
+    }
+
+    public void setVnfNics(List<VnfNicResponse> vnfNics) {
+        this.vnfNics = vnfNics;
+    }
+
+    public Map<String, String> getVnfDetails() {
+        return vnfDetails;
+    }
+
+    public void setVnfDetails(Map<String, String> vnfDetails) {
+        this.vnfDetails = vnfDetails;
+    }
+
+    public void addVnfNic(VnfNicResponse vnfNic) {
+        if (this.vnfNics == null) {
+            this.vnfNics = new ArrayList<>();
+        }
+        this.vnfNics.add(vnfNic);
+    }
+
+    public void addVnfDetail(String key, String value) {
+        if (this.vnfDetails == null) {
+            this.vnfDetails = new LinkedHashMap<>();
+        }
+        this.vnfDetails.put(key,value);
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/VnfNicResponse.java

@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+
+@SuppressWarnings("unused")
+public class VnfNicResponse {
+    @SerializedName(ApiConstants.DEVICE_ID)
+    @Param(description = "Device id of the NIC")
+    private long deviceId;
+
+    @SerializedName(ApiConstants.NAME)
+    @Param(description = "Name of the NIC")
+    private String name;
+
+    @SerializedName(ApiConstants.REQUIRED)
+    @Param(description = "True if the NIC is required. False if optional")
+    private Boolean required;
+
+    @SerializedName(ApiConstants.MANAGEMENT)
+    @Param(description = "True if the NIC is a management interface. False otherwise")
+    private Boolean management;
+
+    @SerializedName(ApiConstants.DESCRIPTION)
+    @Param(description = "Description of the NIC")
+    private String description;
+
+    @SerializedName(ApiConstants.NETWORK_ID)
+    @Param(description = "Network id of the NIC")
+    private String networkId;
+
+    @SerializedName(ApiConstants.NETWORK_NAME)
+    @Param(description = "Network name of the NIC")
+    private String networkName;
+
+    public void setDeviceId(long deviceId) {
+        this.deviceId = deviceId;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public void setRequired(Boolean required) {
+        this.required = required;
+    }
+
+    public void setManagement(Boolean management) {
+        this.management = management;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public void setNetworkId(String networkId) {
+        this.networkId = networkId;
+    }
+
+    public void setNetworkName(String networkName) {
+        this.networkName = networkName;
+    }
+
+    public VnfNicResponse() {
+    }
+
+    public VnfNicResponse(long deviceId, String name, Boolean required, Boolean management, String description) {
+        this.deviceId = deviceId;
+        this.name = name;
+        this.required = required;
+        this.management = management;
+        this.description = description;
+    }
+
+    public long getDeviceId() {
+        return deviceId;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Boolean isRequired() {
+        return required;
+    }
+
+    public Boolean isManagement() {
+        return management;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public String getNetworkId() {
+        return networkId;
+    }
+
+    public String getNetworkName() {
+        return networkName;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/response/VnfTemplateResponse.java

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@SuppressWarnings("unused")
+public class VnfTemplateResponse extends TemplateResponse {
+
+    @SerializedName(ApiConstants.VNF_NICS)
+    @Param(description = "NICs of the VNF template")
+    private List<VnfNicResponse> vnfNics;
+
+    @SerializedName(ApiConstants.VNF_DETAILS)
+    @Param(description = "VNF details")
+    private Map<String, String> vnfDetails;
+
+    public void addVnfNic(VnfNicResponse vnfNic) {
+        if (this.vnfNics == null) {
+            this.vnfNics = new ArrayList<>();
+        }
+        this.vnfNics.add(vnfNic);
+    }
+
+    public void addVnfDetail(String key, String value) {
+        if (this.vnfDetails == null) {
+            this.vnfDetails = new LinkedHashMap<>();
+        }
+        this.vnfDetails.put(key,value);
+    }
+
+    public List<VnfNicResponse> getVnfNics() {
+        return vnfNics;
+    }
+
+    public Map<String, String> getVnfDetails() {
+        return vnfDetails;
+    }
+}

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java

@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import com.cloud.host.Host;
+import com.cloud.offering.ServiceOffering;
+import com.cloud.utils.Pair;
+import com.cloud.utils.Ternary;
+import com.cloud.utils.component.Adapter;
+import com.cloud.vm.VirtualMachine;
+import org.apache.commons.math3.stat.descriptive.moment.Mean;
+import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation;
+
+import javax.naming.ConfigurationException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public interface ClusterDrsAlgorithm extends Adapter {
+
+    /**
+     * Determines whether a DRS operation is needed for a given cluster and host-VM
+     * mapping.
+     *
+     * @param clusterId
+     *         the ID of the cluster to check
+     * @param cpuList
+     *         a list of CPU allocated values for each host in the cluster
+     * @param memoryList
+     *         a list of memory allocated values for each host in the cluster
+     *
+     * @return true if a DRS operation is needed, false otherwise
+     *
+     * @throws ConfigurationException
+     *         if there is an error in the configuration
+     */
+    boolean needsDrs(long clusterId, List<Long> cpuList, List<Long> memoryList) throws ConfigurationException;
+
+
+    /**
+     * Determines the metrics for a given virtual machine and destination host in a DRS cluster.
+     *
+     * @param clusterId
+     *         the ID of the cluster to check
+     * @param vm
+     *         the virtual machine to check
+     * @param serviceOffering
+     *         the service offering for the virtual machine
+     * @param destHost
+     *         the destination host for the virtual machine
+     * @param hostCpuUsedMap
+     *         a map of host IDs to the amount of CPU used on each host
+     * @param hostMemoryUsedMap
+     *         a map of host IDs to the amount of memory used on each host
+     * @param requiresStorageMotion
+     *         whether storage motion is required for the virtual machine
+     *
+     * @return a ternary containing improvement, cost, benefit
+     */
+    Ternary<Double, Double, Double> getMetrics(long clusterId, VirtualMachine vm, ServiceOffering serviceOffering,
+                                               Host destHost, Map<Long, Long> hostCpuUsedMap,
+                                               Map<Long, Long> hostMemoryUsedMap, Boolean requiresStorageMotion);
+
+    /**
+     * Calculates the imbalance of the cluster after a virtual machine migration.
+     *
+     * @param serviceOffering
+     *         the service offering for the virtual machine
+     * @param vm
+     *         the virtual machine being migrated
+     * @param destHost
+     *         the destination host for the virtual machine
+     * @param hostCpuUsedMap
+     *         a map of host IDs to the amount of CPU used on each host
+     * @param hostMemoryUsedMap
+     *         a map of host IDs to the amount of memory used on each host
+     *
+     * @return a pair containing the CPU and memory imbalance of the cluster after the migration
+     */
+    default Pair<Double, Double> getImbalancePostMigration(ServiceOffering serviceOffering, VirtualMachine vm,
+                                                           Host destHost, Map<Long, Long> hostCpuUsedMap,
+                                                           Map<Long, Long> hostMemoryUsedMap) {
+        List<Long> postCpuList = new ArrayList<>();
+        List<Long> postMemoryList = new ArrayList<>();
+        final int vmCpu = serviceOffering.getCpu() * serviceOffering.getSpeed();
+        final long vmRam = serviceOffering.getRamSize() * 1024L * 1024L;
+
+        for (Long hostId : hostCpuUsedMap.keySet()) {
+            long cpu = hostCpuUsedMap.get(hostId);
+            long memory = hostMemoryUsedMap.get(hostId);
+            if (hostId == destHost.getId()) {
+                postCpuList.add(cpu + vmCpu);
+                postMemoryList.add(memory + vmRam);
+            } else if (hostId.equals(vm.getHostId())) {
+                postCpuList.add(cpu - vmCpu);
+                postMemoryList.add(memory - vmRam);
+            } else {
+                postCpuList.add(cpu);
+                postMemoryList.add(memory);
+            }
+        }
+        return new Pair<>(getClusterImbalance(postCpuList), getClusterImbalance(postMemoryList));
+    }
+
+    /**
+     * The cluster imbalance is defined as the percentage deviation from the mean
+     * for a configured metric of the cluster. The standard deviation is used as a
+     * mathematical tool to normalize the metric data for all the resource and the
+     * percentage deviation provides an easy tool to compare a cluster’s current
+     * state against the defined imbalance threshold. Because this is essentially a
+     * percentage, the value is a number between 0.0 and 1.0.
+     * Cluster Imbalance, Ic = σc / mavg , where σc is the standard deviation and
+     * mavg is the mean metric value for the cluster.
+     */
+    default Double getClusterImbalance(List<Long> metricList) {
+        Double clusterMeanMetric = getClusterMeanMetric(metricList);
+        Double clusterStandardDeviation = getClusterStandardDeviation(metricList, clusterMeanMetric);
+        return clusterStandardDeviation / clusterMeanMetric;
+    }
+
+    /**
+     * Mean is the average of a collection or set of metrics. In context of a DRS
+     * cluster, the cluster metrics defined as the average metrics value for some
+     * metric (such as CPU, memory etc.) for every resource such as host.
+     * Cluster Mean Metric, mavg = (∑mi) / N, where mi is a measurable metric for a
+     * resource ‘i’ in a cluster with total N number of resources.
+     */
+    default Double getClusterMeanMetric(List<Long> metricList) {
+        return new Mean().evaluate(metricList.stream().mapToDouble(i -> i).toArray());
+    }
+
+    /**
+     * Standard deviation is defined as the square root of the absolute squared sum
+     * of difference of a metric from its mean for every resource divided by the
+     * total number of resources. In context of the DRS, the cluster standard
+     * deviation is the standard deviation based on a metric of resources in a
+     * cluster such as for the allocation or utilisation CPU/memory metric of hosts
+     * in a cluster.
+     * Cluster Standard Deviation, σc = sqrt((∑∣mi−mavg∣^2) / N), where mavg is the
+     * mean metric value and mi is a measurable metric for some resource ‘i’ in the
+     * cluster with total N number of resources.
+     */
+    default Double getClusterStandardDeviation(List<Long> metricList, Double mean) {
+        if (mean != null) {
+            return new StandardDeviation(false).evaluate(metricList.stream().mapToDouble(i -> i).toArray(), mean);
+        } else {
+            return new StandardDeviation(false).evaluate(metricList.stream().mapToDouble(i -> i).toArray());
+        }
+    }
+}

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsPlan.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+import java.util.Date;
+
+public interface ClusterDrsPlan extends Identity, InternalIdentity {
+
+    long getClusterId();
+
+    Type getType();
+
+    Date getCreated();
+
+    Status getStatus();
+
+    String getUuid();
+
+    long getEventId();
+
+    enum Type {
+        AUTOMATED, MANUAL
+    }
+
+    enum Status {
+        UNDER_REVIEW, READY, IN_PROGRESS, COMPLETED
+    }
+}

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsPlanMigration.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface ClusterDrsPlanMigration extends InternalIdentity {
+}

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.utils.component.Manager;
+import com.cloud.utils.concurrency.Scheduler;
+import org.apache.cloudstack.api.command.admin.cluster.ExecuteClusterDrsPlanCmd;
+import org.apache.cloudstack.api.command.admin.cluster.GenerateClusterDrsPlanCmd;
+import org.apache.cloudstack.api.command.admin.cluster.ListClusterDrsPlanCmd;
+import org.apache.cloudstack.api.response.ClusterDrsPlanResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
+
+public interface ClusterDrsService extends Manager, Configurable, Scheduler {
+
+    ConfigKey<Integer> ClusterDrsPlanExpireInterval = new ConfigKey<>(Integer.class, "drs.plan.expire.interval",
+            ConfigKey.CATEGORY_ADVANCED, "30", "The interval in days after which the DRS events will be cleaned up.",
+            false, ConfigKey.Scope.Global, null, "Expire interval for old DRS plans", null, null, null);
+
+    ConfigKey<Boolean> ClusterDrsEnabled = new ConfigKey<>(Boolean.class, "drs.automatic.enable",
+            ConfigKey.CATEGORY_ADVANCED, "false", "Enable/disable automatic DRS on a cluster.", true,
+            ConfigKey.Scope.Cluster, null, "Enable automatic DRS", null, null, null);
+
+    ConfigKey<Integer> ClusterDrsInterval = new ConfigKey<>(Integer.class, "drs.automatic.interval",
+            ConfigKey.CATEGORY_ADVANCED, "60",
+            "The interval in minutes after which a periodic background thread will schedule DRS for a cluster.", true,
+            ConfigKey.Scope.Cluster, null, "Interval for Automatic DRS ", null, null, null);
+
+    ConfigKey<Integer> ClusterDrsMaxMigrations = new ConfigKey<>(Integer.class, "drs.max.migrations",
+            ConfigKey.CATEGORY_ADVANCED, "50",
+            "Maximum number of live migrations in a DRS execution.",
+            true, ConfigKey.Scope.Cluster, null, "Maximum number of migrations for DRS", null, null, null);
+
+    ConfigKey<String> ClusterDrsAlgorithm = new ConfigKey<>(String.class, "drs.algorithm",
+            ConfigKey.CATEGORY_ADVANCED, "balanced", "The DRS algorithm to be executed on the cluster. Possible values are condensed, balanced.",
+            true, ConfigKey.Scope.Cluster, null, "DRS algorithm", null, null,
+            null, ConfigKey.Kind.Select, "condensed,balanced");
+
+    ConfigKey<Float> ClusterDrsImbalanceThreshold = new ConfigKey<>(Float.class, "drs.imbalance",
+            ConfigKey.CATEGORY_ADVANCED, "0.4",
+            "Value of imbalance allowed in the cluster. 1.0 means no imbalance is allowed and 0.0 means full imbalance is allowed",
+            true, ConfigKey.Scope.Cluster, null, "DRS imbalance", null, null, null);
+
+    ConfigKey<String> ClusterDrsMetric = new ConfigKey<>(String.class, "drs.metric", ConfigKey.CATEGORY_ADVANCED,
+            "memory",
+            "The allocated resource metric used to measure imbalance in a cluster. Possible values are memory, cpu.",
+            true, ConfigKey.Scope.Cluster, null, "DRS metric", null, null, null, ConfigKey.Kind.Select,
+            "memory,cpu");
+
+    /**
+     * Generate a DRS plan for a cluster and save it as per the parameters
+     *
+     * @param cmd
+     *         the GenerateClusterDrsPlanCmd object containing the command parameters
+     *
+     * @return a ClusterDrsPlanResponse object containing information regarding the migrations
+     */
+    ClusterDrsPlanResponse generateDrsPlan(GenerateClusterDrsPlanCmd cmd);
+
+    /**
+     * Executes a DRS plan for a cluster.
+     *
+     * @param cmd
+     *         the ExecuteClusterDrsPlanCmd object containing the ID of the cluster and the map of virtual
+     *         machines to hosts
+     *
+     * @return ClusterDrsPlanResponse response object
+     *
+     * @throws InvalidParameterValueException
+     *         if there is already a plan in READY or IN_PROGRESS state for the
+     *         cluster or if the
+     *         cluster cannot be found by ID
+     */
+    ClusterDrsPlanResponse executeDrsPlan(ExecuteClusterDrsPlanCmd cmd);
+
+    /**
+     * Lists DRS plans for a cluster or a specific plan.
+     *
+     * @param cmd
+     *         the ListClusterDrsPlanCmd object containing the ID of the cluster or the ID of the plan
+     *
+     * @return a ListResponse object containing a list of ClusterDrsPlanResponse objects and the total number of plans
+     *
+     * @throws InvalidParameterValueException
+     *         if both clusterId and planId are specified or if the cluster cannot be
+     *         found by ID
+     */
+    ListResponse<ClusterDrsPlanResponse> listDrsPlan(ListClusterDrsPlanCmd cmd);
+}

api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java

@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import com.cloud.dc.DataCenter;
+import com.cloud.exception.InsufficientAddressCapacityException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.security.SecurityGroup;
+import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.user.Account;
+import com.cloud.uservm.UserVm;
+import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd;
+import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd;
+import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import java.util.List;
+
+public interface VnfTemplateManager {
+
+    ConfigKey<Boolean> VnfTemplateAndApplianceEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
+            "vnf.template.appliance.enabled",
+            "true",
+            "Indicates whether the creation of VNF templates and VNF appliances is enabled or not.",
+            false);
+
+    void persistVnfTemplate(long templateId, RegisterVnfTemplateCmd cmd);
+
+    void updateVnfTemplate(long templateId, UpdateVnfTemplateCmd cmd);
+
+    void validateVnfApplianceNics(VirtualMachineTemplate template, List<Long> networkIds);
+
+    SecurityGroup createSecurityGroupForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, DeployVnfApplianceCmd cmd);
+
+    void createIsolatedNetworkRulesForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner,
+                                                   UserVm vm, DeployVnfApplianceCmd cmd)
+            throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException;
+}

api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java

@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.network.VNF;
+import com.cloud.storage.Storage;
+import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.utils.net.NetUtils;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.command.user.template.DeleteVnfTemplateCmd;
+import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd;
+import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class VnfTemplateUtils {
+    private VnfTemplateUtils() {
+    }
+
+    public static List<VNF.VnfNic> getVnfNicsList(Map vnfNics) {
+        List<VNF.VnfNic> nicsList = new ArrayList<>();
+        if (MapUtils.isNotEmpty(vnfNics)) {
+            Collection nicsCollection = vnfNics.values();
+            Iterator iter = nicsCollection.iterator();
+            while (iter.hasNext()) {
+                HashMap<String, String> nicDetails = (HashMap<String, String>)iter.next();
+                String deviceIdString = nicDetails.get("deviceid");
+                String name = nicDetails.get("name");
+                String requiredString = nicDetails.get("required");
+                String managementString = nicDetails.get("management");
+                String description = nicDetails.get("description");
+                Integer deviceId = null;
+                if (StringUtils.isAnyBlank(name, deviceIdString)) {
+                    throw new InvalidParameterValueException("VNF nic name and deviceid cannot be null");
+                }
+                try {
+                    deviceId = Integer.parseInt(deviceIdString);
+                } catch (NumberFormatException e) {
+                    throw new InvalidParameterValueException("Unable to parse VNF nic deviceId to Integer: " + deviceId);
+                }
+                boolean required = StringUtils.isBlank(requiredString) || Boolean.parseBoolean(requiredString);
+                boolean management = StringUtils.isBlank(managementString) || Boolean.parseBoolean(managementString);
+                nicsList.add(new VNF.VnfNic(deviceId, name, required, management, description));
+            }
+            Collections.sort(nicsList, Comparator.comparing(VNF.VnfNic::getDeviceId));
+        }
+        return nicsList;
+    }
+
+    public static void validateApiCommandParams(Map<String, String> vnfDetails, List<VNF.VnfNic> vnfNics, String templateType) {
+        if (templateType != null && !Storage.TemplateType.VNF.name().equals(templateType)) {
+            throw new InvalidParameterValueException("The template type must be VNF for VNF templates.");
+        }
+
+        if (vnfDetails != null) {
+            for (String vnfDetail : vnfDetails.keySet()) {
+                if (!EnumUtils.isValidEnumIgnoreCase(VNF.VnfDetail.class, vnfDetail) &&
+                        !EnumUtils.isValidEnumIgnoreCase(VNF.AccessDetail.class, vnfDetail)) {
+                    throw new InvalidParameterValueException(String.format("Invalid VNF detail found: %s. Valid values are %s and %s", vnfDetail,
+                            Arrays.stream(VNF.AccessDetail.values()).map(method -> method.toString()).collect(Collectors.joining(", ")),
+                            Arrays.stream(VNF.VnfDetail.values()).map(method -> method.toString()).collect(Collectors.joining(", "))));
+                }
+                if (vnfDetails.get(vnfDetail) == null) {
+                    throw new InvalidParameterValueException("Empty value found for VNF detail: " + vnfDetail);
+                }
+                if (VNF.AccessDetail.ACCESS_METHODS.name().equalsIgnoreCase(vnfDetail)) {
+                    String[] accessMethods = vnfDetails.get(vnfDetail).split(",");
+                    for (String accessMethod : accessMethods) {
+                        if (VNF.AccessMethod.fromValue(accessMethod.trim()) == null) {
+                            throw new InvalidParameterValueException(String.format("Invalid VNF access method found: %s. Valid values are %s", accessMethod,
+                                    Arrays.stream(VNF.AccessMethod.values()).map(method -> method.toString()).sorted().collect(Collectors.joining(", "))));
+                        }
+                    }
+                }
+            }
+        }
+
+        validateVnfNics(vnfNics);
+    }
+
+    public static void validateVnfNics(List<VNF.VnfNic> nicsList) {
+        long deviceId = 0L;
+        boolean required = true;
+        for (VNF.VnfNic nic : nicsList) {
+            if (nic.getDeviceId() != deviceId) {
+                throw new InvalidParameterValueException(String.format("deviceid must be consecutive and start from 0. Nic deviceid should be %s but actual is %s.", deviceId, nic.getDeviceId()));
+            }
+            if (!required && nic.isRequired()) {
+                throw new InvalidParameterValueException(String.format("required cannot be true if a preceding nic is optional. Nic with deviceid %s should be required but actual is optional.", deviceId));
+            }
+            deviceId ++;
+            required = nic.isRequired();
+        }
+    }
+
+    public static void validateApiCommandParams(BaseCmd cmd, VirtualMachineTemplate template) {
+        if (cmd instanceof RegisterVnfTemplateCmd) {
+            RegisterVnfTemplateCmd registerCmd = (RegisterVnfTemplateCmd) cmd;
+            validateApiCommandParams(registerCmd.getVnfDetails(), registerCmd.getVnfNics(), registerCmd.getTemplateType());
+        } else if (cmd instanceof UpdateVnfTemplateCmd) {
+            UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd;
+            if (!Storage.TemplateType.VNF.equals(template.getTemplateType())) {
+                throw new InvalidParameterValueException(String.format("Cannot update as template %s is not a VNF template. The template type is %s.", updateCmd.getId(), template.getTemplateType()));
+            }
+            validateApiCommandParams(updateCmd.getVnfDetails(), updateCmd.getVnfNics(), updateCmd.getTemplateType());
+        } else if (cmd instanceof DeleteVnfTemplateCmd) {
+            if (!Storage.TemplateType.VNF.equals(template.getTemplateType())) {
+                DeleteVnfTemplateCmd deleteCmd = (DeleteVnfTemplateCmd) cmd;
+                throw new InvalidParameterValueException(String.format("Cannot delete as Template %s is not a VNF template. The template type is %s.", deleteCmd.getId(), template.getTemplateType()));
+            }
+        }
+    }
+
+    public static void validateVnfCidrList(List<String> cidrList) {
+        if (CollectionUtils.isEmpty(cidrList)) {
+            return;
+        }
+        for (String cidr : cidrList) {
+            if (!NetUtils.isValidIp4Cidr(cidr)) {
+                throw new InvalidParameterValueException(String.format("Invalid cidr for VNF appliance: %s", cidr));
+            }
+        }
+    }
+}

api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java

@@ -393,6 +393,131 @@ public class OVFHelperTest {
             "</EulaSection>\n" +
             "</VirtualSystem>";
 
+    private String eulaSectionsWithOVFprefix =
+            "<VirtualSystem xmlns=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:cim=\"http://schemas.dmtf.org/wbem/wscim/1/common\" xmlns:ovf=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:rasd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData\" xmlns:vmw=\"http://www.vmware.com/schema/ovf\" xmlns:vssd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n" +
+            "<ovf:EulaSection>\n" +
+            "      <ovf:Info>end-user license agreement</ovf:Info>\n" +
+            "      <ovf:License>END USER LICENSE AGREEMENT\n" +
+            "\n" +
+            "IMPORTANT: PLEASE READ THIS END USER LICENSE AGREEMENT CAREFULLY. IT IS VERY IMPORTANT THAT YOU CHECK THAT YOU ARE PURCHASING CISCO SOFTWARE OR EQUIPMENT FROM AN APPROVED SOURCE AND THAT YOU, OR THE ENTITY YOU REPRESENT (COLLECTIVELY, THE \"CUSTOMER\") HAVE BEEN REGISTERED AS THE END USER FOR THE PURPOSES OF THIS CISCO END USER LICENSE AGREEMENT. IF YOU ARE NOT REGISTERED AS THE END USER YOU HAVE NO LICENSE TO USE THE SOFTWARE AND THE LIMITED WARRANTY IN THIS END USER LICENSE AGREEMENT DOES NOT APPLY. ASSUMING YOU HAVE PURCHASED FROM AN APPROVED SOURCE, DOWNLOADING, INSTALLING OR USING CISCO OR CISCO-SUPPLIED SOFTWARE CONSTITUTES ACCEPTANCE OF THIS AGREEMENT.\n" +
+            "\n" +
+            "CISCO SYSTEMS, INC. OR ITS AFFILIATE LICENSING THE SOFTWARE (\"CISCO\") IS WILLING TO LICENSE THIS SOFTWARE TO YOU ONLY UPON THE CONDITION THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND THAT YOU ACCEPT ALL OF THE TERMS CONTAINED IN THIS END USER LICENSE AGREEMENT PLUS ANY ADDITIONAL LIMITATIONS ON THE LICENSE SET FORTH IN A SUPPLEMENTAL LICENSE AGREEMENT ACCOMPANYING THE PRODUCT, MADE AVAILABLE AT THE TIME OF YOUR ORDER, OR POSTED ON THE CISCO WEBSITE AT www.cisco.com/go/terms (COLLECTIVELY THE \"AGREEMENT\"). TO THE EXTENT OF ANY CONFLICT BETWEEN THE TERMS OF THIS END USER LICENSE AGREEMENT AND ANY SUPPLEMENTAL LICENSE AGREEMENT, THE SUPPLEMENTAL LICENSE AGREEMENT SHALL APPLY. BY DOWNLOADING, INSTALLING, OR USING THE SOFTWARE, YOU ARE REPRESENTING THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND BINDING YOURSELF TO THE AGREEMENT. IF YOU DO " +
+            "NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 DAYS AFTER PURCHASE FROM AN APPROVED SOURCE, AND APPLIES ONLY IF YOU ARE THE ORIGINAL AND REGISTERED END USER PURCHASER. FOR THE PURPOSES OF THIS END USER LICENSE AGREEMENT, AN \"APPROVED SOURCE\" MEANS (A) CISCO; OR (B) A DISTRIBUTOR OR SYSTEMS INTEGRATOR AUTHORIZED BY CISCO TO DISTRIBUTE / SELL CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END " +
+            "USERS; OR (C) A RESELLER AUTHORIZED BY ANY SUCH DISTRIBUTOR OR SYSTEMS INTEGRATOR IN ACCORDANCE WITH THE TERMS OF THE DISTRIBUTOR'S AGREEMENT WITH CISCO TO DISTRIBUTE / SELL THE CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END USERS.\n" +
+            "\n" +
+            "THE FOLLOWING TERMS OF THE AGREEMENT GOVERN CUSTOMER'S USE OF THE SOFTWARE (DEFINED BELOW), EXCEPT TO THE EXTENT: (A) THERE IS A SEPARATE SIGNED CONTRACT BETWEEN CUSTOMER AND CISCO GOVERNING CUSTOMER'S USE OF THE SOFTWARE, OR (B) THE SOFTWARE INCLUDES A SEPARATE \"CLICK-ACCEPT\" LICENSE AGREEMENT OR THIRD PARTY LICENSE AGREEMENT AS PART OF THE INSTALLATION OR DOWNLOAD PROCESS GOVERNING CUSTOMER'S USE OF THE SOFTWARE. TO THE EXTENT OF A CONFLICT BETWEEN THE PROVISIONS OF THE FOREGOING DOCUMENTS, THE ORDER OF PRECEDENCE SHALL BE (1)THE SIGNED CONTRACT, (2) THE CLICK-ACCEPT AGREEMENT OR THIRD PARTY LICENSE AGREEMENT, AND (3) THE AGREEMENT. FOR PURPOSES OF THE AGREEMENT, \"SOFTWARE\" SHALL MEAN COMPUTER PROGRAMS, INCLUDING FIRMWARE AND COMPUTER PROGRAMS EMBEDDED IN CISCO EQUIPMENT, AS PROVIDED TO CUSTOMER BY AN APPROVED SOURCE, AND ANY UPGRADES, UPDATES, BUG FIXES " +
+            "OR MODIFIED VERSIONS THERETO (COLLECTIVELY, \"UPGRADES\"), ANY OF THE SAME WHICH HAS BEEN RELICENSED UNDER THE CISCO SOFTWARE TRANSFER AND RE-LICENSING POLICY (AS MAY BE AMENDED BY CISCO FROM TIME TO TIME) OR BACKUP COPIES OF ANY OF THE FOREGOING.\n" +
+            "\n" +
+            "License. Conditioned upon compliance with the terms and conditions of the Agreement, Cisco grants to Customer a nonexclusive and nontransferable license to use for Customer's internal business purposes the Software and the Documentation for which Customer has paid the required license fees to an Approved Source. \"Documentation\" means written information (whether contained in user or technical manuals, training materials, specifications or otherwise) pertaining to the Software and made available by an Approved Source with the Software in any manner (including on CD-Rom, or on-line). In order to use the Software, Customer may be required to input a registration number or product authorization key and register Customer's copy of the Software online at Cisco's website to obtain the necessary license key or license file.\n" +
+            "\n" +
+            "Customer's license to use the Software shall be limited to, and Customer shall not use the Software in excess of, a single hardware chassis or card or such other limitations as are set forth in the applicable Supplemental License Agreement or in the applicable purchase order which has been accepted by an Approved Source and for which Customer has paid to an Approved Source the required license fee (the \"Purchase Order\").\n" +
+            "\n" +
+            "Unless otherwise expressly provided in the Documentation or any applicable Supplemental License Agreement, Customer shall use the Software solely as embedded in, for execution on, or (where the applicable Documentation permits installation on non-Cisco equipment) for communication with Cisco equipment owned or leased by Customer and used for Customer's internal business purposes. No other licenses are granted by implication, estoppel or otherwise.\n" +
+            "\n" +
+            "For evaluation or beta copies for which Cisco does not charge a license fee, the above requirement to pay license fees does not apply.\n" +
+            "\n" +
+            "General Limitations. This is a license, not a transfer of title, to the Software and Documentation, and Cisco retains ownership of all copies of the Software and Documentation. Customer acknowledges that the Software and Documentation contain trade secrets of Cisco or its suppliers or licensors, including but not limited to the specific internal design and structure of individual programs and associated interface information. Except as otherwise expressly provided under the Agreement, Customer shall only use the Software in connection with the use of Cisco equipment purchased by the Customer from an Approved Source and Customer shall have no right, and Customer specifically agrees not to:\n" +
+            "\n" +
+            "(i) transfer, assign or sublicense its license rights to any other person or entity (other than in compliance with any Cisco relicensing/transfer policy then in force), or use the Software on Cisco equipment not purchased by the Customer from an Approved Source or on secondhand Cisco equipment, and Customer acknowledges that any attempted transfer, assignment, sublicense or use shall be void;\n" +
+            "\n" +
+            "(ii) make error corrections to or otherwise modify or adapt the Software or create derivative works based upon the Software, or permit third parties to do the same;\n" +
+            "\n" +
+            "(iii) reverse engineer or decompile, decrypt, disassemble or otherwise reduce the Software to human-readable form, except to the extent otherwise expressly permitted under applicable law notwithstanding this restriction or except to the extent that Cisco is legally required to permit such specific activity pursuant to any applicable open source license;\n" +
+            "\n" +
+            "(iv) publish any results of benchmark tests run on the Software;\n" +
+            "\n" +
+            "(v) use or permit the Software to be used to perform services for third parties, whether on a service bureau or time sharing basis or otherwise, without the express written authorization of Cisco; or\n" +
+            "\n" +
+            "(vi) disclose, provide, or otherwise make available trade secrets contained within the Software and Documentation in any form to any third party without the prior written consent of Cisco. Customer shall implement reasonable security measures to protect such trade secrets.\n" +
+            "\n" +
+            "To the extent required by applicable law, and at Customer's written request, Cisco shall provide Customer with the interface information needed to achieve interoperability between the Software and another independently created program, on payment of Cisco's applicable fee, if any. Customer shall observe strict obligations of confidentiality with respect to such information and shall use such information in compliance with any applicable terms and conditions upon which Cisco makes such information available.\n" +
+            "\n" +
+            "Software, Upgrades and Additional Copies. NOTWITHSTANDING ANY OTHER PROVISION OF THE AGREEMENT: (1) CUSTOMER HAS NO LICENSE OR RIGHT TO MAKE OR USE ANY ADDITIONAL COPIES OR UPGRADES UNLESS CUSTOMER, AT THE TIME OF MAKING OR ACQUIRING SUCH COPY OR UPGRADE, ALREADY HOLDS A VALID LICENSE TO THE ORIGINAL SOFTWARE AND HAS PAID THE APPLICABLE FEE TO AN APPROVED SOURCE FOR THE UPGRADE OR ADDITIONAL COPIES; (2) USE OF UPGRADES IS LIMITED TO CISCO EQUIPMENT SUPPLIED BY AN APPROVED SOURCE FOR WHICH CUSTOMER IS THE ORIGINAL END USER PURCHASER OR LESSEE OR OTHERWISE HOLDS A VALID LICENSE TO USE THE SOFTWARE WHICH IS BEING UPGRADED; AND (3) THE MAKING AND USE OF ADDITIONAL COPIES IS LIMITED TO NECESSARY BACKUP PURPOSES ONLY.\n" +
+            "\n" +
+            "Proprietary Notices. Customer agrees to maintain and reproduce all copyright, proprietary, and other notices on all copies, in any form, of the Software in the same form and manner that such copyright and other proprietary notices are included on the Software. Except as expressly authorized in the Agreement, Customer shall not make any copies or duplicates of any Software without the prior written permission of Cisco.\n" +
+            "\n" +
+            "Term and Termination. The Agreement and the license granted herein shall remain effective until terminated. Customer may terminate the Agreement and the license at any time by destroying all copies of Software and any Documentation. Customer's rights under the Agreement will terminate immediately without notice from Cisco if Customer fails to comply with any provision of the Agreement. Upon termination, Customer shall destroy all copies of Software and Documentation in its possession or control. All confidentiality obligations of Customer, all restrictions and limitations imposed on the Customer under the section titled \"General Limitations\" and all limitations of liability and disclaimers and restrictions of warranty shall survive termination of this Agreement. In addition, the provisions of the sections titled \"U.S. Government End User Purchasers\" and \"General Terms Applicable to the Limited Warranty Statement " +
+            "and End User License Agreement\" shall survive termination of the Agreement.\n" +
+            "\n" +
+            "Customer Records. Customer grants to Cisco and its independent accountants the right to examine Customer's books, records and accounts during Customer's normal business hours to verify compliance with this Agreement. In the event such audit discloses non-compliance with this Agreement, Customer shall promptly pay to Cisco the appropriate license fees, plus the reasonable cost of conducting the audit.\n" +
+            "\n" +
+            "Export, Re-Export, Transfer and Use Controls. The Software, Documentation and technology or direct products thereof (hereafter referred to as Software and Technology), supplied by Cisco under the Agreement are subject to export controls under the laws and regulations of the United States (\"U.S.\") and any other applicable countries' laws and regulations. Customer shall comply with such laws and regulations governing export, re-export, import, transfer and use of Cisco Software and Technology and will obtain all required U.S. and local authorizations, permits, or licenses. Cisco and Customer each agree to provide the other information, support documents, and assistance as may reasonably be required by the other in connection with securing authorizations or licenses. Information regarding compliance with export, re-export, transfer and use may be located at the following URL: " +
+            "www.cisco.com/web/about/doing_business/legal/global_export_trade/general_export/contract_compliance.html\n" +
+            "\n" +
+            "U.S. Government End User Purchasers. The Software and Documentation qualify as \"commercial items,\" as that term is defined at Federal Acquisition Regulation (\"FAR\") (48 C.F.R.) 2.101, consisting of \"commercial computer software\" and \"commercial computer software documentation\" as such terms are used in FAR 12.212. Consistent with FAR 12.212 and DoD FAR Supp. 227.7202-1 through 227.7202-4, and notwithstanding any other FAR or other contractual clause to the contrary in any agreement into which the Agreement may be incorporated, Customer may provide to Government end user or, if the Agreement is direct, Government end user will acquire, the Software and Documentation with only those rights set forth in the Agreement. Use of either the Software or Documentation or both constitutes agreement by the Government that the Software and Documentation are \"commercial computer software\" and \"commercial computer " +
+            "software documentation,\" and constitutes acceptance of the rights and restrictions herein.\n" +
+            "\n" +
+            "Identified Components; Additional Terms. The Software may contain or be delivered with one or more components, which may include third-party components, identified by Cisco in the Documentation, readme.txt file, third-party click-accept or elsewhere (e.g. on www.cisco.com) (the \"Identified Component(s)\") as being subject to different license agreement terms, disclaimers of warranties, limited warranties or other terms and conditions (collectively, \"Additional Terms\") than those set forth herein. You agree to the applicable Additional Terms for any such Identified Component(s).\n" +
+            "\n" +
+            "Limited Warranty\n" +
+            "\n" +
+            "Subject to the limitations and conditions set forth herein, Cisco warrants that commencing from the date of shipment to Customer (but in case of resale by an Approved Source other than Cisco, commencing not more than ninety (90) days after original shipment by Cisco), and continuing for a period of the longer of (a) ninety (90) days or (b) the warranty period (if any) expressly set forth as applicable specifically to software in the warranty card accompanying the product of which the Software is a part (the \"Product\") (if any): (a) the media on which the Software is furnished will be free of defects in materials and workmanship under normal use; and (b) the Software substantially conforms to the Documentation. The date of shipment of a Product by Cisco is set forth on the packaging material in which the Product is shipped. Except for the foregoing, the Software is provided \"AS IS\". This limited warranty extends only to the " +
+            "Software purchased from an Approved Source by a Customer who is the first registered end user. Customer's sole and exclusive remedy and the entire liability of Cisco and its suppliers under this limited warranty will be (i) replacement of defective media and/or (ii) at Cisco's option, repair, replacement, or refund of the purchase price of the Software, in both cases subject to the condition that any error or defect constituting a breach of this limited warranty is reported to the Approved Source supplying the Software to Customer, within the warranty period. Cisco or the Approved Source supplying the Software to Customer may, at its option, require return of the Software and/or Documentation as a condition to the remedy. In no event does Cisco warrant that the Software is error free or that Customer will be able to operate the Software without problems or interruptions. In addition, due to the continual development of new " +
+            "techniques for intruding upon and attacking networks, Cisco does not warrant that the Software or any equipment, system or network on which the Software is used will be free of vulnerability to intrusion or attack.\n" +
+            "\n" +
+            "Restrictions. This warranty does not apply if the Software, Product or any other equipment upon which the Software is authorized to be used (a) has been altered, except by Cisco or its authorized representative, (b) has not been installed, operated, repaired, or maintained in accordance with instructions supplied by Cisco, (c) has been subjected to abnormal physical or electrical stress, abnormal environmental conditions, misuse, negligence, or accident; or (d) is licensed for beta, evaluation, testing or demonstration purposes. The Software warranty also does not apply to (e) any temporary Software modules; (f) any Software not posted on Cisco's Software Center; (g) any Software that Cisco expressly provides on an \"AS IS\" basis on Cisco's Software Center; (h) any Software for which an Approved Source does not receive a license fee; and (i) Software supplied by any third party which is not an Approved Source.\n" +
+            "\n" +
+            "DISCLAIMER OF WARRANTY\n" +
+            "\n" +
+            "EXCEPT AS SPECIFIED IN THIS WARRANTY SECTION, ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS, AND WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OR CONDITION OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, SATISFACTORY QUALITY, NON-INTERFERENCE, ACCURACY OF INFORMATIONAL CONTENT, OR ARISING FROM A COURSE OF DEALING, LAW, USAGE, OR TRADE PRACTICE, ARE HEREBY EXCLUDED TO THE EXTENT ALLOWED BY APPLICABLE LAW AND ARE EXPRESSLY DISCLAIMED BY CISCO, ITS SUPPLIERS AND LICENSORS. TO THE EXTENT THAT ANY OF THE SAME CANNOT BE EXCLUDED, SUCH IMPLIED CONDITION, REPRESENTATION AND/OR WARRANTY IS LIMITED IN DURATION TO THE EXPRESS WARRANTY PERIOD REFERRED TO IN THE \"LIMITED WARRANTY\" SECTION ABOVE. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATIONS ON HOW LONG AN IMPLIED WARRANTY LASTS, THE ABOVE LIMITATION MAY NOT APPLY IN SUCH STATES. THIS WARRANTY GIVES CUSTOMER SPECIFIC LEGAL RIGHTS, " +
+            "AND CUSTOMER MAY ALSO HAVE OTHER RIGHTS WHICH VARY FROM JURISDICTION TO JURISDICTION. This disclaimer and exclusion shall apply even if the express warranty set forth above fails of its essential purpose.\n" +
+            "\n" +
+            "Disclaimer of Liabilities-Limitation of Liability. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, CANADA, JAPAN OR THE CARIBBEAN, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO ANY APPROVED SOURCE FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT).\n" +
+            "\n" +
+            "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO CISCO FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT). NOTHING IN THE AGREEMENT SHALL LIMIT (I) THE LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS TO CUSTOMER FOR PERSONAL INJURY OR DEATH CAUSED BY THEIR NEGLIGENCE, (II) CISCO'S LIABILITY FOR FRAUDULENT" +
+            " MISREPRESENTATION, OR (III) ANY LIABILITY OF CISCO WHICH CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" +
+            "\n" +
+            "Disclaimer of Liabilities-Waiver of Consequential Damages and Other Losses. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, THE CARIBBEAN OR CANADA, REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO OR ITS SUPPLIERS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ITS SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU.\n" +
+            "\n" +
+            "IF YOU ACQUIRED THE SOFTWARE IN JAPAN, EXCEPT FOR LIABILITY ARISING OUT OF OR IN CONNECTION WITH DEATH OR PERSONAL INJURY, FRAUDULENT MISREPRESENTATION, AND REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ANY APPROVED SOURCE OR THEIR SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n" +
+            "\n" +
+            "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, BE LIABLE FOR ANY LOST REVENUE, LOST PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES, HOWSOEVER ARISING, INCLUDING, WITHOUT LIMITATION, IN CONTRACT, TORT (INCLUDING NEGLIGENCE) OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF, IN EACH CASE, CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT FULLY APPLY TO YOU. THE FOREGOING EXCLUSION SHALL NOT APPLY TO ANY LIABILITY ARISING OUT OF OR IN " +
+            "CONNECTION WITH: (I) DEATH OR PERSONAL INJURY, (II) FRAUDULENT MISREPRESENTATION, OR (III) CISCO'S LIABILITY IN CONNECTION WITH ANY TERMS THAT CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" +
+            "\n" +
+            "Customer acknowledges and agrees that Cisco has set its prices and entered into the Agreement in reliance upon the disclaimers of warranty and the limitations of liability set forth herein, that the same reflect an allocation of risk between the parties (including the risk that a contract remedy may fail of its essential purpose and cause consequential loss), and that the same form an essential basis of the bargain between the parties.\n" +
+            "\n" +
+            "Controlling Law, Jurisdiction. If you acquired, by reference to the address on the purchase order accepted by the Approved Source, the Software in the United States, Latin America, or the Caribbean, the Agreement and warranties (\"Warranties\") are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Canada, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the Province of Ontario, Canada, notwithstanding any conflicts of law provisions; and the courts of the Province of Ontario shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in " +
+            "Europe, the Middle East, Africa, Asia or Oceania (excluding Australia), unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of England, notwithstanding any conflicts of law provisions; and the English courts shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. In addition, if the Agreement is controlled by the laws of England, no person who is not a party to the Agreement shall be entitled to enforce or take the benefit of any of its terms under the Contracts (Rights of Third Parties) Act 1999. If you acquired the Software in Japan, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of Japan, notwithstanding any conflicts of law provisions; and the Tokyo District Court of Japan shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. " +
+            "If you acquired the Software in Australia, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of New South Wales, Australia, notwithstanding any conflicts of law provisions; and the State and federal courts of New South Wales shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in any other country, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties.\n" +
+            "\n" +
+            "For all countries referred to above, the parties specifically disclaim the application of the UN Convention on Contracts for the International Sale of Goods. Notwithstanding the foregoing, either party may seek interim injunctive relief in any court of appropriate jurisdiction with respect to any alleged breach of such party's intellectual property or proprietary rights. If any portion hereof is found to be void or unenforceable, the remaining provisions of the Agreement and Warranties shall remain in full force and effect. Except as expressly provided herein, the Agreement constitutes the entire agreement between the parties with respect to the license of the Software and Documentation and supersedes any conflicting or additional terms contained in any Purchase Order or elsewhere, all of which terms are excluded. The Agreement has been written in the English language, and the parties agree that the English version will govern.\n" +
+            "\n" +
+            "Product warranty terms and other information applicable to Cisco products are available at the following URL: www.cisco.com/go/warranty\n" +
+            "\n" +
+            "Cisco and the Cisco logo are trademarks or registered trademarks of Cisco and/or its affiliates in the U.S. and other countries. To view a list of Cisco trademarks, go to this URL: www.cisco.com/go/trademarks. Third-party trademarks mentioned are the property of their respective owners. The use of the word partner does not imply a partnership relationship between Cisco and any other company. (1110R)\n" +
+            "\n" +
+            "© 1998, 2001, 2003, 2008-2014 Cisco Systems, Inc. All rights reserved.</ovf:License>\n" +
+            "</ovf:EulaSection>\n" +
+            "<ovf:EulaSection>\n" +
+            "      <ovf:Info>supplemental end-user license agreement</ovf:Info>\n" +
+            "      <ovf:License>SUPPLEMENTAL END USER LICENSE AGREEMENT FOR VIRTUAL SOFTWARE PRODUCTS\n" +
+            "\n" +
+            "IMPORTANT: READ CAREFULLY\n" +
+            "\n" +
+            "This Supplemental End User License Agreement (\"SEULA\") contains additional terms and conditions for the Software licensed under the End User License Agreement (\"EULA\") between you and Cisco (collectively, the \"Agreement\"). Capitalized terms used in this SEULA but not defined will have the meanings assigned to them in the EULA. To the extent that there is a conflict between the terms and conditions of the EULA and this SEULA, the terms and conditions of this SEULA will take precedence. In addition to the limitations set forth in the EULA on your access and use of the Software, you agree to comply at all times with the terms and conditions provided in this SEULA.\n" +
+            "\n" +
+            "DOWNLOADING, INSTALLING, OR USING THE SOFTWARE CONSTITUTES ACCEPTANCE OF THE AGREEMENT, AND YOU ARE BINDING YOURSELF AND THE BUSINESS ENTITY THAT YOU REPRESENT (COLLECTIVELY, \"CUSTOMER\") TO THE AGREEMENT. IF YOU DO NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 DAYS AFTER PURCHASE FROM CISCO OR AN AUTHORIZED CISCO RESELLER, AND APPLIES ONLY IF YOU ARE THE ORIGINAL END USER PURCHASER.\n" +
+            "\n" +
+            "Definitions\n" +
+            "\"CPU\" means a central processing unit that encompasses part of a Server.\n" +
+            "\"Failover Pair\"  means a primary Instance and a standby Instance with the same Software configuration where the standby Instance can take over in case of failure of the primary Instance.\n" +
+            "\"Instance\" means a single copy of the Software. Each copy of the Software loaded into memory is an Instance.\n" +
+            "\"Server\" means a single physical computer or device on a network that manages or provides network resources for multiple users.\n" +
+            "\"Service Provider\" means a company that provides information technology services to  external end user customers.\n" +
+            "\"Software\" means Cisco's Adaptive Security Virtual Appliance (\"ASAv\"), Adaptive Security Appliance 1000V Cloud Firewall Software (\"ASA 1000V\"), Nexus 1000V series switch products, Virtual Security Gateway products, or other Cisco virtual software products that Cisco includes under this SEULA.\n" +
+            "\"vCPU\" means a virtual central processing resource assigned to the VM by the underlying virtualization technology.\n" +
+            "\"Virtual Machine\" or \"VM\" means a software container that can run its own operating system and execute applications like a Server.\n" +
+            "\n" +
+            "Additional License Terms and Conditions\n" +
+            "1. Cisco hereby grants Customer the right to install and use the Software on single or multiple Cisco or non-Cisco Servers or on Virtual Machines. In order to use the Software Customer may be required to input a registration number or product activation key and register each Instance online at Cisco's website in order to obtain the necessary entitlements.\n" +
+            "2. Customer shall pay a unit license fee to Cisco or an authorized Cisco reseller, as applicable, for each Instance installed on a Cisco or non-Cisco Server CPU, vCPU or Virtual Machine, as determined by Cisco.\n" +
+            "3. For the ASA 1000V, Customer is licensed the number of Instances equal to the number of CPUs covered by the unit license fee. If Customer deploys a Failover Pair, then the fee for the additional standby Instance is included in the fee for each primary Instance.\n" +
+            "4. If Customer is a Service Provider, Customer may use the Software under the terms of this Agreement for the purpose of delivering hosted information technology services to Customer's end user customers, subject to payment of the required license fee(s).\n" +
+            "5. Customer may also use the Software under the terms of this Agreement to deliver hosted information technology services to Customer affiliates, subject to payment of the required license fee(s).\n" +
+            "6. If the Software is subject to Cisco's Smart Licensing program, Cisco will be able to assess if Customer is using the Software within the limits and entitlements paid for by Customer. If the Smart Licensing program is applicable, Customer will be required to enter into a separate terms of service agreement relating to Smart Licensing.</ovf:License>\n" +
+            "</ovf:EulaSection>\n" +
+            "</VirtualSystem>";
+
     private String productSectionWithCategories =
             "<VirtualSystem ovf:id=\"VMware-vCenter-Server-Appliance\" xmlns=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:cim=\"http://schemas.dmtf.org/wbem/wscim/1/common\" xmlns:ovf=\"http://schemas.dmtf.org/ovf/envelope/1\" xmlns:rasd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData\" xmlns:vmw=\"http://www.vmware.com/schema/ovf\" xmlns:vssd=\"http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_VirtualSystemSettingData\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">\n" +
             "<ProductSection ovf:required=\"false\">\n" +
@@ -731,6 +856,12 @@ public class OVFHelperTest {
         Assert.assertEquals(2, eulas.size());
     }
 
+    @Test
+    public void testGetOVFEulaSectionValidOVFwithOVFprefix() throws IOException, SAXException {
+        List<OVFEulaSectionTO> eulas = ovfHelper.getOVFEulaSectionFromXmlString(eulaSectionsWithOVFprefix);
+        Assert.assertEquals(2, eulas.size());
+    }
+
     @Test
     public void testGetOVFPropertiesWithCategories() throws IOException, SAXException {
         List<OVFPropertyTO> props = ovfHelper.getOVFPropertiesFromXmlString(productSectionWithCategories);

api/src/test/java/com/cloud/network/VNFTest.java

@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+
+@RunWith(MockitoJUnitRunner.class)
+public class VNFTest {
+
+    static long deviceId = 0L;
+    static String deviceName = "eth0";
+    static boolean required = true;
+    static boolean management = false;
+    static String description = "description of vnf nic";
+
+    @Before
+    public void setUp() {
+    }
+
+    @Test
+    public void testAccessMethods() {
+        Assert.assertEquals(VNF.AccessMethod.CONSOLE, VNF.AccessMethod.fromValue("console"));
+        Assert.assertEquals(VNF.AccessMethod.HTTP, VNF.AccessMethod.fromValue("http"));
+        Assert.assertEquals(VNF.AccessMethod.HTTPS, VNF.AccessMethod.fromValue("https"));
+        Assert.assertEquals(VNF.AccessMethod.SSH_WITH_KEY, VNF.AccessMethod.fromValue("ssh-key"));
+        Assert.assertEquals(VNF.AccessMethod.SSH_WITH_PASSWORD, VNF.AccessMethod.fromValue("ssh-password"));
+    }
+
+    @Test
+    public void testVnfNic() {
+        VNF.VnfNic vnfNic = new VNF.VnfNic(deviceId, deviceName, required, management, description);
+
+        Assert.assertEquals(deviceId, vnfNic.getDeviceId());
+        Assert.assertEquals(deviceName, vnfNic.getName());
+        Assert.assertEquals(required, vnfNic.isRequired());
+        Assert.assertEquals(management, vnfNic.isManagement());
+        Assert.assertEquals(description, vnfNic.getDescription());
+    }
+}

api/src/test/java/org/apache/cloudstack/api/response/VnfNicResponseTest.java

@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public final class VnfNicResponseTest {
+
+    static long deviceId = 0L;
+    static String deviceName = "eth0";
+    static boolean required = true;
+    static boolean management = false;
+    static String description = "description of vnf nic";
+
+    static String networkUuid = "networkuuid";
+    static String networkName = "networkname";
+
+    @Test
+    public void testNewVnfNicResponse() {
+        final VnfNicResponse response = new VnfNicResponse(deviceId, deviceName, required, management, description);
+        Assert.assertEquals(deviceId, response.getDeviceId());
+        Assert.assertEquals(deviceName, response.getName());
+        Assert.assertEquals(required, response.isRequired());
+        Assert.assertEquals(management, response.isManagement());
+        Assert.assertEquals(description, response.getDescription());
+    }
+
+    @Test
+    public void testSetVnfNicResponse() {
+        final VnfNicResponse response = new VnfNicResponse();
+        response.setNetworkId(networkUuid);
+        response.setNetworkName(networkName);
+        Assert.assertEquals(networkUuid, response.getNetworkId());
+        Assert.assertEquals(networkName, response.getNetworkName());
+    }
+}

api/src/test/java/org/apache/cloudstack/api/response/VnfTemplateResponseTest.java

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.response;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public final class VnfTemplateResponseTest {
+
+    @Test
+    public void testAddVnfNicToResponse() {
+        final VnfTemplateResponse response = new VnfTemplateResponse();
+
+        response.addVnfNic(new VnfNicResponse());
+        response.addVnfNic(new VnfNicResponse());
+
+        Assert.assertEquals(2, response.getVnfNics().size());
+    }
+
+    @Test
+    public void testAddVnfDetailToResponse() {
+        final VnfTemplateResponse response = new VnfTemplateResponse();
+
+        response.addVnfDetail("key1", "value1");
+        response.addVnfDetail("key2", "value2");
+        response.addVnfDetail("key3", "value3");
+
+        Assert.assertEquals(3, response.getVnfDetails().size());
+    }
+}

api/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateUtilsTest.java

@@ -0,0 +1,261 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.storage.template;
+
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.network.VNF.VnfNic;
+import com.cloud.storage.Storage;
+import com.cloud.template.VirtualMachineTemplate;
+import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd;
+import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@RunWith(MockitoJUnitRunner.class)
+public class VnfTemplateUtilsTest {
+
+    @Test
+    public void testGetVnfNicsListAllGood() {
+        final Map<String, Object> vnfNics = new HashMap<>();
+        vnfNics.put("0", new HashMap<>(Map.ofEntries(
+                Map.entry("deviceid", "1"),
+                Map.entry("name", "eth1"),
+                Map.entry("required", "true"),
+                Map.entry("description", "The second NIC of VNF appliance")
+        )));
+        vnfNics.put("1", new HashMap<>(Map.ofEntries(
+                Map.entry("deviceid", "2"),
+                Map.entry("name", "eth2"),
+                Map.entry("required", "false"),
+                Map.entry("description", "The third NIC of VNF appliance")
+        )));
+        vnfNics.put("2", new HashMap<>(Map.ofEntries(
+                Map.entry("deviceid", "0"),
+                Map.entry("name", "eth0"),
+                Map.entry("description", "The first NIC of VNF appliance")
+        )));
+
+        Map<String, Object> vnfNicsMock = Mockito.mock(Map.class);
+        Mockito.when(vnfNicsMock.values()).thenReturn(vnfNics.values());
+
+        List<VnfNic> nicsList = VnfTemplateUtils.getVnfNicsList(vnfNicsMock);
+        Mockito.verify(vnfNicsMock).values();
+
+        Assert.assertEquals(3, nicsList.size());
+        Assert.assertEquals(0, nicsList.get(0).getDeviceId());
+        Assert.assertEquals("eth0", nicsList.get(0).getName());
+        Assert.assertTrue(nicsList.get(0).isRequired());
+        Assert.assertEquals(1, nicsList.get(1).getDeviceId());
+        Assert.assertEquals("eth1", nicsList.get(1).getName());
+        Assert.assertTrue(nicsList.get(1).isRequired());
+        Assert.assertEquals(2, nicsList.get(2).getDeviceId());
+        Assert.assertEquals("eth2", nicsList.get(2).getName());
+        Assert.assertFalse(nicsList.get(2).isRequired());
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testGetVnfNicsListWithEmptyName() {
+        final Map<String, Object> vnfNics = new HashMap<>();
+        vnfNics.put("0", new HashMap<>(Map.ofEntries(
+                Map.entry("deviceid", "1"),
+                Map.entry("required", "true"),
+                Map.entry("description", "The second NIC of VNF appliance")
+        )));
+
+        Map<String, Object> vnfNicsMock = Mockito.mock(Map.class);
+        Mockito.when(vnfNicsMock.values()).thenReturn(vnfNics.values());
+
+        List<VnfNic> nicsList = VnfTemplateUtils.getVnfNicsList(vnfNicsMock);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testGetVnfNicsListWithEmptyDeviceId() {
+        final Map<String, Object> vnfNics = new HashMap<>();
+        vnfNics.put("0", new HashMap<>(Map.ofEntries(
+                Map.entry("name", "eth1"),
+                Map.entry("required", "true"),
+                Map.entry("description", "The second NIC of VNF appliance")
+        )));
+
+        Map<String, Object> vnfNicsMock = Mockito.mock(Map.class);
+        Mockito.when(vnfNicsMock.values()).thenReturn(vnfNics.values());
+
+        List<VnfNic> nicsList = VnfTemplateUtils.getVnfNicsList(vnfNicsMock);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testGetVnfNicsListWithInvalidDeviceId() {
+        final Map<String, Object> vnfNics = new HashMap<>();
+        vnfNics.put("0", new HashMap<>(Map.ofEntries(
+                Map.entry("deviceid", "invalid"),
+                Map.entry("name", "eth1"),
+                Map.entry("required", "true"),
+                Map.entry("description", "The second NIC of VNF appliance")
+        )));
+
+        Map<String, Object> vnfNicsMock = Mockito.mock(Map.class);
+        Mockito.when(vnfNicsMock.values()).thenReturn(vnfNics.values());
+
+        List<VnfNic> nicsList = VnfTemplateUtils.getVnfNicsList(vnfNicsMock);
+    }
+
+    @Test
+    public void testValidateVnfNicsAllGood() {
+        VnfNic nic1 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic1.getDeviceId()).thenReturn(0L);
+        Mockito.when(nic1.isRequired()).thenReturn(true);
+
+        VnfNic nic2 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic2.getDeviceId()).thenReturn(1L);
+        Mockito.when(nic2.isRequired()).thenReturn(true);
+
+        VnfNic nic3 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic3.getDeviceId()).thenReturn(2L);
+        Mockito.when(nic3.isRequired()).thenReturn(false);
+
+        List<VnfNic> nicsList = Arrays.asList(nic1, nic2, nic3);
+
+        VnfTemplateUtils.validateVnfNics(nicsList);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateVnfNicsStartWithNonzero() {
+        VnfNic nic1 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic1.getDeviceId()).thenReturn(1L);
+
+        VnfNic nic2 = Mockito.mock(VnfNic.class);
+
+        VnfNic nic3 = Mockito.mock(VnfNic.class);
+
+        List<VnfNic> nicsList = Arrays.asList(nic1, nic2, nic3);
+
+        VnfTemplateUtils.validateVnfNics(nicsList);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateVnfNicsWithNonConstantDeviceIds() {
+        VnfNic nic1 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic1.getDeviceId()).thenReturn(0L);
+        Mockito.when(nic1.isRequired()).thenReturn(true);
+
+        VnfNic nic2 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic2.getDeviceId()).thenReturn(2L);
+
+        VnfNic nic3 = Mockito.mock(VnfNic.class);
+
+        List<VnfNic> nicsList = Arrays.asList(nic1, nic2, nic3);
+
+        VnfTemplateUtils.validateVnfNics(nicsList);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateVnfNicsWithInvalidRequired() {
+        VnfNic nic1 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic1.getDeviceId()).thenReturn(0L);
+        Mockito.when(nic1.isRequired()).thenReturn(true);
+
+        VnfNic nic2 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic2.getDeviceId()).thenReturn(1L);
+        Mockito.when(nic2.isRequired()).thenReturn(false);
+
+        VnfNic nic3 = Mockito.mock(VnfNic.class);
+        Mockito.when(nic3.getDeviceId()).thenReturn(2L);
+        Mockito.when(nic3.isRequired()).thenReturn(true);
+
+        List<VnfNic> nicsList = Arrays.asList(nic1, nic2, nic3);
+
+        VnfTemplateUtils.validateVnfNics(nicsList);
+    }
+
+    @Test
+    public void testValidateApiCommandParamsAllGood() {
+        VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class);
+        RegisterVnfTemplateCmd cmd = Mockito.mock(RegisterVnfTemplateCmd.class);
+        Map<String, String> vnfDetails = Mockito.spy(new HashMap<>());
+        vnfDetails.put("username", "admin");
+        vnfDetails.put("password", "password");
+        vnfDetails.put("version", "4.19.0");
+        vnfDetails.put("vendor", "cloudstack");
+        Mockito.when(cmd.getVnfDetails()).thenReturn(vnfDetails);
+
+        VnfTemplateUtils.validateApiCommandParams(cmd, template);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateApiCommandParamsInvalidAccessMethods() {
+        VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class);
+        Mockito.when(template.getTemplateType()).thenReturn(Storage.TemplateType.VNF);
+        UpdateVnfTemplateCmd cmd = Mockito.mock(UpdateVnfTemplateCmd.class);
+        Map<String, String> vnfDetails = Mockito.spy(new HashMap<>());
+        vnfDetails.put("access_methods", "invalid");
+        Mockito.when(cmd.getVnfDetails()).thenReturn(vnfDetails);
+
+        VnfTemplateUtils.validateApiCommandParams(cmd, template);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateApiCommandParamsInvalidAccessDetails() {
+        VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class);
+        Mockito.when(template.getTemplateType()).thenReturn(Storage.TemplateType.VNF);
+        UpdateVnfTemplateCmd cmd = Mockito.mock(UpdateVnfTemplateCmd.class);
+        Map<String, String> vnfDetails = Mockito.spy(new HashMap<>());
+        vnfDetails.put("invalid", "value");
+        Mockito.when(cmd.getVnfDetails()).thenReturn(vnfDetails);
+
+        VnfTemplateUtils.validateApiCommandParams(cmd, template);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateApiCommandParamsInvalidTemplateType() {
+        VirtualMachineTemplate template = Mockito.mock(VirtualMachineTemplate.class);
+        Mockito.when(template.getTemplateType()).thenReturn(Storage.TemplateType.USER);
+        UpdateVnfTemplateCmd cmd = Mockito.mock(UpdateVnfTemplateCmd.class);
+
+        VnfTemplateUtils.validateApiCommandParams(cmd, template);
+    }
+
+    @Test
+    public void testValidateVnfCidrList() {
+        List<String> cidrList = new ArrayList<>();
+        cidrList.add("10.10.10.0/24");
+        VnfTemplateUtils.validateVnfCidrList(cidrList);
+    }
+
+    @Test
+    public void testValidateVnfCidrListWithEmptyList() {
+        List<String> cidrList = new ArrayList<>();
+        VnfTemplateUtils.validateVnfCidrList(cidrList);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testValidateVnfCidrListwithInvalidList() {
+        List<String> cidrList = new ArrayList<>();
+        cidrList.add("10.10.10.0/24");
+        cidrList.add("10.10.10.0/33");
+        VnfTemplateUtils.validateVnfCidrList(cidrList);
+    }
+}

client/pom.xml

@@ -532,6 +532,16 @@
             <artifactId>cloud-plugin-network-globodns</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-cluster-drs-balanced</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-cluster-drs-condensed</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.cloudstack</groupId>
             <artifactId>cloud-plugin-database-quota</artifactId>

core/src/main/resources/META-INF/cloudstack/cluster/module.properties

@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name=cluster
+parent=core

core/src/main/resources/META-INF/cloudstack/cluster/spring-core-lifecycle-cluster-context-inheritable.xml

@@ -0,0 +1,31 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                      http://www.springframework.org/schema/beans/spring-beans-3.0.xsd"
+>
+    <bean class="org.apache.cloudstack.spring.lifecycle.registry.RegistryLifecycle">
+        <property name="registry" ref="clusterDrsAlgorithmRegistry" />
+        <property name="typeClass" value="org.apache.cloudstack.cluster.ClusterDrsAlgorithm" />
+    </bean>
+
+</beans>

core/src/main/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml

@@ -346,4 +346,8 @@
     <bean id="userDataProvidersRegistry"
           class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
     </bean>
+
+    <bean id="clusterDrsAlgorithmRegistry"
+          class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry">
+    </bean>
 </beans>

engine/components-api/src/main/java/com/cloud/template/TemplateManager.java

@@ -18,6 +18,7 @@ package com.cloud.template;
 
 import java.util.List;
 
+import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -29,6 +30,7 @@ import com.cloud.deploy.DeployDestination;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage.TemplateType;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VMTemplateStoragePoolVO;
 import com.cloud.storage.VMTemplateVO;
@@ -133,5 +135,7 @@ public interface TemplateManager {
     public static final String MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT = "Message.RegisterPublicTemplate.Event";
     public static final String MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT = "Message.ResetTemplatePermission.Event";
 
+    TemplateType validateTemplateType(BaseCmd cmd, boolean isAdmin, boolean isCrossZones);
+
     List<DatadiskTO> getTemplateDisksOnImageStore(Long templateId, DataStoreRole role, String configurationId);
 }

engine/schema/src/main/java/com/cloud/storage/VnfTemplateDetailVO.java

@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Lob;
+import javax.persistence.Table;
+
+import org.apache.cloudstack.api.ResourceDetail;
+
+@Entity
+@Table(name = "vnf_template_details")
+public class VnfTemplateDetailVO implements ResourceDetail {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    private long id;
+
+    @Column(name = "template_id")
+    private long resourceId;
+
+    @Column(name = "name")
+    private String name;
+
+    @Lob
+    @Column(name = "value", length = 65535)
+    private String value;
+
+    @Column(name = "display")
+    private boolean display = true;
+
+    public VnfTemplateDetailVO() {
+    }
+
+    public VnfTemplateDetailVO(long templateId, String name, String value, boolean display) {
+        this.resourceId = templateId;
+        this.name = name;
+        this.value = value;
+        this.display = display;
+    }
+
+    @Override
+    public long getId() {
+        return id;
+    }
+
+    @Override
+    public long getResourceId() {
+        return resourceId;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public String getValue() {
+        return value;
+    }
+
+    @Override
+    public boolean isDisplay() {
+        return display;
+    }
+
+    public void setId(long id) {
+        this.id = id;
+    }
+
+    public void setResourceId(long resourceId) {
+        this.resourceId = resourceId;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+}

engine/schema/src/main/java/com/cloud/storage/VnfTemplateNicVO.java

@@ -0,0 +1,101 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage;
+
+import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "vnf_template_nics")
+public class VnfTemplateNicVO implements InternalIdentity {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id")
+    private long id;
+
+    @Column(name = "template_id")
+    private long templateId;
+
+    @Column(name = "device_id")
+    private long deviceId;
+
+    @Column(name = "device_name")
+    private String deviceName;
+
+    @Column(name = "required")
+    private boolean required = true;
+
+    @Column(name = "management")
+    private boolean management = true;
+
+    @Column(name = "description")
+    private String description;
+
+    public VnfTemplateNicVO() {
+    }
+
+    public VnfTemplateNicVO(long templateId, long deviceId, String deviceName, boolean required, boolean management, String description) {
+        this.templateId = templateId;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.required = required;
+        this.management = management;
+        this.description = description;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("Template %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "templateId", "deviceId", "required"));
+    }
+
+    @Override
+    public long getId() {
+        return id;
+    }
+
+    public long getTemplateId() {
+        return templateId;
+    }
+
+    public long getDeviceId() {
+        return deviceId;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public boolean isRequired() {
+        return required;
+    }
+
+    public boolean isManagement() {
+        return management;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+}

engine/schema/src/main/java/com/cloud/storage/dao/VnfTemplateDetailsDao.java

@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage.dao;
+
+import com.cloud.storage.VnfTemplateDetailVO;
+import com.cloud.utils.db.GenericDao;
+
+import org.apache.cloudstack.resourcedetail.ResourceDetailsDao;
+
+public interface VnfTemplateDetailsDao extends GenericDao<VnfTemplateDetailVO, Long>, ResourceDetailsDao<VnfTemplateDetailVO> {
+
+}

engine/schema/src/main/java/com/cloud/storage/dao/VnfTemplateDetailsDaoImpl.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage.dao;
+
+import com.cloud.storage.VnfTemplateDetailVO;
+
+import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase;
+import org.springframework.stereotype.Component;
+
+@Component
+public class VnfTemplateDetailsDaoImpl extends ResourceDetailsDaoBase<VnfTemplateDetailVO> implements VnfTemplateDetailsDao {
+
+    @Override
+    public void addDetail(long resourceId, String key, String value, boolean display) {
+        super.addDetail(new VnfTemplateDetailVO(resourceId, key, value, display));
+    }
+}

engine/schema/src/main/java/com/cloud/storage/dao/VnfTemplateNicDao.java

@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage.dao;
+
+import java.util.List;
+
+import com.cloud.storage.VnfTemplateNicVO;
+import com.cloud.utils.db.GenericDao;
+
+public interface VnfTemplateNicDao extends GenericDao<VnfTemplateNicVO, Long> {
+
+    List<VnfTemplateNicVO> listByTemplateId(long templateId);
+
+    void deleteByTemplateId(long templateId);
+}

engine/schema/src/main/java/com/cloud/storage/dao/VnfTemplateNicDaoImpl.java

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage.dao;
+
+import java.util.List;
+
+import com.cloud.storage.VnfTemplateNicVO;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.TransactionLegacy;
+
+public class VnfTemplateNicDaoImpl extends GenericDaoBase<VnfTemplateNicVO, Long> implements VnfTemplateNicDao {
+
+    protected SearchBuilder<VnfTemplateNicVO> TemplateSearch;
+
+    public VnfTemplateNicDaoImpl() {
+        TemplateSearch = createSearchBuilder();
+        TemplateSearch.and("templateId", TemplateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
+        TemplateSearch.done();
+    }
+
+    @Override
+    public List<VnfTemplateNicVO> listByTemplateId(long templateId) {
+        SearchCriteria<VnfTemplateNicVO> sc = TemplateSearch.create();
+        sc.setParameters("templateId", templateId);
+        return listBy(sc);
+    }
+
+    @Override
+    public void deleteByTemplateId(long templateId) {
+        SearchCriteria<VnfTemplateNicVO> sc = TemplateSearch.create();
+        sc.setParameters("templateId", templateId);
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        txn.start();
+        remove(sc);
+        txn.commit();
+    }
+}

engine/schema/src/main/java/com/cloud/vm/NicVO.java

@@ -333,6 +333,8 @@ public class NicVO implements Nic {
             .append("-")
             .append(instanceId)
             .append("-")
+            .append(deviceId)
+            .append("-")
             .append(reservationId)
             .append("-")
             .append(iPv4Address)

engine/schema/src/main/java/org/apache/cloudstack/cluster/ClusterDrsPlanMigrationVO.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import org.apache.cloudstack.jobs.JobInfo;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+@Entity
+@Table(name = "cluster_drs_plan_migration")
+public class ClusterDrsPlanMigrationVO implements ClusterDrsPlanMigration {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id", nullable = false)
+    Long id;
+
+    @Column(name = "plan_id", nullable = false)
+    private long planId;
+
+    @Column(name = "vm_id", nullable = false)
+    private long vmId;
+
+    @Column(name = "src_host_id", nullable = false)
+    private long srcHostId;
+
+    @Column(name = "dest_host_id", nullable = false)
+    private long destHostId;
+
+    @Column(name = "job_id")
+    private Long jobId;
+
+    @Column(name = "status")
+    private JobInfo.Status status;
+
+
+    public ClusterDrsPlanMigrationVO(long planId, long vmId, long srcHostId, long destHostId) {
+        this.planId = planId;
+        this.vmId = vmId;
+        this.srcHostId = srcHostId;
+        this.destHostId = destHostId;
+    }
+
+    protected ClusterDrsPlanMigrationVO() {
+
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public long getPlanId() {
+        return planId;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public long getSrcHostId() {
+        return srcHostId;
+    }
+
+    public long getDestHostId() {
+        return destHostId;
+    }
+
+    public Long getJobId() {
+        return jobId;
+    }
+
+    public void setJobId(long jobId) {
+        this.jobId = jobId;
+    }
+
+    public JobInfo.Status getStatus() {
+        return status;
+    }
+
+    public void setStatus(JobInfo.Status status) {
+        this.status = status;
+    }
+
+}

engine/schema/src/main/java/org/apache/cloudstack/cluster/ClusterDrsPlanVO.java

@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import com.cloud.utils.db.GenericDao;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import java.util.Date;
+import java.util.UUID;
+
+@Entity
+@Table(name = "cluster_drs_plan")
+public class ClusterDrsPlanVO implements ClusterDrsPlan {
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id", nullable = false)
+    Long id;
+
+    @Column(name = "uuid", nullable = false)
+    String uuid;
+
+    @Column(name = GenericDao.CREATED_COLUMN)
+    Date created;
+
+    @Column(name = "cluster_id")
+    private long clusterId;
+
+    @Column(name = "event_id")
+    private long eventId;
+
+    @Column(name = "type")
+    private Type type;
+
+    @Column(name = "status")
+    private Status status;
+
+    public ClusterDrsPlanVO(long clusterId, long eventId, Type type, Status status) {
+        uuid = UUID.randomUUID().toString();
+        this.clusterId = clusterId;
+        this.eventId = eventId;
+        this.type = type;
+        this.status = status;
+    }
+
+    protected ClusterDrsPlanVO() {
+        uuid = UUID.randomUUID().toString();
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public long getEventId() {
+        return eventId;
+    }
+
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    public Type getType() {
+        return type;
+    }
+
+    public Date getCreated() {
+        return created;
+    }
+
+    @Override
+    public Status getStatus() {
+        return status;
+    }
+
+    public String getUuid() {
+        return uuid;
+    }
+
+    public void setStatus(Status status) {
+        this.status = status;
+    }
+}

engine/schema/src/main/java/org/apache/cloudstack/cluster/dao/ClusterDrsPlanDao.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster.dao;
+
+import com.cloud.utils.Pair;
+import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.cluster.ClusterDrsPlan;
+import org.apache.cloudstack.cluster.ClusterDrsPlanVO;
+
+import java.util.Date;
+import java.util.List;
+
+public interface ClusterDrsPlanDao extends GenericDao<ClusterDrsPlanVO, Long> {
+    List<ClusterDrsPlanVO> listByStatus(ClusterDrsPlan.Status status);
+
+    List<ClusterDrsPlanVO> listByClusterIdAndStatus(Long clusterId, ClusterDrsPlan.Status status);
+
+    ClusterDrsPlanVO listLatestPlanForClusterId(Long clusterId);
+
+    Pair<List<ClusterDrsPlanVO>, Integer> searchAndCount(Long clusterId, Long planId, Long startIndex,
+                                                         Long pageSizeVal);
+
+    int expungeBeforeDate(Date date);
+}

engine/schema/src/main/java/org/apache/cloudstack/cluster/dao/ClusterDrsPlanDaoImpl.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster.dao;
+
+import com.cloud.utils.Pair;
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.cluster.ClusterDrsPlan;
+import org.apache.cloudstack.cluster.ClusterDrsPlanVO;
+
+import java.util.Date;
+import java.util.List;
+
+public class ClusterDrsPlanDaoImpl extends GenericDaoBase<ClusterDrsPlanVO, Long> implements ClusterDrsPlanDao {
+    public ClusterDrsPlanDaoImpl() {
+    }
+
+    @Override
+    public List<ClusterDrsPlanVO> listByStatus(ClusterDrsPlan.Status status) {
+        SearchBuilder<ClusterDrsPlanVO> sb;
+        sb = createSearchBuilder();
+        sb.and(ApiConstants.STATUS, sb.entity().getStatus(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanVO> sc = sb.create();
+        sc.setParameters(ApiConstants.STATUS, status);
+        return search(sc, null);
+    }
+
+    @Override
+    public List<ClusterDrsPlanVO> listByClusterIdAndStatus(Long clusterId, ClusterDrsPlan.Status status) {
+        SearchBuilder<ClusterDrsPlanVO> sb;
+        sb = createSearchBuilder();
+        sb.and(ApiConstants.CLUSTER_ID, sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.and(ApiConstants.STATUS, sb.entity().getStatus(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanVO> sc = sb.create();
+        sc.setParameters(ApiConstants.CLUSTER_ID, clusterId);
+        sc.setParameters(ApiConstants.STATUS, status);
+        return search(sc, null);
+    }
+
+    @Override
+    public ClusterDrsPlanVO listLatestPlanForClusterId(Long clusterId) {
+        SearchBuilder<ClusterDrsPlanVO> sb;
+        sb = createSearchBuilder();
+        sb.and(ApiConstants.CLUSTER_ID, sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanVO> sc = sb.create();
+        sc.setParameters(ApiConstants.CLUSTER_ID, clusterId);
+        Filter filter = new Filter(ClusterDrsPlanVO.class, "id", false, 0L, 1L);
+        List<ClusterDrsPlanVO> plans = listBy(sc, filter);
+        if (plans != null && !plans.isEmpty()) {
+            return plans.get(0);
+        }
+        return null;
+    }
+
+    @Override
+    public Pair<List<ClusterDrsPlanVO>, Integer> searchAndCount(Long clusterId, Long planId, Long startIndex,
+                                                                Long pageSizeVal) {
+        SearchBuilder<ClusterDrsPlanVO> sb;
+        sb = createSearchBuilder();
+        sb.and(ApiConstants.CLUSTER_ID, sb.entity().getClusterId(), SearchCriteria.Op.EQ);
+        sb.and(ApiConstants.ID, sb.entity().getId(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanVO> sc = sb.create();
+        if (clusterId != null) {
+            sc.setParameters(ApiConstants.CLUSTER_ID, clusterId);
+        }
+        if (planId != null) {
+            sc.setParameters(ApiConstants.ID, planId);
+        }
+        Filter filter = new Filter(ClusterDrsPlanVO.class, "id", false, startIndex, pageSizeVal);
+        return searchAndCount(sc, filter);
+    }
+
+    @Override
+    public int expungeBeforeDate(Date date) {
+        SearchBuilder<ClusterDrsPlanVO> sb;
+        sb = createSearchBuilder();
+        sb.and(ApiConstants.CREATED, sb.entity().getCreated(), SearchCriteria.Op.LT);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanVO> sc = sb.create();
+        sc.setParameters(ApiConstants.CREATED, date);
+        return expunge(sc);
+    }
+}

engine/schema/src/main/java/org/apache/cloudstack/cluster/dao/ClusterDrsPlanMigrationDao.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster.dao;
+
+import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.cluster.ClusterDrsPlanMigrationVO;
+
+import java.util.List;
+
+public interface ClusterDrsPlanMigrationDao extends GenericDao<ClusterDrsPlanMigrationVO, Long> {
+    List<ClusterDrsPlanMigrationVO> listByPlanId(long planId);
+
+    List<ClusterDrsPlanMigrationVO> listPlanMigrationsToExecute(Long id);
+
+    List<ClusterDrsPlanMigrationVO> listPlanMigrationsInProgress(Long id);
+}

engine/schema/src/main/java/org/apache/cloudstack/cluster/dao/ClusterDrsPlanMigrationDaoImpl.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster.dao;
+
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import org.apache.cloudstack.cluster.ClusterDrsPlanMigrationVO;
+import org.apache.cloudstack.jobs.JobInfo;
+
+import java.util.List;
+
+public class ClusterDrsPlanMigrationDaoImpl extends GenericDaoBase<ClusterDrsPlanMigrationVO, Long> implements ClusterDrsPlanMigrationDao {
+    public ClusterDrsPlanMigrationDaoImpl() {
+    }
+
+    @Override
+    public List<ClusterDrsPlanMigrationVO> listByPlanId(long planId) {
+        SearchBuilder<ClusterDrsPlanMigrationVO> sb = createSearchBuilder();
+        sb.and("planId", sb.entity().getPlanId(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanMigrationVO> sc = sb.create();
+        sc.setParameters("planId", planId);
+        Filter filter = new Filter(ClusterDrsPlanMigrationVO.class, "id", true, null, null);
+        return search(sc, filter);
+    }
+
+    @Override
+    public List<ClusterDrsPlanMigrationVO> listPlanMigrationsToExecute(Long id) {
+        SearchBuilder<ClusterDrsPlanMigrationVO> sb = createSearchBuilder();
+        sb.and("planId", sb.entity().getPlanId(), SearchCriteria.Op.EQ);
+        sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.NULL);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanMigrationVO> sc = sb.create();
+        sc.setParameters("planId", id);
+        Filter filter = new Filter(ClusterDrsPlanMigrationVO.class, "id", true, null, null);
+        return search(sc, filter);
+    }
+
+    @Override
+    public List<ClusterDrsPlanMigrationVO> listPlanMigrationsInProgress(Long id) {
+        SearchBuilder<ClusterDrsPlanMigrationVO> sb = createSearchBuilder();
+        sb.and("planId", sb.entity().getPlanId(), SearchCriteria.Op.EQ);
+        sb.and("status", sb.entity().getStatus(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<ClusterDrsPlanMigrationVO> sc = sb.create();
+        sc.setParameters("planId", id);
+        sc.setParameters("status", JobInfo.Status.IN_PROGRESS);
+        Filter filter = new Filter(ClusterDrsPlanMigrationVO.class, "id", true, null, null);
+        return search(sc, filter);
+    }
+}

engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -279,4 +279,8 @@
   <bean id="PassphraseDaoImpl" class="org.apache.cloudstack.secret.dao.PassphraseDaoImpl" />
   <bean id="VMScheduleDaoImpl" class="org.apache.cloudstack.vm.schedule.dao.VMScheduleDaoImpl" />
   <bean id="VMScheduledJobDaoImpl" class="org.apache.cloudstack.vm.schedule.dao.VMScheduledJobDaoImpl" />
+  <bean id="vnfTemplateDetailsDaoImpl" class="com.cloud.storage.dao.VnfTemplateDetailsDaoImpl" />
+  <bean id="vnfTemplateNicDaoImpl" class="com.cloud.storage.dao.VnfTemplateNicDaoImpl" />
+  <bean id="ClusterDrsPlanDaoImpl" class="org.apache.cloudstack.cluster.dao.ClusterDrsPlanDaoImpl" />
+  <bean id="ClusterDrsPlanDetailsDaoImpl" class="org.apache.cloudstack.cluster.dao.ClusterDrsPlanMigrationDaoImpl" />
 </beans>

engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql

@@ -290,6 +290,267 @@ GROUP BY
 -- Set removed state for all removed accounts
 UPDATE `cloud`.`account` SET state='removed' WHERE `removed` IS NOT NULL;
 
+
+-- New tables for VNF
+CREATE TABLE IF NOT EXISTS `cloud`.`vnf_template_nics` (
+    `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+    `template_id` bigint unsigned NOT NULL COMMENT 'id of the VNF template',
+    `device_id` bigint unsigned NOT NULL COMMENT 'Device id of the NIC when plugged into the VNF appliances',
+    `device_name` varchar(1024) NOT NULL COMMENT 'Name of the NIC',
+    `required` tinyint NOT NULL DEFAULT '1' COMMENT 'True if the NIC is required. False if optional',
+    `management` tinyint NOT NULL DEFAULT '1' COMMENT 'True if the NIC is a management interface',
+    `description` varchar(1024) COMMENT 'Description of the NIC',
+    PRIMARY KEY (`id`),
+    UNIQUE KEY `uk_template_id_device_id` (`template_id`, `device_id`),
+    KEY `fk_vnf_template_nics__template_id` (`template_id`),
+    CONSTRAINT `fk_vnf_template_nics__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template` (`id`) ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+CREATE TABLE IF NOT EXISTS `cloud`.`vnf_template_details` (
+    `id` bigint unsigned NOT NULL AUTO_INCREMENT,
+    `template_id` bigint unsigned NOT NULL COMMENT 'id of the VNF template',
+    `name` varchar(255) NOT NULL,
+    `value` varchar(1024) NOT NULL,
+    `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user',
+    PRIMARY KEY (`id`),
+    KEY `fk_vnf_template_details__template_id` (`template_id`),
+    CONSTRAINT `fk_vnf_template_details__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template` (`id`) ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+DROP VIEW IF EXISTS `cloud`.`user_vm_view`;
+CREATE
+    VIEW `user_vm_view` AS
+SELECT
+    `vm_instance`.`id` AS `id`,
+    `vm_instance`.`name` AS `name`,
+    `user_vm`.`display_name` AS `display_name`,
+    `user_vm`.`user_data` AS `user_data`,
+    `account`.`id` AS `account_id`,
+    `account`.`uuid` AS `account_uuid`,
+    `account`.`account_name` AS `account_name`,
+    `account`.`type` AS `account_type`,
+    `domain`.`id` AS `domain_id`,
+    `domain`.`uuid` AS `domain_uuid`,
+    `domain`.`name` AS `domain_name`,
+    `domain`.`path` AS `domain_path`,
+    `projects`.`id` AS `project_id`,
+    `projects`.`uuid` AS `project_uuid`,
+    `projects`.`name` AS `project_name`,
+    `instance_group`.`id` AS `instance_group_id`,
+    `instance_group`.`uuid` AS `instance_group_uuid`,
+    `instance_group`.`name` AS `instance_group_name`,
+    `vm_instance`.`uuid` AS `uuid`,
+    `vm_instance`.`user_id` AS `user_id`,
+    `vm_instance`.`last_host_id` AS `last_host_id`,
+    `vm_instance`.`vm_type` AS `type`,
+    `vm_instance`.`limit_cpu_use` AS `limit_cpu_use`,
+    `vm_instance`.`created` AS `created`,
+    `vm_instance`.`state` AS `state`,
+    `vm_instance`.`update_time` AS `update_time`,
+    `vm_instance`.`removed` AS `removed`,
+    `vm_instance`.`ha_enabled` AS `ha_enabled`,
+    `vm_instance`.`hypervisor_type` AS `hypervisor_type`,
+    `vm_instance`.`instance_name` AS `instance_name`,
+    `vm_instance`.`guest_os_id` AS `guest_os_id`,
+    `vm_instance`.`display_vm` AS `display_vm`,
+    `guest_os`.`uuid` AS `guest_os_uuid`,
+    `vm_instance`.`pod_id` AS `pod_id`,
+    `host_pod_ref`.`uuid` AS `pod_uuid`,
+    `vm_instance`.`private_ip_address` AS `private_ip_address`,
+    `vm_instance`.`private_mac_address` AS `private_mac_address`,
+    `vm_instance`.`vm_type` AS `vm_type`,
+    `data_center`.`id` AS `data_center_id`,
+    `data_center`.`uuid` AS `data_center_uuid`,
+    `data_center`.`name` AS `data_center_name`,
+    `data_center`.`is_security_group_enabled` AS `security_group_enabled`,
+    `data_center`.`networktype` AS `data_center_network_type`,
+    `host`.`id` AS `host_id`,
+    `host`.`uuid` AS `host_uuid`,
+    `host`.`name` AS `host_name`,
+    `host`.`cluster_id` AS `cluster_id`,
+    `host`.`status` AS `host_status`,
+    `host`.`resource_state` AS `host_resource_state`,
+    `vm_template`.`id` AS `template_id`,
+    `vm_template`.`uuid` AS `template_uuid`,
+    `vm_template`.`name` AS `template_name`,
+    `vm_template`.`type` AS `template_type`,
+    `vm_template`.`display_text` AS `template_display_text`,
+    `vm_template`.`enable_password` AS `password_enabled`,
+    `iso`.`id` AS `iso_id`,
+    `iso`.`uuid` AS `iso_uuid`,
+    `iso`.`name` AS `iso_name`,
+    `iso`.`display_text` AS `iso_display_text`,
+    `service_offering`.`id` AS `service_offering_id`,
+    `service_offering`.`uuid` AS `service_offering_uuid`,
+    `disk_offering`.`uuid` AS `disk_offering_uuid`,
+    `disk_offering`.`id` AS `disk_offering_id`,
+    (CASE
+         WHEN ISNULL(`service_offering`.`cpu`) THEN `custom_cpu`.`value`
+         ELSE `service_offering`.`cpu`
+        END) AS `cpu`,
+    (CASE
+         WHEN ISNULL(`service_offering`.`speed`) THEN `custom_speed`.`value`
+         ELSE `service_offering`.`speed`
+        END) AS `speed`,
+    (CASE
+         WHEN ISNULL(`service_offering`.`ram_size`) THEN `custom_ram_size`.`value`
+         ELSE `service_offering`.`ram_size`
+        END) AS `ram_size`,
+    `backup_offering`.`uuid` AS `backup_offering_uuid`,
+    `backup_offering`.`id` AS `backup_offering_id`,
+    `service_offering`.`name` AS `service_offering_name`,
+    `disk_offering`.`name` AS `disk_offering_name`,
+    `backup_offering`.`name` AS `backup_offering_name`,
+    `storage_pool`.`id` AS `pool_id`,
+    `storage_pool`.`uuid` AS `pool_uuid`,
+    `storage_pool`.`pool_type` AS `pool_type`,
+    `volumes`.`id` AS `volume_id`,
+    `volumes`.`uuid` AS `volume_uuid`,
+    `volumes`.`device_id` AS `volume_device_id`,
+    `volumes`.`volume_type` AS `volume_type`,
+    `security_group`.`id` AS `security_group_id`,
+    `security_group`.`uuid` AS `security_group_uuid`,
+    `security_group`.`name` AS `security_group_name`,
+    `security_group`.`description` AS `security_group_description`,
+    `nics`.`id` AS `nic_id`,
+    `nics`.`uuid` AS `nic_uuid`,
+    `nics`.`device_id` AS `nic_device_id`,
+    `nics`.`network_id` AS `network_id`,
+    `nics`.`ip4_address` AS `ip_address`,
+    `nics`.`ip6_address` AS `ip6_address`,
+    `nics`.`ip6_gateway` AS `ip6_gateway`,
+    `nics`.`ip6_cidr` AS `ip6_cidr`,
+    `nics`.`default_nic` AS `is_default_nic`,
+    `nics`.`gateway` AS `gateway`,
+    `nics`.`netmask` AS `netmask`,
+    `nics`.`mac_address` AS `mac_address`,
+    `nics`.`broadcast_uri` AS `broadcast_uri`,
+    `nics`.`isolation_uri` AS `isolation_uri`,
+    `vpc`.`id` AS `vpc_id`,
+    `vpc`.`uuid` AS `vpc_uuid`,
+    `networks`.`uuid` AS `network_uuid`,
+    `networks`.`name` AS `network_name`,
+    `networks`.`traffic_type` AS `traffic_type`,
+    `networks`.`guest_type` AS `guest_type`,
+    `user_ip_address`.`id` AS `public_ip_id`,
+    `user_ip_address`.`uuid` AS `public_ip_uuid`,
+    `user_ip_address`.`public_ip_address` AS `public_ip_address`,
+    `ssh_details`.`value` AS `keypair_names`,
+    `resource_tags`.`id` AS `tag_id`,
+    `resource_tags`.`uuid` AS `tag_uuid`,
+    `resource_tags`.`key` AS `tag_key`,
+    `resource_tags`.`value` AS `tag_value`,
+    `resource_tags`.`domain_id` AS `tag_domain_id`,
+    `domain`.`uuid` AS `tag_domain_uuid`,
+    `domain`.`name` AS `tag_domain_name`,
+    `resource_tags`.`account_id` AS `tag_account_id`,
+    `account`.`account_name` AS `tag_account_name`,
+    `resource_tags`.`resource_id` AS `tag_resource_id`,
+    `resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
+    `resource_tags`.`resource_type` AS `tag_resource_type`,
+    `resource_tags`.`customer` AS `tag_customer`,
+    `async_job`.`id` AS `job_id`,
+    `async_job`.`uuid` AS `job_uuid`,
+    `async_job`.`job_status` AS `job_status`,
+    `async_job`.`account_id` AS `job_account_id`,
+    `affinity_group`.`id` AS `affinity_group_id`,
+    `affinity_group`.`uuid` AS `affinity_group_uuid`,
+    `affinity_group`.`name` AS `affinity_group_name`,
+    `affinity_group`.`description` AS `affinity_group_description`,
+    `autoscale_vmgroups`.`id` AS `autoscale_vmgroup_id`,
+    `autoscale_vmgroups`.`uuid` AS `autoscale_vmgroup_uuid`,
+    `autoscale_vmgroups`.`name` AS `autoscale_vmgroup_name`,
+    `vm_instance`.`dynamically_scalable` AS `dynamically_scalable`,
+    `user_data`.`id` AS `user_data_id`,
+    `user_data`.`uuid` AS `user_data_uuid`,
+    `user_data`.`name` AS `user_data_name`,
+    `user_vm`.`user_data_details` AS `user_data_details`,
+    `vm_template`.`user_data_link_policy` AS `user_data_policy`
+FROM
+    (((((((((((((((((((((((((((((((((((`user_vm`
+        JOIN `vm_instance` ON (((`vm_instance`.`id` = `user_vm`.`id`)
+            AND ISNULL(`vm_instance`.`removed`))))
+        JOIN `account` ON ((`vm_instance`.`account_id` = `account`.`id`)))
+        JOIN `domain` ON ((`vm_instance`.`domain_id` = `domain`.`id`)))
+        LEFT JOIN `guest_os` ON ((`vm_instance`.`guest_os_id` = `guest_os`.`id`)))
+        LEFT JOIN `host_pod_ref` ON ((`vm_instance`.`pod_id` = `host_pod_ref`.`id`)))
+        LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
+        LEFT JOIN `instance_group_vm_map` ON ((`vm_instance`.`id` = `instance_group_vm_map`.`instance_id`)))
+        LEFT JOIN `instance_group` ON ((`instance_group_vm_map`.`group_id` = `instance_group`.`id`)))
+        LEFT JOIN `data_center` ON ((`vm_instance`.`data_center_id` = `data_center`.`id`)))
+        LEFT JOIN `host` ON ((`vm_instance`.`host_id` = `host`.`id`)))
+        LEFT JOIN `vm_template` ON ((`vm_instance`.`vm_template_id` = `vm_template`.`id`)))
+        LEFT JOIN `vm_template` `iso` ON ((`iso`.`id` = `user_vm`.`iso_id`)))
+        LEFT JOIN `volumes` ON ((`vm_instance`.`id` = `volumes`.`instance_id`)))
+        LEFT JOIN `service_offering` ON ((`vm_instance`.`service_offering_id` = `service_offering`.`id`)))
+        LEFT JOIN `disk_offering` `svc_disk_offering` ON ((`volumes`.`disk_offering_id` = `svc_disk_offering`.`id`)))
+        LEFT JOIN `disk_offering` ON ((`volumes`.`disk_offering_id` = `disk_offering`.`id`)))
+        LEFT JOIN `backup_offering` ON ((`vm_instance`.`backup_offering_id` = `backup_offering`.`id`)))
+        LEFT JOIN `storage_pool` ON ((`volumes`.`pool_id` = `storage_pool`.`id`)))
+        LEFT JOIN `security_group_vm_map` ON ((`vm_instance`.`id` = `security_group_vm_map`.`instance_id`)))
+        LEFT JOIN `security_group` ON ((`security_group_vm_map`.`security_group_id` = `security_group`.`id`)))
+        LEFT JOIN `user_data` ON ((`user_data`.`id` = `user_vm`.`user_data_id`)))
+        LEFT JOIN `nics` ON (((`vm_instance`.`id` = `nics`.`instance_id`)
+        AND ISNULL(`nics`.`removed`))))
+        LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`)))
+        LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`)
+        AND ISNULL(`vpc`.`removed`))))
+        LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`)))
+        LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`)
+        AND (`ssh_details`.`name` = 'SSH.KeyPairNames'))))
+        LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`)
+        AND (`resource_tags`.`resource_type` = 'UserVm'))))
+        LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `vm_instance`.`id`)
+        AND (`async_job`.`instance_type` = 'VirtualMachine')
+        AND (`async_job`.`job_status` = 0))))
+        LEFT JOIN `affinity_group_vm_map` ON ((`vm_instance`.`id` = `affinity_group_vm_map`.`instance_id`)))
+        LEFT JOIN `affinity_group` ON ((`affinity_group_vm_map`.`affinity_group_id` = `affinity_group`.`id`)))
+        LEFT JOIN `autoscale_vmgroup_vm_map` ON ((`autoscale_vmgroup_vm_map`.`instance_id` = `vm_instance`.`id`)))
+        LEFT JOIN `autoscale_vmgroups` ON ((`autoscale_vmgroup_vm_map`.`vmgroup_id` = `autoscale_vmgroups`.`id`)))
+        LEFT JOIN `user_vm_details` `custom_cpu` ON (((`custom_cpu`.`vm_id` = `vm_instance`.`id`)
+        AND (`custom_cpu`.`name` = 'CpuNumber'))))
+        LEFT JOIN `user_vm_details` `custom_speed` ON (((`custom_speed`.`vm_id` = `vm_instance`.`id`)
+        AND (`custom_speed`.`name` = 'CpuSpeed'))))
+        LEFT JOIN `user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `vm_instance`.`id`)
+        AND (`custom_ram_size`.`name` = 'memory'))));
+
+-- Add tables for Cluster DRS
+DROP TABLE IF EXISTS `cloud`.`cluster_drs_plan`;
+CREATE TABLE `cloud`.`cluster_drs_plan` (
+  `id` bigint unsigned NOT NULL auto_increment COMMENT 'id',
+  `cluster_id` bigint unsigned NOT NULL,
+  `event_id` bigint unsigned NOT NULL,
+  `uuid` varchar(40) NOT NULL COMMENT 'schedule uuid',
+  `type` varchar(20) NOT NULL COMMENT 'type of plan',
+  `status` varchar(20) NOT NULL COMMENT 'status of plan',
+  `created` datetime NOT NULL COMMENT 'date created',
+  PRIMARY KEY (`id`),
+  INDEX `i_cluster_drs_plan__cluster_id_status`(`cluster_id`, `status`),
+  INDEX `i_cluster_drs_plan__status`(`status`),
+  INDEX `i_cluster_drs_plan__created`(`created`),
+  CONSTRAINT `fk_cluster_drs_plan__cluster_id` FOREIGN KEY (`cluster_id`) REFERENCES `cluster`(`id`) ON DELETE CASCADE
+) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+
+DROP TABLE IF EXISTS `cloud`.`cluster_drs_plan_migration`;
+CREATE TABLE `cloud`.`cluster_drs_plan_migration` (
+  `id` bigint unsigned NOT NULL auto_increment COMMENT 'id',
+  `plan_id` bigint unsigned NOT NULL,
+  `vm_id` bigint unsigned NOT NULL,
+  `src_host_id` bigint unsigned NOT NULL,
+  `dest_host_id` bigint unsigned NOT NULL,
+  `job_id` bigint unsigned NULL,
+  `status` varchar(20) NULL COMMENT 'status of async job',
+  PRIMARY KEY (`id`),
+  INDEX `i_cluster_drs_plan_migration__plan_id_status`(`plan_id`, `status`),
+  CONSTRAINT `fk_cluster_drs_plan_migration__plan_id` FOREIGN KEY (`plan_id`) REFERENCES `cluster_drs_plan`(`id`) ON DELETE CASCADE
+) ENGINE = InnoDB DEFAULT CHARSET = utf8;
+
+INSERT INTO `cloud`.`configuration_subgroup` (`name`, `keywords`, `precedence`, `group_id`) VALUES ('DRS', 'drs', 4, (SELECT id FROM `cloud`.`configuration_group` WHERE `name` = 'Miscellaneous'));
+
+UPDATE `cloud`.`configuration`
+    SET subgroup_id = (SELECT id FROM `cloud`.`configuration_subgroup` WHERE name = 'DRS')
+    WHERE name IN ('drs.automatic.enable', 'drs.algorithm', 'drs.automatic.interval', 'drs.max.migrations', 'drs.imbalance', 'drs.metric', 'drs.plan.expire.interval');
+
 -- Add table for snapshot zone reference
 CREATE TABLE  `cloud`.`snapshot_zone_ref` (
   `id` bigint unsigned NOT NULL auto_increment,

engine/schema/src/test/java/com/cloud/storage/VnfTemplateDetailVOTest.java

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class VnfTemplateDetailVOTest {
+
+    static long templateId  = 100L;
+    static String name = "key1";
+    static String value = "value1";
+    static boolean display = true;
+
+    @Test
+    public void testVnfTemplateNicVOProperties() {
+        VnfTemplateDetailVO detailVO = new VnfTemplateDetailVO(templateId, name, value, display);
+
+        Assert.assertEquals(templateId, detailVO.getResourceId());
+        Assert.assertEquals(name, detailVO.getName());
+        Assert.assertEquals(value, detailVO.getValue());
+        Assert.assertEquals(display, detailVO.isDisplay());
+    }
+}

engine/schema/src/test/java/com/cloud/storage/VnfTemplateNicVOTest.java

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class VnfTemplateNicVOTest {
+
+    static long templateId  = 100L;
+    static long deviceId = 0L;
+    static String deviceName = "eth0";
+    static boolean required = true;
+    static boolean management = false;
+    static String description = "description of vnf nic";
+
+
+    @Test
+    public void testVnfTemplateNicVOProperties() {
+        VnfTemplateNicVO nicVO = new VnfTemplateNicVO(templateId, deviceId, deviceName, required, management, description);
+
+        Assert.assertEquals(templateId, nicVO.getTemplateId());
+        Assert.assertEquals(deviceId, nicVO.getDeviceId());
+        Assert.assertEquals(deviceName, nicVO.getDeviceName());
+        Assert.assertEquals(required, nicVO.isRequired());
+        Assert.assertEquals(management, nicVO.isManagement());
+        Assert.assertEquals(description, nicVO.getDescription());
+
+        String expected = String.format("Template {\"deviceId\":%d,\"id\":0,\"required\":%s,\"templateId\":%d}", deviceId, required, templateId);
+        Assert.assertEquals(expected, nicVO.toString());
+    }
+}

engine/schema/src/test/java/com/cloud/storage/dao/VnfTemplateNicDaoImplTest.java

@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.storage.dao;
+
+import com.cloud.storage.VnfTemplateNicVO;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import com.cloud.utils.db.TransactionLegacy;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.util.List;
+
+@RunWith(MockitoJUnitRunner.class)
+public class VnfTemplateNicDaoImplTest {
+
+    @Mock
+    SearchBuilder<VnfTemplateNicVO> searchBuilderVnfTemplateNicVOMock;
+
+    @Mock
+    SearchCriteria<VnfTemplateNicVO> searchCriteriaVnfTemplateNicVOMock;
+
+    @Mock
+    List<VnfTemplateNicVO> listVnfTemplateNicVOMock;
+
+    @Mock
+    private TransactionLegacy transactionMock;
+
+    @Spy
+    VnfTemplateNicDaoImpl vnfTemplateNicDaoImplSpy;
+
+    @Before
+    public void setUp() {
+        vnfTemplateNicDaoImplSpy.TemplateSearch = searchBuilderVnfTemplateNicVOMock;
+        Mockito.doReturn(searchCriteriaVnfTemplateNicVOMock).when(searchBuilderVnfTemplateNicVOMock).create();
+        Mockito.doNothing().when(searchCriteriaVnfTemplateNicVOMock).setParameters(Mockito.anyString(), Mockito.any());
+    }
+
+    @Test
+    public void testListByTemplateId() {
+        Mockito.doReturn(listVnfTemplateNicVOMock).when(vnfTemplateNicDaoImplSpy).listBy(Mockito.any(SearchCriteria.class));
+        long templateId = 100L;
+
+        List<VnfTemplateNicVO> result = vnfTemplateNicDaoImplSpy.listByTemplateId(templateId);
+
+        Assert.assertEquals(listVnfTemplateNicVOMock, result);
+        Mockito.verify(searchCriteriaVnfTemplateNicVOMock).setParameters("templateId", templateId);
+    }
+
+    @Test
+    public void testDeleteByTemplateId() {
+        Mockito.doReturn(0).when(vnfTemplateNicDaoImplSpy).remove(searchCriteriaVnfTemplateNicVOMock);
+        long templateId = 100L;
+
+        try (MockedStatic<TransactionLegacy> ignore = Mockito.mockStatic(TransactionLegacy.class)) {
+            Mockito.when(TransactionLegacy.currentTxn()).thenReturn(transactionMock);
+            Mockito.doNothing().when(transactionMock).start();
+            Mockito.doReturn(true).when(transactionMock).commit();
+
+            vnfTemplateNicDaoImplSpy.deleteByTemplateId(templateId);
+
+            Mockito.verify(transactionMock, Mockito.times(1)).start();
+            Mockito.verify(vnfTemplateNicDaoImplSpy, Mockito.times(1)).remove(searchCriteriaVnfTemplateNicVOMock);
+            Mockito.verify(transactionMock, Mockito.times(1)).commit();
+        }
+    }
+}

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -66,10 +66,13 @@ import com.cloud.configuration.Config;
 import com.cloud.host.Host;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Snapshot.Type;
+import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.db.DB;
@@ -97,6 +100,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
 
     @Inject
     StorageManager storageManager;
+    @Inject
+    SnapshotDao snapshotDao;
 
     @Override
     public StrategyPriority canHandle(DataObject srcData, DataObject destData) {
@@ -583,8 +588,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
             fullSnapshot = snapshotFullBackup;
         }
         Map<String, String> options = new HashMap<String, String>();
-        options.put("fullSnapshot", fullSnapshot.toString());
-        options.put(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key(), String.valueOf(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()));
+
+        addCommandOptions(snapshotInfo, fullSnapshot, options);
         boolean encryptionRequired = anyVolumeRequiresEncryption(srcData, destData);
 
         Answer answer = null;
@@ -631,6 +636,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
 
     }
 
+    private void addCommandOptions(SnapshotInfo snapshotInfo, Boolean fullSnapshot, Map<String, String> options) {
+        SnapshotVO snap = snapshotDao.findById(snapshotInfo.getSnapshotId());
+        if (snap != null && Type.FROM_GROUP.name().equals(snap.getTypeDescription())) {
+            options.put("typeDescription", snap.getTypeDescription());
+        }
+        options.put("fullSnapshot", fullSnapshot.toString());
+        options.put(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key(), String.valueOf(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value()));
+    }
+
     @Override
     public void copyAsync(Map<VolumeInfo, DataStore> volumeMap, VirtualMachineTO vmTo, Host srcHost, Host destHost, AsyncCompletionCallback<CopyCommandResult> callback) {
         CopyCommandResult result = new CopyCommandResult(null, null);

framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java

@@ -1264,6 +1264,11 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
 
     @DB()
     protected void addJoins(StringBuilder str, Collection<JoinBuilder<SearchCriteria<?>>> joins) {
+        addJoins(str, joins, new HashMap<>());
+    }
+
+    @DB()
+    protected void addJoins(StringBuilder str, Collection<JoinBuilder<SearchCriteria<?>>> joins, Map<String, Integer> joinedTableNames) {
         boolean hasWhereClause = true;
         int fromIndex = str.lastIndexOf("WHERE");
         if (fromIndex == -1) {
@@ -1274,18 +1279,27 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
         }
 
         for (JoinBuilder<SearchCriteria<?>> join : joins) {
+            String joinTableName = join.getSecondAttribute().table;
+            String joinTableAlias = findNextJoinTableName(joinTableName, joinedTableNames);
             StringBuilder onClause = new StringBuilder();
             onClause.append(" ")
             .append(join.getType().getName())
             .append(" ")
-            .append(join.getSecondAttribute().table)
-            .append(" ON ")
+            .append(joinTableName);
+            if (!joinTableAlias.equals(joinTableName)) {
+                onClause.append(" ").append(joinTableAlias);
+            }
+            onClause.append(" ON ")
             .append(join.getFirstAttribute().table)
             .append(".")
             .append(join.getFirstAttribute().columnName)
-            .append("=")
-            .append(join.getSecondAttribute().table)
-            .append(".")
+            .append("=");
+            if(!joinTableAlias.equals(joinTableName)) {
+                onClause.append(joinTableAlias);
+            } else {
+                onClause.append(joinTableName);
+            }
+            onClause.append(".")
             .append(join.getSecondAttribute().columnName)
             .append(" ");
             str.insert(fromIndex, onClause);
@@ -1306,11 +1320,22 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
 
         for (JoinBuilder<SearchCriteria<?>> join : joins) {
             if (join.getT().getJoins() != null) {
-                addJoins(str, join.getT().getJoins());
+                addJoins(str, join.getT().getJoins(), joinedTableNames);
             }
         }
     }
 
+    protected static String findNextJoinTableName(String tableName, Map<String, Integer> usedTableNames) {
+        if (usedTableNames.containsKey(tableName)) {
+            Integer tableCounter = usedTableNames.get(tableName);
+            usedTableNames.put(tableName, ++tableCounter);
+            tableName = tableName + tableCounter;
+        } else {
+            usedTableNames.put(tableName, 0);
+        }
+        return tableName;
+    }
+
     private void removeAndClause(StringBuilder sql) {
         sql.delete(sql.length() - 4, sql.length());
     }

framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java

@@ -18,6 +18,10 @@ package com.cloud.utils.db;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.junit.Assert;
 import org.junit.Before;
@@ -36,6 +40,7 @@ public class GenericDaoBaseTest {
     @Mock
     SQLException mockedSQLException;
 
+    private static final DbTestDao dbTestDao = new DbTestDao();
     private static final String INTEGRITY_CONSTRAINT_VIOLATION = "23000";
     private static final int DUPLICATE_ENTRY_ERRO_CODE = 1062;
 
@@ -214,4 +219,51 @@ public class GenericDaoBaseTest {
 
         Assert.assertEquals(resultSetSize, result);
     }
+
+    @Test
+    public void addJoinsTest() {
+        StringBuilder joinString = new StringBuilder();
+        Collection<JoinBuilder<SearchCriteria<?>>> joins = new ArrayList<>();
+
+        Attribute attr1 = new Attribute("table1", "column1");
+        Attribute attr2 = new Attribute("table2", "column2");
+        Attribute attr3 = new Attribute("table3", "column1");
+        Attribute attr4 = new Attribute("table4", "column2");
+
+        joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), attr1, attr2, JoinBuilder.JoinType.INNER));
+        joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), attr3, attr4, JoinBuilder.JoinType.INNER));
+        dbTestDao.addJoins(joinString, joins);
+
+        Assert.assertEquals(" INNER JOIN table2 ON table1.column1=table2.column2  INNER JOIN table4 ON table3.column1=table4.column2 ", joinString.toString());
+    }
+
+    @Test
+    public void multiJoinSameTableTest() {
+        StringBuilder joinString = new StringBuilder();
+        Collection<JoinBuilder<SearchCriteria<?>>> joins = new ArrayList<>();
+
+        Attribute tAc1 = new Attribute("tableA", "column1");
+        Attribute tAc2 = new Attribute("tableA", "column2");
+        Attribute tAc3 = new Attribute("tableA", "column3");
+        Attribute tBc2 = new Attribute("tableB", "column2");
+        Attribute tCc3 = new Attribute("tableC", "column3");
+        Attribute tDc4 = new Attribute("tableD", "column4");
+
+        joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), tBc2, tAc1, JoinBuilder.JoinType.INNER));
+        joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), tCc3, tAc2, JoinBuilder.JoinType.INNER));
+        joins.add(new JoinBuilder<>(dbTestDao.createSearchCriteria(), tDc4, tAc3, JoinBuilder.JoinType.INNER));
+        dbTestDao.addJoins(joinString, joins);
+
+        Assert.assertEquals(" INNER JOIN tableA ON tableB.column2=tableA.column1  INNER JOIN tableA tableA1 ON tableC.column3=tableA1.column2  INNER JOIN tableA tableA2 ON tableD.column4=tableA2.column3 ", joinString.toString());
+    }
+
+    @Test
+    public void findNextTableNameTest() {
+        Map<String, Integer> usedTables = new HashMap<>();
+
+        Assert.assertEquals("tableA", GenericDaoBase.findNextJoinTableName("tableA", usedTables));
+        Assert.assertEquals("tableA1", GenericDaoBase.findNextJoinTableName("tableA", usedTables));
+        Assert.assertEquals("tableA2", GenericDaoBase.findNextJoinTableName("tableA", usedTables));
+        Assert.assertEquals("tableA3", GenericDaoBase.findNextJoinTableName("tableA", usedTables));
+    }
 }

plugins/affinity-group-processors/explicit-dedication/src/main/java/org/apache/cloudstack/affinity/ExplicitDedicationProcessor.java

@@ -86,7 +86,7 @@ public class ExplicitDedicationProcessor extends AffinityProcessorBase implement
      * This IncludeList is then used to update the avoid list for a given data center.
      */
     @Override
-    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) throws AffinityConflictException {
+    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, List<VirtualMachine> vmList) throws AffinityConflictException {
         VirtualMachine vm = vmProfile.getVirtualMachine();
         List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
         DataCenter dc = _dcDao.findById(vm.getDataCenterId());

plugins/affinity-group-processors/host-affinity/src/main/java/org/apache/cloudstack/affinity/HostAffinityProcessor.java

@@ -16,14 +16,16 @@
 // under the License.
 package org.apache.cloudstack.affinity;
 
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.HashSet;
 import java.util.ArrayList;
 
 import javax.inject.Inject;
 
-import com.cloud.vm.VMInstanceVO;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
 
@@ -50,37 +52,56 @@ public class HostAffinityProcessor extends AffinityProcessorBase implements Affi
     protected AffinityGroupVMMapDao _affinityGroupVMMapDao;
 
     @Override
-    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) throws AffinityConflictException {
+    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, List<VirtualMachine> vmList) throws AffinityConflictException {
         VirtualMachine vm = vmProfile.getVirtualMachine();
         List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
         if (CollectionUtils.isNotEmpty(vmGroupMappings)) {
             for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
-                processAffinityGroup(vmGroupMapping, plan, vm);
+                processAffinityGroup(vmGroupMapping, plan, vm, vmList);
             }
         }
     }
 
+
     /**
      * Process Affinity Group for VM deployment
      */
-    protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm) {
+    protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm, List<VirtualMachine> vmList) {
         AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId());
         s_logger.debug("Processing affinity group " + group.getName() + " for VM Id: " + vm.getId());
 
         List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId());
         groupVMIds.remove(vm.getId());
 
-        List<Long> preferredHosts = getPreferredHostsFromGroupVMIds(groupVMIds);
+        List<Long> preferredHosts = getPreferredHostsFromGroupVMIds(groupVMIds, vmList);
         plan.setPreferredHosts(preferredHosts);
     }
 
+
+    /**
+     * Process Affinity Group for VM deployment
+     */
+    protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm) {
+        processAffinityGroup(vmGroupMapping, plan, vm, Collections.emptyList());
+    }
+
     /**
      * Get host ids set from vm ids list
      */
+
     protected Set<Long> getHostIdSet(List<Long> vmIds) {
+        return getHostIdSet(vmIds, Collections.emptyList());
+    }
+
+    protected Set<Long> getHostIdSet(List<Long> vmIds, List<VirtualMachine> vmList) {
         Set<Long> hostIds = new HashSet<>();
+        Map<Long, VirtualMachine> vmIdVmMap = getVmIdVmMap(vmList);
         for (Long groupVMId : vmIds) {
-            VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId);
+            VirtualMachine groupVM = vmIdVmMap.get(groupVMId);
+            if (groupVM == null) {
+                groupVM = _vmInstanceDao.findById(groupVMId);
+            }
+
             if (groupVM != null && groupVM.getHostId() != null) {
                 hostIds.add(groupVM.getHostId());
             }
@@ -88,11 +109,19 @@ public class HostAffinityProcessor extends AffinityProcessorBase implements Affi
         return hostIds;
     }
 
+    protected Map<Long, VirtualMachine> getVmIdVmMap(List<VirtualMachine> vmList) {
+        Map<Long, VirtualMachine> vmIdVmMap = new HashMap<>();
+        for (VirtualMachine vm : vmList) {
+            vmIdVmMap.put(vm.getId(), vm);
+        }
+        return vmIdVmMap;
+    }
+
     /**
      * Get preferred host ids list from the affinity group VMs
      */
-    protected List<Long> getPreferredHostsFromGroupVMIds(List<Long> vmIds) {
-        return new ArrayList<>(getHostIdSet(vmIds));
+    protected List<Long> getPreferredHostsFromGroupVMIds(List<Long> vmIds, List<VirtualMachine> vmList) {
+        return new ArrayList<>(getHostIdSet(vmIds, vmList));
     }
 
     @Override

plugins/affinity-group-processors/host-affinity/src/test/java/org/apache/cloudstack/affinity/HostAffinityProcessorTest.java

@@ -36,6 +36,7 @@ import org.mockito.Spy;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
@@ -139,7 +140,7 @@ public class HostAffinityProcessorTest {
     @Test
     public void testGetPreferredHostsFromGroupVMIdsMultipleVMs() {
         List<Long> list = new ArrayList<>(Arrays.asList(GROUP_VM_1_ID, GROUP_VM_2_ID));
-        List<Long> preferredHosts = processor.getPreferredHostsFromGroupVMIds(list);
+        List<Long> preferredHosts = processor.getPreferredHostsFromGroupVMIds(list, Collections.emptyList());
         assertNotNull(preferredHosts);
         assertEquals(1, preferredHosts.size());
         assertEquals(HOST_ID, preferredHosts.get(0));
@@ -148,7 +149,7 @@ public class HostAffinityProcessorTest {
     @Test
     public void testGetPreferredHostsFromGroupVMIdsEmptyVMsList() {
         List<Long> list = new ArrayList<>();
-        List<Long> preferredHosts = processor.getPreferredHostsFromGroupVMIds(list);
+        List<Long> preferredHosts = processor.getPreferredHostsFromGroupVMIds(list, Collections.emptyList());
         assertNotNull(preferredHosts);
         assertTrue(preferredHosts.isEmpty());
     }

plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java

@@ -63,7 +63,7 @@ public class HostAntiAffinityProcessor extends AffinityProcessorBase implements
     protected VMReservationDao _reservationDao;
 
     @Override
-    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) throws AffinityConflictException {
+    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, List<VirtualMachine> vmList) throws AffinityConflictException {
         VirtualMachine vm = vmProfile.getVirtualMachine();
         List<AffinityGroupVMMapVO> vmGroupMappings = _affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
 

plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java

@@ -63,7 +63,7 @@ public class NonStrictHostAffinityProcessor extends AffinityProcessorBase implem
     private int vmCapacityReleaseInterval;
 
     @Override
-    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid) throws AffinityConflictException {
+    public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, List<VirtualMachine> vmList) throws AffinityConflictException {
         VirtualMachine vm = vmProfile.getVirtualMachine();
         List<AffinityGroupVMMapVO> vmGroupMappings = affinityGroupVMMapDao.findByVmIdType(vm.getId(), getType());
 

plugins/drs/cluster/balanced/pom.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <name>Apache CloudStack Plugin - Cluster DRS Algorithm - Balanced</name>
+    <artifactId>cloud-plugin-cluster-drs-balanced</artifactId>
+    <parent>
+        <groupId>org.apache.cloudstack</groupId>
+        <artifactId>cloudstack-plugins</artifactId>
+        <version>4.19.0.0-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+</project>

plugins/drs/cluster/balanced/src/main/java/org/apache/cloudstack/cluster/Balanced.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import com.cloud.host.Host;
+import com.cloud.offering.ServiceOffering;
+import com.cloud.utils.Pair;
+import com.cloud.utils.Ternary;
+import com.cloud.utils.component.AdapterBase;
+import com.cloud.vm.VirtualMachine;
+
+import javax.naming.ConfigurationException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsImbalanceThreshold;
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsMetric;
+
+public class Balanced extends AdapterBase implements ClusterDrsAlgorithm {
+
+    @Override
+    public String getName() {
+        return "balanced";
+    }
+
+    @Override
+    public boolean needsDrs(long clusterId, List<Long> cpuList, List<Long> memoryList) throws ConfigurationException {
+        Double cpuImbalance = getClusterImbalance(cpuList);
+        Double memoryImbalance = getClusterImbalance(memoryList);
+        double threshold = getThreshold(clusterId);
+        String metric = ClusterDrsMetric.valueIn(clusterId);
+        switch (metric) {
+            case "cpu":
+                return cpuImbalance > threshold;
+            case "memory":
+                return memoryImbalance > threshold;
+            default:
+                throw new ConfigurationException(
+                        String.format("Invalid metric: %s for cluster: %d", metric, clusterId));
+        }
+    }
+
+    private double getThreshold(long clusterId) throws ConfigurationException {
+        return 1.0 - ClusterDrsImbalanceThreshold.valueIn(clusterId);
+    }
+
+    @Override
+    public Ternary<Double, Double, Double> getMetrics(long clusterId, VirtualMachine vm,
+                                                      ServiceOffering serviceOffering, Host destHost,
+                                                      Map<Long, Long> hostCpuUsedMap, Map<Long, Long> hostMemoryUsedMap,
+                                                      Boolean requiresStorageMotion) {
+        Double preCpuImbalance = getClusterImbalance(new ArrayList<>(hostCpuUsedMap.values()));
+        Double preMemoryImbalance = getClusterImbalance(new ArrayList<>(hostMemoryUsedMap.values()));
+
+        Pair<Double, Double> imbalancePair = getImbalancePostMigration(serviceOffering, vm, destHost, hostCpuUsedMap,
+                hostMemoryUsedMap);
+        Double postCpuImbalance = imbalancePair.first();
+        Double postMemoryImbalance = imbalancePair.second();
+
+        // This needs more research to determine the cost and benefit of a migration
+        // TODO: Cost should be a factor of the VM size and the host capacity
+        // TODO: Benefit should be a factor of the VM size and the host capacity and the number of VMs on the host
+        double cost = 0.0;
+        double benefit = 1.0;
+
+        String metric = ClusterDrsMetric.valueIn(clusterId);
+        final double improvement;
+        switch (metric) {
+            case "cpu":
+                improvement = preCpuImbalance - postCpuImbalance;
+                break;
+            case "memory":
+                improvement = preMemoryImbalance - postMemoryImbalance;
+                break;
+            default:
+                improvement = preCpuImbalance + preMemoryImbalance - postCpuImbalance - postMemoryImbalance;
+        }
+
+        return new Ternary<>(improvement, cost, benefit);
+    }
+}

plugins/drs/cluster/balanced/src/main/resources/META-INF/cloudstack/balanced/module.properties

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+name=balanced
+parent=cluster

plugins/drs/cluster/balanced/src/main/resources/META-INF/cloudstack/balanced/spring-balanced-context.xml

@@ -0,0 +1,33 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                      http://www.springframework.org/schema/beans/spring-beans.xsd
+                      http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+                      http://www.springframework.org/schema/context
+                      http://www.springframework.org/schema/context/spring-context.xsd"
+                      >
+
+  <bean id="balanced" class="org.apache.cloudstack.cluster.Balanced">
+      <property name="name" value="balanced" />
+  </bean>
+</beans>

plugins/drs/cluster/balanced/src/test/java/org/apache/cloudstack/cluster/BalancedTest.java

@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import com.cloud.host.Host;
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.service.dao.ServiceOfferingDao;
+import com.cloud.utils.Ternary;
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import javax.naming.ConfigurationException;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsImbalanceThreshold;
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsMetric;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BalancedTest {
+
+    @InjectMocks
+    Balanced balanced;
+
+    VirtualMachine vm1, vm2, vm3;
+
+    Host destHost;
+
+    ServiceOfferingVO serviceOffering;
+
+    long clusterId = 1L;
+
+    Map<Long, List<VirtualMachine>> hostVmMap;
+
+    List<Long> cpuList, memoryList;
+
+    Map<Long, Long> hostCpuUsedMap, hostMemoryUsedMap;
+
+
+    @Mock
+    private ServiceOfferingDao serviceOfferingDao;
+
+
+    private AutoCloseable closeable;
+
+    @Before
+    public void setUp() throws NoSuchFieldException, IllegalAccessException {
+        closeable = MockitoAnnotations.openMocks(this);
+
+
+        vm1 = Mockito.mock(VirtualMachine.class);
+        vm2 = Mockito.mock(VirtualMachine.class);
+        vm3 = Mockito.mock(VirtualMachine.class); // vm to migrate
+
+        destHost = Mockito.mock(Host.class);
+        hostVmMap = new HashMap<>();
+        hostVmMap.put(1L, Collections.singletonList(vm1));
+        hostVmMap.put(2L, Arrays.asList(vm2, vm3));
+
+        serviceOffering = Mockito.mock(ServiceOfferingVO.class);
+        Mockito.when(vm3.getHostId()).thenReturn(2L);
+
+        Mockito.when(destHost.getId()).thenReturn(1L);
+
+        Mockito.when(serviceOffering.getCpu()).thenReturn(1);
+        Mockito.when(serviceOffering.getSpeed()).thenReturn(1000);
+        Mockito.when(serviceOffering.getRamSize()).thenReturn(512);
+
+        overrideDefaultConfigValue(ClusterDrsImbalanceThreshold, "_defaultValue", "0.5");
+
+        cpuList = Arrays.asList(1L, 2L);
+        memoryList = Arrays.asList(512L, 2048L);
+
+        hostCpuUsedMap = new HashMap<>();
+        hostCpuUsedMap.put(1L, 1000L);
+        hostCpuUsedMap.put(2L, 2000L);
+
+        hostMemoryUsedMap = new HashMap<>();
+        hostMemoryUsedMap.put(1L, 512L * 1024L * 1024L);
+        hostMemoryUsedMap.put(2L, 2048L * 1024L * 1024L);
+    }
+
+    private void overrideDefaultConfigValue(final ConfigKey configKey, final String name,
+                                            final Object o) throws IllegalAccessException, NoSuchFieldException {
+        Field f = ConfigKey.class.getDeclaredField(name);
+        f.setAccessible(true);
+        f.set(configKey, o);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        closeable.close();
+    }
+
+    /**
+     * <b>needsDrs tests</b>
+     * <p>Scenarios to test for needsDrs
+     * <p>1. cluster with cpu metric
+     * <p>2. cluster with memory metric
+     * <p>3. cluster with "unknown" metric
+     * <p>
+     * <p>CPU imbalance = 0.333
+     * <p>Memory imbalance = 0.6
+     */
+
+    /*
+     1. cluster with cpu metric
+     0.3333 > 0.5 -> False
+    */
+    @Test
+    public void needsDrsWithCpu() throws ConfigurationException, NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "cpu");
+        assertFalse(balanced.needsDrs(clusterId, cpuList, memoryList));
+    }
+
+    /*
+     2. cluster with memory metric
+     0.6 > 0.5 -> True
+    */
+    @Test
+    public void needsDrsWithMemory() throws ConfigurationException, NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "memory");
+        assertTrue(balanced.needsDrs(clusterId, cpuList, memoryList));
+    }
+
+    /* 3. cluster with "unknown" metric */
+    @Test
+    public void needsDrsWithUnknown() throws ConfigurationException, NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "unknown");
+        assertThrows(ConfigurationException.class, () -> balanced.needsDrs(clusterId, cpuList, memoryList));
+    }
+
+    /**
+     * getMetrics tests
+     * <p>Scenarios to test for getMetrics
+     * <p>1. cluster with cpu metric
+     * <p>2. cluster with memory metric
+     * <p>3. cluster with default metric
+     * <p>
+     * <p>Pre
+     * <p>CPU imbalance = 0.333333
+     * <p>Memory imbalance = 0.6
+     * <p>
+     * <p>Post
+     * <p>CPU imbalance = 0.3333
+     * <p>Memory imbalance = 0.2
+     * <p>
+     * <p>Cost 512.0
+     * <p>Benefit  (0.6-0.2) * 8192 = 3276.8
+     */
+
+    /*
+     1. cluster with cpu metric
+     improvement = 0.3333 - 0.3333  = 0.0
+    */
+    @Test
+    public void getMetricsWithCpu() throws NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "cpu");
+        Ternary<Double, Double, Double> result = balanced.getMetrics(clusterId, vm3, serviceOffering, destHost,
+                hostCpuUsedMap, hostMemoryUsedMap, false);
+        assertEquals(0.0, result.first(), 0.01);
+        assertEquals(0.0, result.second(), 0.0);
+        assertEquals(1.0, result.third(), 0.0);
+    }
+
+    /*
+     2. cluster with memory metric
+     improvement = 0.6 - 0.2 = 0.4
+    */
+    @Test
+    public void getMetricsWithMemory() throws NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "memory");
+        Ternary<Double, Double, Double> result = balanced.getMetrics(clusterId, vm3, serviceOffering, destHost,
+                hostCpuUsedMap, hostMemoryUsedMap, false);
+        assertEquals(0.4, result.first(), 0.01);
+        assertEquals(0, result.second(), 0.0);
+        assertEquals(1, result.third(), 0.0);
+    }
+
+    /*
+     3. cluster with default metric
+     improvement = 0.3333 + 0.6 - 0.3333 - 0.2 = 0.4
+    */
+    @Test
+    public void getMetricsWithDefault() throws NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "both");
+        Ternary<Double, Double, Double> result = balanced.getMetrics(clusterId, vm3, serviceOffering, destHost,
+                hostCpuUsedMap, hostMemoryUsedMap, false);
+        assertEquals(0.4, result.first(), 0.01);
+        assertEquals(0, result.second(), 0.0);
+        assertEquals(1, result.third(), 0.0);
+    }
+}

plugins/drs/cluster/condensed/pom.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <name>Apache CloudStack Plugin - Cluster DRS Algorithm - Condensed</name>
+    <artifactId>cloud-plugin-cluster-drs-condensed</artifactId>
+    <parent>
+        <groupId>org.apache.cloudstack</groupId>
+        <artifactId>cloudstack-plugins</artifactId>
+        <version>4.19.0.0-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+</project>

plugins/drs/cluster/condensed/src/main/java/org/apache/cloudstack/cluster/Condensed.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import com.cloud.host.Host;
+import com.cloud.offering.ServiceOffering;
+import com.cloud.utils.Pair;
+import com.cloud.utils.Ternary;
+import com.cloud.utils.component.AdapterBase;
+import com.cloud.vm.VirtualMachine;
+
+import javax.naming.ConfigurationException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsImbalanceThreshold;
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsMetric;
+
+public class Condensed extends AdapterBase implements ClusterDrsAlgorithm {
+
+    @Override
+    public String getName() {
+        return "condensed";
+    }
+
+    @Override
+    public boolean needsDrs(long clusterId, List<Long> cpuList, List<Long> memoryList) throws ConfigurationException {
+        Double cpuImbalance = getClusterImbalance(cpuList);
+        Double memoryImbalance = getClusterImbalance(memoryList);
+        double threshold = getThreshold(clusterId);
+        String metric = ClusterDrsMetric.valueIn(clusterId);
+        switch (metric) {
+            case "cpu":
+                return cpuImbalance < threshold;
+            case "memory":
+                return memoryImbalance < threshold;
+            default:
+                throw new ConfigurationException(
+                        String.format("Invalid metric: %s for cluster: %d", metric, clusterId));
+        }
+    }
+
+    private double getThreshold(long clusterId) throws ConfigurationException {
+        return ClusterDrsImbalanceThreshold.valueIn(clusterId);
+    }
+
+    @Override
+    public Ternary<Double, Double, Double> getMetrics(long clusterId, VirtualMachine vm,
+                                                      ServiceOffering serviceOffering, Host destHost,
+                                                      Map<Long, Long> hostCpuUsedMap, Map<Long, Long> hostMemoryUsedMap,
+                                                      Boolean requiresStorageMotion) {
+        Double preCpuImbalance = getClusterImbalance(new ArrayList<>(hostCpuUsedMap.values()));
+        Double preMemoryImbalance = getClusterImbalance(new ArrayList<>(hostMemoryUsedMap.values()));
+
+        Pair<Double, Double> imbalancePair = getImbalancePostMigration(serviceOffering, vm, destHost, hostCpuUsedMap,
+                hostMemoryUsedMap);
+        Double postCpuImbalance = imbalancePair.first();
+        Double postMemoryImbalance = imbalancePair.second();
+
+        // This needs more research to determine the cost and benefit of a migration
+        // TODO: Cost should be a factor of the VM size and the host capacity
+        // TODO: Benefit should be a factor of the VM size and the host capacity and the number of VMs on the host
+        double cost = 0;
+        double benefit = 1;
+
+        String metric = ClusterDrsMetric.valueIn(clusterId);
+        double improvement;
+        switch (metric) {
+            case "cpu":
+                improvement = postCpuImbalance - preCpuImbalance;
+                break;
+            case "memory":
+                improvement = postMemoryImbalance - preMemoryImbalance;
+                break;
+            default:
+                improvement = postCpuImbalance + postMemoryImbalance - preCpuImbalance - preMemoryImbalance;
+        }
+        return new Ternary<>(improvement, cost, benefit);
+    }
+}

plugins/drs/cluster/condensed/src/main/resources/META-INF/cloudstack/condensed/module.properties

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+name=condensed
+parent=cluster

plugins/drs/cluster/condensed/src/main/resources/META-INF/cloudstack/condensed/spring-condensed-context.xml

@@ -0,0 +1,33 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+<beans xmlns="http://www.springframework.org/schema/beans"
+       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xmlns:context="http://www.springframework.org/schema/context"
+       xmlns:aop="http://www.springframework.org/schema/aop"
+       xsi:schemaLocation="http://www.springframework.org/schema/beans
+                      http://www.springframework.org/schema/beans/spring-beans.xsd
+                      http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop.xsd
+                      http://www.springframework.org/schema/context
+                      http://www.springframework.org/schema/context/spring-context.xsd"
+                      >
+
+  <bean id="condensed" class="org.apache.cloudstack.cluster.Condensed">
+      <property name="name" value="condensed" />
+  </bean>
+</beans>

plugins/drs/cluster/condensed/src/test/java/org/apache/cloudstack/cluster/CondensedTest.java

@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.cloudstack.cluster;
+
+import com.cloud.host.Host;
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.utils.Ternary;
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import javax.naming.ConfigurationException;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsImbalanceThreshold;
+import static org.apache.cloudstack.cluster.ClusterDrsService.ClusterDrsMetric;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class CondensedTest {
+
+    @InjectMocks
+    Condensed condensed;
+
+    VirtualMachine vm1, vm2, vm3;
+
+    Host destHost;
+
+    ServiceOfferingVO serviceOffering;
+
+    long clusterId = 1L;
+
+    Map<Long, List<VirtualMachine>> hostVmMap;
+
+    List<Long> cpuList, memoryList;
+
+    Map<Long, Long> hostCpuUsedMap, hostMemoryUsedMap;
+
+
+    private AutoCloseable closeable;
+
+    @Before
+    public void setUp() throws NoSuchFieldException, IllegalAccessException {
+        closeable = MockitoAnnotations.openMocks(this);
+
+        vm1 = Mockito.mock(VirtualMachine.class);
+        vm2 = Mockito.mock(VirtualMachine.class);
+        vm3 = Mockito.mock(VirtualMachine.class); // vm to migrate
+
+        destHost = Mockito.mock(Host.class);
+        hostVmMap = new HashMap<>();
+        hostVmMap.put(1L, Collections.singletonList(vm1));
+        hostVmMap.put(2L, Arrays.asList(vm2, vm3));
+
+        serviceOffering = Mockito.mock(ServiceOfferingVO.class);
+        Mockito.when(vm3.getHostId()).thenReturn(2L);
+
+        Mockito.when(destHost.getId()).thenReturn(1L);
+
+        Mockito.when(serviceOffering.getCpu()).thenReturn(1);
+        Mockito.when(serviceOffering.getSpeed()).thenReturn(1000);
+        Mockito.when(serviceOffering.getRamSize()).thenReturn(512);
+
+        overrideDefaultConfigValue(ClusterDrsImbalanceThreshold, "_defaultValue", "0.5");
+
+        cpuList = Arrays.asList(1L, 2L);
+        memoryList = Arrays.asList(512L, 2048L);
+
+        hostCpuUsedMap = new HashMap<>();
+        hostCpuUsedMap.put(1L, 1000L);
+        hostCpuUsedMap.put(2L, 2000L);
+
+        hostMemoryUsedMap = new HashMap<>();
+        hostMemoryUsedMap.put(1L, 512L * 1024L * 1024L);
+        hostMemoryUsedMap.put(2L, 2048L * 1024L * 1024L);
+    }
+
+    private void overrideDefaultConfigValue(final ConfigKey configKey,
+                                            final String name,
+                                            final Object o) throws IllegalAccessException, NoSuchFieldException {
+        Field f = ConfigKey.class.getDeclaredField(name);
+        f.setAccessible(true);
+        f.set(configKey, o);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        closeable.close();
+    }
+
+    /**
+     * <p>needsDrs tests
+     * <p>Scenarios to test for needsDrs
+     * <p>1. cluster with cpu metric
+     * <p>2. cluster with memory metric
+     * <p>3. cluster with "unknown" metric
+     * <p>
+     * <p>CPU imbalance = 0.333
+     * <p>Memory imbalance = 0.6
+     */
+
+    /*
+     1. cluster with cpu metric
+     0.3333 < 0.5 -> True
+    */
+    @Test
+    public void needsDrsWithCpu() throws ConfigurationException, NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "cpu");
+        assertTrue(condensed.needsDrs(clusterId, cpuList, memoryList));
+    }
+
+    /*
+     2. cluster with memory metric
+     0.6 < 0.5 -> False
+    */
+    @Test
+    public void needsDrsWithMemory() throws ConfigurationException, NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "memory");
+        assertFalse(condensed.needsDrs(clusterId, cpuList, memoryList));
+    }
+
+    /* 3. cluster with "unknown" metric */
+    @Test
+    public void needsDrsWithUnknown() throws ConfigurationException, NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "unknown");
+        assertThrows(ConfigurationException.class, () -> condensed.needsDrs(clusterId, cpuList, memoryList));
+    }
+
+    /**
+     * getMetrics tests
+     * <p>Scenarios to test for getMetrics
+     * <p>1. cluster with cpu metric
+     * <p>2. cluster with memory metric
+     * <p>3. cluster with default metric
+     * <p>
+     * <p>Pre
+     * <p>CPU imbalance = 0.333333
+     * <p>Memory imbalance = 0.6
+     * <p>
+     * <p>Post
+     * <p>CPU imbalance = 0.3333
+     * <p>Memory imbalance = 0.2
+     * <p>
+     * <p>Cost 512.0
+     * <p>Benefit  (0.2-0.6) * 8192 = -3276.8
+     */
+
+    /*
+     1. cluster with cpu metric
+     improvement = 0.3333 - 0.3333 = 0.0
+    */
+    @Test
+    public void getMetricsWithCpu() throws NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "cpu");
+        Ternary<Double, Double, Double> result = condensed.getMetrics(clusterId, vm3, serviceOffering, destHost,
+                hostCpuUsedMap, hostMemoryUsedMap, false);
+        assertEquals(0.0, result.first(), 0.0);
+        assertEquals(0, result.second(), 0.0);
+        assertEquals(1, result.third(), 0.0);
+    }
+
+    /*
+     2. cluster with memory metric
+     improvement = 0.2 - 0.6 = -0.4
+    */
+    @Test
+    public void getMetricsWithMemory() throws NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "memory");
+        Ternary<Double, Double, Double> result = condensed.getMetrics(clusterId, vm3, serviceOffering, destHost,
+                hostCpuUsedMap, hostMemoryUsedMap, false);
+        assertEquals(-0.4, result.first(), 0.01);
+        assertEquals(0, result.second(), 0.0);
+        assertEquals(1, result.third(), 0.0);
+    }
+
+    /*
+     3. cluster with default metric
+     improvement = 0.3333 + 0.2 - 0.3333 - 0.6 = -0.4
+    */
+    @Test
+    public void getMetricsWithDefault() throws NoSuchFieldException, IllegalAccessException {
+        overrideDefaultConfigValue(ClusterDrsMetric, "_defaultValue", "both");
+        Ternary<Double, Double, Double> result = condensed.getMetrics(clusterId, vm3, serviceOffering, destHost,
+                hostCpuUsedMap, hostMemoryUsedMap, false);
+        assertEquals(-0.4, result.first(), 0.0001);
+        assertEquals(0, result.second(), 0.0);
+        assertEquals(1, result.third(), 0.0);
+    }
+}

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -1014,9 +1014,9 @@ public class KVMStorageProcessor implements StorageProcessor {
                 command.add("-b", isCreatedFromVmSnapshot ? snapshotDisk.getPath() : snapshot.getPath());
                 command.add(NAME_OPTION, snapshotName);
                 command.add("-p", snapshotDestPath);
-                if (isCreatedFromVmSnapshot) {
-                    descName = UUID.randomUUID().toString();
-                }
+
+                descName = UUID.randomUUID().toString();
+
                 command.add("-t", descName);
                 final String result = command.execute();
                 if (result != null) {
@@ -1041,18 +1041,7 @@ public class KVMStorageProcessor implements StorageProcessor {
             if (isCreatedFromVmSnapshot) {
                 s_logger.debug("Ignoring removal of vm snapshot on primary as this snapshot is created from vm snapshot");
             } else if (primaryPool.getType() != StoragePoolType.RBD) {
-                String snapshotPath = snapshot.getPath();
-                String backupSnapshotAfterTakingSnapshot = cmd.getOptions() == null ? null : cmd.getOptions().get(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key());
-
-                if (backupSnapshotAfterTakingSnapshot == null || BooleanUtils.toBoolean(backupSnapshotAfterTakingSnapshot)) {
-                    try {
-                        Files.deleteIfExists(Paths.get(snapshotPath));
-                    } catch (IOException ex) {
-                        s_logger.error(String.format("Failed to delete snapshot [%s] on primary storage [%s].", snapshotPath, primaryPool.getUuid()), ex);
-                    }
-                } else {
-                    s_logger.debug(String.format("This backup is temporary, not deleting snapshot [%s] on primary storage [%s]", snapshotPath, primaryPool.getUuid()));
-                }
+                deleteSnapshotOnPrimary(cmd, snapshot, primaryPool);
             }
 
             try {
@@ -1064,6 +1053,27 @@ public class KVMStorageProcessor implements StorageProcessor {
             }
         }
     }
+
+    private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObjectTO snapshot,
+            KVMStoragePool primaryPool) {
+        String snapshotPath = snapshot.getPath();
+        String backupSnapshotAfterTakingSnapshot = null;
+        boolean deleteSnapshotOnPrimary = true;
+        if (cmd.getOptions() != null) {
+            backupSnapshotAfterTakingSnapshot = cmd.getOptions().get(SnapshotInfo.BackupSnapshotAfterTakingSnapshot.key());
+            deleteSnapshotOnPrimary = cmd.getOptions().get("typeDescription") == null;
+        }
+
+        if ((backupSnapshotAfterTakingSnapshot == null || BooleanUtils.toBoolean(backupSnapshotAfterTakingSnapshot)) && deleteSnapshotOnPrimary) {
+            try {
+                Files.deleteIfExists(Paths.get(snapshotPath));
+            } catch (IOException ex) {
+                s_logger.error(String.format("Failed to delete snapshot [%s] on primary storage [%s].", snapshotPath, primaryPool.getUuid()), ex);
+            }
+        } else {
+            s_logger.debug(String.format("This backup is temporary, not deleting snapshot [%s] on primary storage [%s]", snapshotPath, primaryPool.getUuid()));
+        }
+    }
     protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params) throws
             LibvirtException, InternalErrorException {
         DiskDef iso = new DiskDef();

plugins/pom.xml

@@ -73,6 +73,9 @@
         <module>deployment-planners/user-concentrated-pod</module>
         <module>deployment-planners/user-dispersing</module>
 
+        <module>drs/cluster/balanced</module>
+        <module>drs/cluster/condensed</module>
+
         <module>event-bus/inmemory</module>
         <module>event-bus/kafka</module>
         <module>event-bus/rabbitmq</module>

server/src/main/java/com/cloud/api/ApiDBUtils.java

@@ -1533,6 +1533,10 @@ public class ApiDBUtils {
         return s_ipAddressDao.findByAssociatedVmId(vmId);
     }
 
+    public static IpAddress findIpByAssociatedVmIdAndNetworkId(long vmId, long networkId) {
+        return s_ipAddressDao.findByVmIdAndNetworkId(networkId, vmId);
+    }
+
     public static String getHaTag() {
         return s_haMgr.getHaTag();
     }
@@ -1864,6 +1868,10 @@ public class ApiDBUtils {
         return s_userVmJoinDao.newUserVmView(userVms);
     }
 
+    public static List<UserVmJoinVO> newUserVmView(VirtualMachine... vms) {
+        return s_userVmJoinDao.newUserVmView(vms);
+    }
+
     public static SecurityGroupResponse newSecurityGroupResponse(SecurityGroupJoinVO vsg, Account caller) {
         return s_securityGroupJoinDao.newSecurityGroupResponse(vsg, caller);
     }

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -3739,7 +3739,7 @@ public class ApiResponseHelper implements ResponseGenerator {
         response.setName(guestOS.getDisplayName());
         response.setDescription(guestOS.getDisplayName());
         response.setId(guestOS.getUuid());
-        response.setIsUserDefined(String.valueOf(guestOS.getIsUserDefined()));
+        response.setIsUserDefined(guestOS.getIsUserDefined());
         response.setForDisplay(guestOS.getForDisplay());
         GuestOSCategoryVO category = ApiDBUtils.findGuestOsCategoryById(guestOS.getCategoryId());
         if (category != null) {