Merge branch 'main' into disk-controller-mappings

.asf.yaml

@@ -53,13 +53,12 @@ github:
     - acs-robot
     - gpordeus
     - hsato03
-    - bernardodemarco
     - FelipeM525
     - lucas-a-martins
     - nicoschmdt
     - abh1sar
-    - sudo87
     - rosi-shapeblue
+    - sudo87
 
   protected_branches: ~
 

.github/linters/.markdown-lint.yml

@@ -86,3 +86,6 @@ MD046: false
 
 # MD052/reference-links-images Reference links and images should use a label that is defined
 MD052: false
+
+# MD059/descriptive-link-text Link text should be descriptive
+MD059: false

.github/workflows/ci.yml

@@ -89,7 +89,10 @@ jobs:
                   smoke/test_nested_virtualization
                   smoke/test_set_sourcenat
                   smoke/test_webhook_lifecycle
-                  smoke/test_purge_expunged_vms",
+                  smoke/test_purge_expunged_vms
+                  smoke/test_extension_lifecycle
+                  smoke/test_extension_custom_action_lifecycle
+                  smoke/test_extension_custom",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
@@ -137,6 +140,7 @@ jobs:
                   smoke/test_vm_deployment_planner
                   smoke/test_vm_strict_host_tags
                   smoke/test_vm_schedule
+                  smoke/test_deploy_vgpu_enabled_vm
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm

.github/workflows/ui.yml

@@ -56,6 +56,7 @@ jobs:
           npm run test:unit
 
       - uses: codecov/codecov-action@v4
+        if: github.repository == 'apache/cloudstack'
         with:
           working-directory: ui
           files: ./coverage/lcov.info

.pre-commit-config.yaml

@@ -15,18 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   # force all unspecified Python hooks to run python3
   python: python3
-minimum_pre_commit_version: "2.17.0"
+minimum_pre_commit_version: "3.2.0"
 repos:
   - repo: meta
     hooks:
       - id: identity
       - id: check-hooks-apply
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.27.2
+    hooks:
+      - id: gitleaks
+        name: run gitleaks
+        description: detect hardcoded secrets
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       #- id: check-added-large-files
       - id: check-case-conflict
@@ -69,7 +75,7 @@ repos:
         name: run codespell
         description: Check spelling with codespell
         args: [--ignore-words=.github/linters/codespell.txt]
-        exclude: ^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$
+        exclude: ^systemvm/agent/noVNC/|^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
@@ -87,7 +93,7 @@ repos:
         ^setup/bindir/cloud-setup-encryption\.in$|
         ^venv/.*$
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.43.0
+    rev: v0.45.0
     hooks:
       - id: markdownlint
         name: run markdownlint

README.md

@@ -160,3 +160,11 @@ The following provides more details on the included cryptographic software:
 * CloudStack makes use of the Bouncy Castle general-purpose encryption library.
 * CloudStack can optionally interact with and control OpenSwan-based VPNs.
 * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation.
+
+## Star History
+
+[![Apache CloudStack Star History](https://api.star-history.com/svg?repos=apache/cloudstack&type=Date)](https://www.star-history.com/#apache/cloudstack&Date)
+
+## Contributors
+
+[![Apache CloudStack Contributors](https://contrib.rocks/image?repo=apache/cloudstack&anon=0&max=500)](https://github.com/apache/cloudstack/graphs/contributors)

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -155,6 +155,14 @@ public class AgentProperties{
      */
     public static final Property<Integer> CMDS_TIMEOUT = new Property<>("cmds.timeout", 7200);
 
+    /**
+     * The timeout (in seconds) for the snapshot merge operation, mainly used for classic volume snapshots and disk-only VM snapshots on file-based storage.<br>
+     * This configuration is only considered if libvirt.events.enabled is also true. <br>
+     * Data type: Integer.<br>
+     * Default value: <code>259200</code>
+     */
+    public static final Property<Integer> QCOW2_DELTA_MERGE_TIMEOUT = new Property<>("qcow2.delta.merge.timeout", 60 * 60 * 72);
+
     /**
      * This parameter sets the VM migration speed (in mbps). The default value is -1,<br>
      * which means that the agent will try to guess the speed of the guest network and consume all possible bandwidth.<br>
@@ -213,6 +221,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.script", "libvirt-vm-xml-transformer.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM XML transformer shell script.<br>
+     * The shell script is used to execute the Libvirt VM XML transformer script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-xml-transformer.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.shell_script", "libvirt-vm-xml-transformer.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_xml_transformer.script properties to define the Libvirt VM XML transformer method.<br>
      * Libvirt XML transformer hook does XML-to-XML transformation.<br>
@@ -233,6 +250,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on start shell script.<br>
+     * The shell script is used to execute the Libvirt VM on start script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_start.script properties to define the Libvirt VM on start method.<br>
      * The hook is called right after Libvirt successfully launched the VM.<br>
@@ -252,6 +278,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on stop shell script.<br>
+     * The shell script is used to execute the Libvirt VM on stop script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_stop.script properties to define the Libvirt VM on stop method.<br>
      * The hook is called right after libvirt successfully stopped the VM.<br>
@@ -833,7 +868,7 @@ public class AgentProperties{
         private T defaultValue;
         private Class<T> typeClass;
 
-        Property(String name, T value) {
+        public Property(String name, T value) {
             init(name, value);
         }
 

api/src/main/java/com/cloud/agent/api/Command.java

@@ -19,9 +19,10 @@ package com.cloud.agent.api;
 import java.util.HashMap;
 import java.util.Map;
 
-import com.cloud.agent.api.LogLevel.Log4jLevel;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
 
 /**
  * implemented by classes that extends the Command class. Command specifies
@@ -60,6 +61,7 @@ public abstract class Command {
     private int wait;  //in second
     private boolean bypassHostMaintenance = false;
     private transient long requestSequence = 0L;
+    protected Map<String, Map<String, String>> externalDetails;
 
     protected Command() {
         this.wait = 0;
@@ -128,6 +130,14 @@ public abstract class Command {
         this.requestSequence = requestSequence;
     }
 
+    public void setExternalDetails(Map<String, Map<String, String>> externalDetails) {
+        this.externalDetails = externalDetails;
+    }
+
+    public Map<String, Map<String, String>> getExternalDetails() {
+        return externalDetails;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

api/src/main/java/com/cloud/agent/api/VgpuTypesInfo.java

@@ -15,10 +15,24 @@
 // specific language governing permissions and limitations
 // under the License.
 package com.cloud.agent.api;
+
+import org.apache.cloudstack.gpu.GpuDevice;
+
 public class VgpuTypesInfo {
 
+    private boolean passthroughEnabled = true;
+    private GpuDevice.DeviceType deviceType;
+    private String parentBusAddress;
+    private String busAddress;
+    private String numaNode;
+    private String pciRoot;
+    private String deviceId;
+    private String deviceName;
+    private String vendorId;
+    private String vendorName;
     private String modelName;
     private String groupName;
+    private String vmName;
     private Long maxHeads;
     private Long videoRam;
     private Long maxResolutionX;
@@ -26,6 +40,7 @@ public class VgpuTypesInfo {
     private Long maxVgpuPerGpu;
     private Long remainingCapacity;
     private Long maxCapacity;
+    private boolean display = false;
 
     public String getModelName() {
         return modelName;
@@ -39,22 +54,42 @@ public class VgpuTypesInfo {
         return videoRam;
     }
 
+    public void setVideoRam(Long videoRam) {
+        this.videoRam = videoRam;
+    }
+
     public Long getMaxHeads() {
         return maxHeads;
     }
 
+    public void setMaxHeads(Long maxHeads) {
+        this.maxHeads = maxHeads;
+    }
+
     public Long getMaxResolutionX() {
         return maxResolutionX;
     }
 
+    public void setMaxResolutionX(Long maxResolutionX) {
+        this.maxResolutionX = maxResolutionX;
+    }
+
     public Long getMaxResolutionY() {
         return maxResolutionY;
     }
 
+    public void setMaxResolutionY(Long maxResolutionY) {
+        this.maxResolutionY = maxResolutionY;
+    }
+
     public Long getMaxVpuPerGpu() {
         return maxVgpuPerGpu;
     }
 
+    public void setMaxVgpuPerGpu(Long maxVgpuPerGpu) {
+        this.maxVgpuPerGpu = maxVgpuPerGpu;
+    }
+
     public Long getRemainingCapacity() {
         return remainingCapacity;
     }
@@ -71,8 +106,133 @@ public class VgpuTypesInfo {
         this.maxCapacity = maxCapacity;
     }
 
-    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX, Long maxResolutionY, Long maxVgpuPerGpu,
-            Long remainingCapacity, Long maxCapacity) {
+    public boolean isPassthroughEnabled() {
+        return passthroughEnabled;
+    }
+
+    public void setPassthroughEnabled(boolean passthroughEnabled) {
+        this.passthroughEnabled = passthroughEnabled;
+    }
+
+    public GpuDevice.DeviceType getDeviceType() {
+        return deviceType;
+    }
+
+    public void setDeviceType(GpuDevice.DeviceType deviceType) {
+        this.deviceType = deviceType;
+    }
+
+    public String getParentBusAddress() {
+        return parentBusAddress;
+    }
+
+    public void setParentBusAddress(String parentBusAddress) {
+        this.parentBusAddress = parentBusAddress;
+    }
+
+    public String getBusAddress() {
+        return busAddress;
+    }
+
+    public void setBusAddress(String busAddress) {
+        this.busAddress = busAddress;
+    }
+
+    public String getNumaNode() {
+        return numaNode;
+    }
+
+    public void setNumaNode(String numaNode) {
+        this.numaNode = numaNode;
+    }
+
+    public String getPciRoot() {
+        return pciRoot;
+    }
+
+    public void setPciRoot(String pciRoot) {
+        this.pciRoot = pciRoot;
+    }
+
+    public String getDeviceId() {
+        return deviceId;
+    }
+
+    public void setDeviceId(String deviceId) {
+        this.deviceId = deviceId;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public void setDeviceName(String deviceName) {
+        this.deviceName = deviceName;
+    }
+
+    public String getVendorId() {
+        return vendorId;
+    }
+
+    public void setVendorId(String vendorId) {
+        this.vendorId = vendorId;
+    }
+
+    public String getVendorName() {
+        return vendorName;
+    }
+
+    public void setVendorName(String vendorName) {
+        this.vendorName = vendorName;
+    }
+
+    public String getVmName() {
+        return vmName;
+    }
+
+    public void setVmName(String vmName) {
+        this.vmName = vmName;
+    }
+
+    public boolean isDisplay() {
+        return display;
+    }
+
+    public void setDisplay(boolean display) {
+        this.display = display;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+            String vendorId, String vendorName, String deviceId, String deviceName, String numaNode, String pciRoot
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+        this.numaNode = numaNode;
+        this.pciRoot = pciRoot;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+                         String vendorId, String vendorName, String deviceId, String deviceName
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+    }
+
+    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX,
+            Long maxResolutionY, Long maxVgpuPerGpu, Long remainingCapacity, Long maxCapacity
+    ) {
         this.groupName = groupName;
         this.modelName = modelName;
         this.videoRam = videoRam;

api/src/main/java/com/cloud/agent/api/to/FirewallRuleTO.java

@@ -47,7 +47,7 @@ public class FirewallRuleTO implements InternalIdentity {
     int[] srcPortRange;
     boolean revoked;
     boolean alreadyAdded;
-    private List<String> sourceCidrList;
+    protected List<String> sourceCidrList;
     private List<String> destCidrList;
     FirewallRule.Purpose purpose;
     private Integer icmpType;

api/src/main/java/com/cloud/agent/api/to/GPUDeviceTO.java

@@ -16,7 +16,9 @@
 // under the License.
 package com.cloud.agent.api.to;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 import com.cloud.agent.api.VgpuTypesInfo;
 
@@ -24,9 +26,23 @@ public class GPUDeviceTO {
 
     private String gpuGroup;
     private String vgpuType;
+    private int gpuCount;
     private HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails = new HashMap<String, HashMap<String, VgpuTypesInfo>>();
+    private List<VgpuTypesInfo> gpuDevices = new ArrayList<>();
 
-    public GPUDeviceTO( String gpuGroup, String vgpuType, HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
+    public GPUDeviceTO(String gpuGroup, String vgpuType, int gpuCount,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails,
+                       List<VgpuTypesInfo> gpuDevices) {
+        this.gpuGroup = gpuGroup;
+        this.vgpuType = vgpuType;
+        this.groupDetails = groupDetails;
+        this.gpuCount = gpuCount;
+        this.gpuDevices = gpuDevices;
+
+    }
+
+    public GPUDeviceTO(String gpuGroup, String vgpuType,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
         this.gpuGroup = gpuGroup;
         this.vgpuType = vgpuType;
         this.groupDetails = groupDetails;
@@ -48,6 +64,14 @@ public class GPUDeviceTO {
         this.vgpuType = vgpuType;
     }
 
+    public int getGpuCount() {
+        return gpuCount;
+    }
+
+    public void setGpuCount(int gpuCount) {
+        this.gpuCount = gpuCount;
+    }
+
     public HashMap<String, HashMap<String, VgpuTypesInfo>> getGroupDetails() {
         return groupDetails;
     }
@@ -56,4 +80,11 @@ public class GPUDeviceTO {
         this.groupDetails = groupDetails;
     }
 
+    public List<VgpuTypesInfo> getGpuDevices() {
+        return gpuDevices;
+    }
+
+    public void setGpuDevices(List<VgpuTypesInfo> gpuDevices) {
+        this.gpuDevices = gpuDevices;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java

@@ -21,8 +21,6 @@ import com.cloud.network.rules.PortForwardingRule;
 import com.cloud.utils.net.NetUtils;
 import org.apache.commons.lang3.StringUtils;
 
-import java.util.List;
-
 /**
  * PortForwardingRuleTO specifies one port forwarding rule.
  *
@@ -32,8 +30,6 @@ public class PortForwardingRuleTO extends FirewallRuleTO {
     String dstIp;
     int[] dstPortRange;
 
-    List<String> sourceCidrList;
-
     protected PortForwardingRuleTO() {
         super();
     }

api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -19,12 +19,14 @@ package com.cloud.agent.api.to;
 import java.util.List;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.stream.Collectors;
 
 import com.cloud.agent.api.LogLevel;
 import com.cloud.network.element.NetworkElement;
 import com.cloud.template.VirtualMachineTemplate.BootloaderType;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.Type;
+import com.cloud.vm.VmDetailConstants;
 
 public class VirtualMachineTO {
     private long id;
@@ -496,4 +498,16 @@ public class VirtualMachineTO {
     public String toString() {
         return String.format("VM {id: \"%s\", name: \"%s\", uuid: \"%s\", type: \"%s\"}", id, name, uuid, type);
     }
+
+    public Map<String, String> getExternalDetails() {
+        if (details == null) {
+            return new HashMap<>();
+        }
+        return details.entrySet().stream()
+                .filter(entry -> entry.getKey().startsWith(VmDetailConstants.EXTERNAL_DETAIL_PREFIX))
+                .collect(Collectors.toMap(
+                        entry -> entry.getKey().substring(VmDetailConstants.EXTERNAL_DETAIL_PREFIX.length()),
+                        Map.Entry::getValue
+                ));
+    }
 }

api/src/main/java/com/cloud/configuration/ConfigurationService.java

@@ -17,7 +17,11 @@
 package com.cloud.configuration;
 
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 
+import com.cloud.network.Network;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.config.ResetCfgCmd;
 import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
 import org.apache.cloudstack.api.command.admin.network.CreateGuestNetworkIpv6PrefixCmd;
@@ -373,4 +377,16 @@ public interface ConfigurationService {
     List<? extends PortableIp> listPortableIps(long id);
 
     Boolean isAccountAllowedToCreateOfferingsWithTags(IsAccountAllowedToCreateOfferingsWithTagsCmd cmd);
+
+    public static final Map<String, String> ProviderDetailKeyMap = Map.of(
+            Network.Provider.Nsx.getName(), ApiConstants.NSX_DETAIL_KEY,
+            Network.Provider.Netris.getName(), ApiConstants.NETRIS_DETAIL_KEY
+    );
+
+    public static boolean IsIpRangeForProvider(Network.Provider provider) {
+        if (Objects.isNull(provider)) {
+            return false;
+        }
+        return ProviderDetailKeyMap.containsKey(provider.getName());
+    }
 }

api/src/main/java/com/cloud/configuration/Resource.java

@@ -37,7 +37,8 @@ public interface Resource {
         backup("backup", 12),
         backup_storage("backup_storage", 13),
         bucket("bucket", 14),
-        object_storage("object_storage", 15);
+        object_storage("object_storage", 15),
+        gpu("gpu", 16);
 
         private String name;
         private int ordinal;

api/src/main/java/com/cloud/event/EventTypes.java

@@ -29,13 +29,18 @@ import org.apache.cloudstack.api.response.PodResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.config.Configuration;
 import org.apache.cloudstack.datacenter.DataCenterIpv4GuestSubnet;
+import org.apache.cloudstack.extension.Extension;
+import org.apache.cloudstack.extension.ExtensionCustomAction;
+import org.apache.cloudstack.gpu.GpuCard;
+import org.apache.cloudstack.gpu.GpuDevice;
+import org.apache.cloudstack.gpu.VgpuProfile;
 import org.apache.cloudstack.ha.HAConfig;
 import org.apache.cloudstack.network.BgpPeer;
 import org.apache.cloudstack.network.Ipv4GuestSubnetNetworkMap;
 import org.apache.cloudstack.quota.QuotaTariff;
-import org.apache.cloudstack.storage.sharedfs.SharedFS;
 import org.apache.cloudstack.storage.object.Bucket;
 import org.apache.cloudstack.storage.object.ObjectStore;
+import org.apache.cloudstack.storage.sharedfs.SharedFS;
 import org.apache.cloudstack.usage.Usage;
 import org.apache.cloudstack.vm.schedule.VMSchedule;
 
@@ -376,6 +381,21 @@ public class EventTypes {
     public static final String EVENT_DISK_OFFERING_EDIT = "DISK.OFFERING.EDIT";
     public static final String EVENT_DISK_OFFERING_DELETE = "DISK.OFFERING.DELETE";
 
+    // GPU Cards
+    public static final String EVENT_GPU_CARD_CREATE = "GPU.CARD.CREATE";
+    public static final String EVENT_GPU_CARD_EDIT = "GPU.CARD.EDIT";
+    public static final String EVENT_GPU_CARD_DELETE = "GPU.CARD.DELETE";
+
+    // vGPU Profile
+    public static final String EVENT_VGPU_PROFILE_CREATE = "VGPU.PROFILE.CREATE";
+    public static final String EVENT_VGPU_PROFILE_EDIT = "VGPU.PROFILE.EDIT";
+    public static final String EVENT_VGPU_PROFILE_DELETE = "VGPU.PROFILE.DELETE";
+
+    // GPU Devices
+    public static final String EVENT_GPU_DEVICE_CREATE = "GPU.DEVICE.CREATE";
+    public static final String EVENT_GPU_DEVICE_EDIT = "GPU.DEVICE.EDIT";
+    public static final String EVENT_GPU_DEVICE_DELETE = "GPU.DEVICE.DELETE";
+
     // Network offerings
     public static final String EVENT_NETWORK_OFFERING_CREATE = "NETWORK.OFFERING.CREATE";
     public static final String EVENT_NETWORK_OFFERING_ASSIGN = "NETWORK.OFFERING.ASSIGN";
@@ -499,6 +519,8 @@ public class EventTypes {
 
     public static final String EVENT_ZONE_VLAN_ASSIGN = "ZONE.VLAN.ASSIGN";
     public static final String EVENT_ZONE_VLAN_RELEASE = "ZONE.VLAN.RELEASE";
+    public static final String EVENT_ZONE_VXLAN_ASSIGN = "ZONE.VXLAN.ASSIGN";
+    public static final String EVENT_ZONE_VXLAN_RELEASE = "ZONE.VXLAN.RELEASE";
 
     // Projects
     public static final String EVENT_PROJECT_CREATE = "PROJECT.CREATE";
@@ -804,11 +826,30 @@ public class EventTypes {
     // Management Server
     public static final String EVENT_MANAGEMENT_SERVER_REMOVE = "MANAGEMENT.SERVER.REMOVE";
 
+    // VM Lease
     public static final String VM_LEASE_EXPIRED = "VM.LEASE.EXPIRED";
     public static final String VM_LEASE_DISABLED = "VM.LEASE.DISABLED";
     public static final String VM_LEASE_CANCELLED = "VM.LEASE.CANCELLED";
     public static final String VM_LEASE_EXPIRING = "VM.LEASE.EXPIRING";
 
+    // GUI Theme
+    public static final String EVENT_GUI_THEME_CREATE = "GUI.THEME.CREATE";
+    public static final String EVENT_GUI_THEME_REMOVE = "GUI.THEME.REMOVE";
+    public static final String EVENT_GUI_THEME_UPDATE = "GUI.THEME.UPDATE";
+
+    // Extension
+    public static final String EVENT_EXTENSION_CREATE = "EXTENSION.CREATE";
+    public static final String EVENT_EXTENSION_UPDATE = "EXTENSION.UPDATE";
+    public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE";
+    public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER";
+    public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_UPDATE = "EXTENSION.CUSTOM.ACTION.UPDATE";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_DELETE = "EXTENSION.CUSTOM.ACTION.DELETE";
+
+    // Custom Action
+    public static final String EVENT_CUSTOM_ACTION = "CUSTOM.ACTION";
+
     static {
 
         // TODO: need a way to force author adding event types to declare the entity details as well, with out braking
@@ -1003,6 +1044,21 @@ public class EventTypes {
         entityEventDetails.put(EVENT_DISK_OFFERING_EDIT, DiskOffering.class);
         entityEventDetails.put(EVENT_DISK_OFFERING_DELETE, DiskOffering.class);
 
+        // GPU Cards
+        entityEventDetails.put(EVENT_GPU_CARD_CREATE, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_EDIT, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_DELETE, GpuCard.class);
+
+        // vGPU Profiles
+        entityEventDetails.put(EVENT_VGPU_PROFILE_CREATE, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_EDIT, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_DELETE, VgpuProfile.class);
+
+        // GPU Devices
+        entityEventDetails.put(EVENT_GPU_DEVICE_CREATE, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_EDIT, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_DELETE, GpuDevice.class);
+
         // Network offerings
         entityEventDetails.put(EVENT_NETWORK_OFFERING_CREATE, NetworkOffering.class);
         entityEventDetails.put(EVENT_NETWORK_OFFERING_ASSIGN, NetworkOffering.class);
@@ -1312,6 +1368,21 @@ public class EventTypes {
         entityEventDetails.put(VM_LEASE_EXPIRING, VirtualMachine.class);
         entityEventDetails.put(VM_LEASE_DISABLED, VirtualMachine.class);
         entityEventDetails.put(VM_LEASE_CANCELLED, VirtualMachine.class);
+
+        // GUI theme
+        entityEventDetails.put(EVENT_GUI_THEME_CREATE, "GuiTheme");
+        entityEventDetails.put(EVENT_GUI_THEME_REMOVE, "GuiTheme");
+        entityEventDetails.put(EVENT_GUI_THEME_UPDATE, "GuiTheme");
+
+        // Extension
+        entityEventDetails.put(EVENT_EXTENSION_CREATE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_UPDATE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_DELETE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_RESOURCE_REGISTER, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_RESOURCE_UNREGISTER, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_UPDATE, ExtensionCustomAction.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_DELETE, ExtensionCustomAction.class);
     }
 
     public static boolean isNetworkEvent(String eventType) {

api/src/main/java/com/cloud/hypervisor/Hypervisor.java

@@ -54,6 +54,7 @@ public class Hypervisor {
         public static final HypervisorType Ovm3 = new HypervisorType("Ovm3", ImageFormat.RAW);
         public static final HypervisorType LXC = new HypervisorType("LXC");
         public static final HypervisorType Custom = new HypervisorType("Custom", null, EnumSet.of(RootDiskSizeOverride));
+        public static final HypervisorType External = new HypervisorType("External", null, EnumSet.of(RootDiskSizeOverride));
         public static final HypervisorType Any = new HypervisorType("Any"); /*If you don't care about the hypervisor type*/
         private final String name;
         private final ImageFormat imageFormat;

api/src/main/java/com/cloud/network/IpAddress.java

@@ -99,4 +99,5 @@ public interface IpAddress extends ControlledEntity, Identity, InternalIdentity,
 
     boolean isForSystemVms();
 
+    boolean isForRouter();
 }

api/src/main/java/com/cloud/network/Network.java

@@ -206,6 +206,7 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
         public static final Provider Tungsten = new Provider("Tungsten", false);
 
         public static final Provider Nsx = new Provider("Nsx", false);
+        public static final Provider Netris = new Provider("Netris", false);
 
         private final String name;
         private final boolean isExternal;

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -305,6 +305,8 @@ public interface NetworkModel {
 
     NicProfile getNicProfile(VirtualMachine vm, long networkId, String broadcastUri);
 
+    NicProfile getNicProfile(VirtualMachine vm, Nic nic, DataCenter dataCenter);
+
     Set<Long> getAvailableIps(Network network, String requestedIp);
 
     String getDomainNetworkDomain(long domainId, long zoneId);

api/src/main/java/com/cloud/network/NetworkService.java

@@ -19,7 +19,6 @@ package com.cloud.network;
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.dc.DataCenter;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin;
 import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd;
@@ -39,13 +38,16 @@ import org.apache.cloudstack.api.command.user.network.UpdateNetworkCmd;
 import org.apache.cloudstack.api.command.user.vm.ListNicsCmd;
 import org.apache.cloudstack.api.response.AcquirePodIpCmdResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
 
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.dc.DataCenter;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.network.Network.IpAddresses;
 import com.cloud.network.Network.Service;
 import com.cloud.network.Networks.TrafficType;
@@ -57,7 +59,6 @@ import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicSecondaryIp;
-import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
 
 /**
  * The NetworkService interface is the "public" api to entities that make requests to the orchestration engine
@@ -270,4 +271,6 @@ public interface NetworkService {
     List<InternalLoadBalancerElementService> getInternalLoadBalancerElements();
 
     boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException;
+
+    String getNicVlanValueForExternalVm(NicTO nic);
 }

api/src/main/java/com/cloud/network/Networks.java

@@ -129,7 +129,8 @@ public class Networks {
         UnDecided(null, null),
         OpenDaylight("opendaylight", String.class),
         TUNGSTEN("tf", String.class),
-        NSX("nsx", String.class);
+        NSX("nsx", String.class),
+        Netris("netris", String.class);
 
         private final String scheme;
         private final Class<?> type;

api/src/main/java/com/cloud/network/SDNProviderNetworkRule.java

@@ -0,0 +1,358 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network;
+
+import java.util.List;
+
+public class SDNProviderNetworkRule {
+
+    protected long domainId;
+    protected long accountId;
+    protected long zoneId;
+    protected Long networkResourceId;
+    protected String networkResourceName;
+    protected boolean isVpcResource;
+    protected long vmId;
+    protected long ruleId;
+    protected String publicIp;
+    protected String vmIp;
+    protected String publicPort;
+    protected String privatePort;
+    protected String protocol;
+    protected String algorithm;
+    protected List<String> sourceCidrList;
+    protected List<String> destinationCidrList;
+    protected Integer icmpCode;
+
+    protected Integer icmpType;
+    protected String trafficType;
+    protected Network.Service service;
+
+    public long getDomainId() {
+        return domainId;
+    }
+
+    public void setDomainId(long domainId) {
+        this.domainId = domainId;
+    }
+
+    public long getAccountId() {
+        return accountId;
+    }
+
+    public void setAccountId(long accountId) {
+        this.accountId = accountId;
+    }
+
+    public long getZoneId() {
+        return zoneId;
+    }
+
+    public void setZoneId(long zoneId) {
+        this.zoneId = zoneId;
+    }
+
+    public Long getNetworkResourceId() {
+        return networkResourceId;
+    }
+
+    public void setNetworkResourceId(Long networkResourceId) {
+        this.networkResourceId = networkResourceId;
+    }
+
+    public String getNetworkResourceName() {
+        return networkResourceName;
+    }
+
+    public void setNetworkResourceName(String networkResourceName) {
+        this.networkResourceName = networkResourceName;
+    }
+
+    public boolean isVpcResource() {
+        return isVpcResource;
+    }
+
+    public void setVpcResource(boolean vpcResource) {
+        isVpcResource = vpcResource;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public void setVmId(long vmId) {
+        this.vmId = vmId;
+    }
+
+    public long getRuleId() {
+        return ruleId;
+    }
+
+    public void setRuleId(long ruleId) {
+        this.ruleId = ruleId;
+    }
+
+    public String getPublicIp() {
+        return publicIp;
+    }
+
+    public void setPublicIp(String publicIp) {
+        this.publicIp = publicIp;
+    }
+
+    public String getVmIp() {
+        return vmIp;
+    }
+
+    public void setVmIp(String vmIp) {
+        this.vmIp = vmIp;
+    }
+
+    public String getPublicPort() {
+        return publicPort;
+    }
+
+    public void setPublicPort(String publicPort) {
+        this.publicPort = publicPort;
+    }
+
+    public String getPrivatePort() {
+        return privatePort;
+    }
+
+    public void setPrivatePort(String privatePort) {
+        this.privatePort = privatePort;
+    }
+
+    public String getProtocol() {
+        return protocol;
+    }
+
+    public void setProtocol(String protocol) {
+        this.protocol = protocol;
+    }
+
+    public void setAlgorithm(String algorithm) {
+        this.algorithm = algorithm;
+    }
+
+    public String getAlgorithm() {
+        return algorithm;
+    }
+
+    public Network.Service getService() {
+        return service;
+    }
+
+    public void setService(Network.Service service) {
+        this.service = service;
+    }
+
+    public Integer getIcmpCode() {
+        return icmpCode;
+    }
+
+    public void setIcmpCode(Integer icmpCode) {
+        this.icmpCode = icmpCode;
+    }
+
+    public Integer getIcmpType() {
+        return icmpType;
+    }
+
+    public void setIcmpType(Integer icmpType) {
+        this.icmpType = icmpType;
+    }
+
+    public List<String> getSourceCidrList() {
+        return sourceCidrList;
+    }
+
+    public void setSourceCidrList(List<String> sourceCidrList) {
+        this.sourceCidrList = sourceCidrList;
+    }
+
+    public List<String> getDestinationCidrList() {
+        return destinationCidrList;
+    }
+
+    public void setDestinationCidrList(List<String> destinationCidrList) {
+        this.destinationCidrList = destinationCidrList;
+    }
+
+    public String getTrafficType() {
+        return trafficType;
+    }
+
+    public void setTrafficType(String trafficType) {
+        this.trafficType = trafficType;
+    }
+
+    public static class Builder {
+        public long domainId;
+        public long accountId;
+        public long zoneId;
+        public Long networkResourceId;
+        public String networkResourceName;
+        public boolean isVpcResource;
+        public long vmId;
+
+        public long ruleId;
+        public String publicIp;
+        public String vmIp;
+        public String publicPort;
+        public String privatePort;
+        public String protocol;
+        public String algorithm;
+        public List<String> sourceCidrList;
+        public List<String> destinationCidrList;
+        public String trafficType;
+        public Integer icmpType;
+        public Integer icmpCode;
+        public Network.Service service;
+
+        public Builder() {
+            // Default constructor
+        }
+
+        public Builder setDomainId(long domainId) {
+            this.domainId = domainId;
+            return this;
+        }
+
+        public Builder setAccountId(long accountId) {
+            this.accountId = accountId;
+            return this;
+        }
+
+        public Builder setZoneId(long zoneId) {
+            this.zoneId = zoneId;
+            return this;
+        }
+
+        public Builder setNetworkResourceId(Long networkResourceId) {
+            this.networkResourceId = networkResourceId;
+            return this;
+        }
+
+        public Builder setNetworkResourceName(String networkResourceName) {
+            this.networkResourceName = networkResourceName;
+            return this;
+        }
+
+        public Builder setVpcResource(boolean isVpcResource) {
+            this.isVpcResource = isVpcResource;
+            return this;
+        }
+
+
+        public Builder setVmId(long vmId) {
+            this.vmId = vmId;
+            return this;
+        }
+
+        public Builder setRuleId(long ruleId) {
+            this.ruleId = ruleId;
+            return this;
+        }
+
+        public Builder setPublicIp(String publicIp) {
+            this.publicIp = publicIp;
+            return this;
+        }
+
+        public Builder setVmIp(String vmIp) {
+            this.vmIp = vmIp;
+            return this;
+        }
+
+        public Builder setPublicPort(String publicPort) {
+            this.publicPort = publicPort;
+            return this;
+        }
+
+        public Builder setPrivatePort(String privatePort) {
+            this.privatePort = privatePort;
+            return this;
+        }
+
+        public Builder setProtocol(String protocol) {
+            this.protocol = protocol;
+            return this;
+        }
+
+        public Builder setAlgorithm(String algorithm) {
+            this.algorithm = algorithm;
+            return this;
+        }
+
+        public Builder setTrafficType(String trafficType) {
+            this.trafficType = trafficType;
+            return this;
+        }
+
+        public Builder setIcmpType(Integer icmpType) {
+            this.icmpType = icmpType;
+            return this;
+        }
+
+        public Builder setIcmpCode(Integer icmpCode) {
+            this.icmpCode = icmpCode;
+            return this;
+        }
+
+        public Builder setSourceCidrList(List<String> sourceCidrList) {
+            this.sourceCidrList = sourceCidrList;
+            return this;
+        }
+
+        public Builder setDestinationCidrList(List<String> destinationCidrList) {
+            this.destinationCidrList = destinationCidrList;
+            return this;
+        }
+
+        public Builder setService(Network.Service service) {
+            this.service = service;
+            return this;
+        }
+
+        public SDNProviderNetworkRule build() {
+            SDNProviderNetworkRule rule = new SDNProviderNetworkRule();
+            rule.setDomainId(this.domainId);
+            rule.setAccountId(this.accountId);
+            rule.setZoneId(this.zoneId);
+            rule.setNetworkResourceId(this.networkResourceId);
+            rule.setNetworkResourceName(this.networkResourceName);
+            rule.setVpcResource(this.isVpcResource);
+            rule.setVmId(this.vmId);
+            rule.setVmIp(this.vmIp);
+            rule.setPublicIp(this.publicIp);
+            rule.setPublicPort(this.publicPort);
+            rule.setPrivatePort(this.privatePort);
+            rule.setProtocol(this.protocol);
+            rule.setRuleId(this.ruleId);
+            rule.setAlgorithm(this.algorithm);
+            rule.setIcmpType(this.icmpType);
+            rule.setIcmpCode(this.icmpCode);
+            rule.setSourceCidrList(this.sourceCidrList);
+            rule.setDestinationCidrList(this.destinationCidrList);
+            rule.setTrafficType(this.trafficType);
+            rule.setService(service);
+            return rule;
+        }
+    }
+}

api/src/main/java/com/cloud/network/Site2SiteVpnConnection.java

@@ -24,7 +24,7 @@ import org.apache.cloudstack.api.InternalIdentity;
 
 public interface Site2SiteVpnConnection extends ControlledEntity, InternalIdentity, Displayable {
     enum State {
-        Pending, Connecting, Connected, Disconnected, Error,
+        Pending, Connecting, Connected, Disconnected, Error, Removed
     }
 
     @Override

api/src/main/java/com/cloud/network/element/NetworkElement.java

@@ -23,6 +23,7 @@ import com.cloud.deploy.DeployDestination;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
 import com.cloud.network.Network.Capability;
 import com.cloud.network.Network.Provider;
@@ -87,6 +88,14 @@ public interface NetworkElement extends Adapter {
     boolean release(Network network, NicProfile nic, VirtualMachineProfile vm, ReservationContext context) throws ConcurrentOperationException,
         ResourceUnavailableException;
 
+    /**
+     * Release IP from the network provider if reserved
+     * @param ipAddress
+     */
+    default boolean releaseIp(IpAddress ipAddress) {
+        return true;
+    }
+
     /**
      * The network is being shutdown.
      * @param network

api/src/main/java/com/cloud/network/element/PortForwardingServiceProvider.java

@@ -17,12 +17,40 @@
 package com.cloud.network.element;
 
 import java.util.List;
+import java.util.Objects;
 
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.Network;
+import com.cloud.network.rules.FirewallRule;
 import com.cloud.network.rules.PortForwardingRule;
+import com.cloud.network.vpc.NetworkACLItem;
 
 public interface PortForwardingServiceProvider extends NetworkElement, IpDeployingRequester {
+
+    static String getPublicPortRange(PortForwardingRule rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
+    static String getPrivatePFPortRange(PortForwardingRule rule) {
+        return rule.getDestinationPortStart() == rule.getDestinationPortEnd() ?
+                String.valueOf(rule.getDestinationPortStart()) :
+                String.valueOf(rule.getDestinationPortStart()).concat("-").concat(String.valueOf(rule.getDestinationPortEnd()));
+    }
+
+    static String getPrivatePortRange(FirewallRule rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
+    static String getPrivatePortRangeForACLRule(NetworkACLItem rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
     /**
      * Apply rules
      * @param network

api/src/main/java/com/cloud/network/element/VpcProvider.java

@@ -55,4 +55,8 @@ public interface VpcProvider extends NetworkElement {
     boolean applyACLItemsToPrivateGw(PrivateGateway gateway, List<? extends NetworkACLItem> rules) throws ResourceUnavailableException;
 
     boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+
+    default boolean updateVpc(Vpc vpc, String previousVpcName) {
+        return true;
+    }
 }

api/src/main/java/com/cloud/network/guru/NetworkGuru.java

@@ -215,4 +215,8 @@ public interface NetworkGuru extends Adapter {
     default boolean isSlaacV6Only() {
         return true;
     }
+
+    default boolean update(Network network, String prevNetworkName) {
+        return true;
+    }
 }

api/src/main/java/com/cloud/network/netris/NetrisLbBackend.java

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+public class NetrisLbBackend {
+    private long vmId;
+    private String vmIp;
+    private int port;
+
+    public NetrisLbBackend(long vmId, String vmIp, int port) {
+        this.vmId = vmId;
+        this.vmIp = vmIp;
+        this.port = port;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public String getVmIp() {
+        return vmIp;
+    }
+
+    public int getPort() {
+        return port;
+    }
+}

api/src/main/java/com/cloud/network/netris/NetrisNetworkRule.java

@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import com.cloud.network.SDNProviderNetworkRule;
+
+
+import java.util.List;
+
+public class NetrisNetworkRule {
+    public enum NetrisRuleAction {
+        PERMIT, DENY
+    }
+
+    private SDNProviderNetworkRule baseRule;
+    private NetrisRuleAction aclAction;
+    private List<NetrisLbBackend> lbBackends;
+    private String lbRuleName;
+    private String lbCidrList;
+    private String reason;
+
+    public NetrisNetworkRule(Builder builder) {
+        this.baseRule = builder.baseRule;
+        this.aclAction = builder.aclAction;
+        this.lbBackends = builder.lbBackends;
+        this.reason = builder.reason;
+        this.lbCidrList = builder.lbCidrList;
+        this.lbRuleName = builder.lbRuleName;
+    }
+
+    public NetrisRuleAction getAclAction() {
+        return aclAction;
+    }
+
+    public List<NetrisLbBackend> getLbBackends() {
+        return lbBackends;
+    }
+
+    public String getReason() {
+        return reason;
+    }
+
+    public String getLbCidrList() {return lbCidrList; }
+
+    public String getLbRuleName() { return lbRuleName; }
+
+    public SDNProviderNetworkRule getBaseRule() {
+        return baseRule;
+    }
+
+    // Builder class extending the parent builder
+    public static class Builder {
+        private SDNProviderNetworkRule baseRule;
+        private NetrisRuleAction aclAction;
+        private List<NetrisLbBackend> lbBackends;
+        private String reason;
+        private String lbCidrList;
+        private String lbRuleName;
+
+        public Builder baseRule(SDNProviderNetworkRule baseRule) {
+            this.baseRule = baseRule;
+            return this;
+        }
+
+        public Builder aclAction(NetrisRuleAction aclAction) {
+            this.aclAction = aclAction;
+            return this;
+        }
+
+        public Builder lbBackends(List<NetrisLbBackend> lbBackends) {
+            this.lbBackends = lbBackends;
+            return this;
+        }
+
+        public Builder reason(String reason) {
+            this.reason = reason;
+            return this;
+        }
+
+        public Builder lbCidrList(String lbCidrList) {
+            this.lbCidrList = lbCidrList;
+            return this;
+        }
+
+        public Builder lbRuleName(String lbRuleName) {
+            this.lbRuleName = lbRuleName;
+            return this;
+        }
+
+        public NetrisNetworkRule build() {
+            return new NetrisNetworkRule(this);
+        }
+    }
+}

api/src/main/java/com/cloud/network/netris/NetrisProvider.java

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface NetrisProvider extends InternalIdentity, Identity {
+    long getZoneId();
+    String getName();
+    String getUrl();
+    String getUsername();
+    String getSiteName();
+    String getTenantName();
+    String getNetrisTag();
+}

api/src/main/java/com/cloud/network/netris/NetrisService.java

@@ -0,0 +1,310 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import com.cloud.network.IpAddress;
+import com.cloud.network.Network;
+import com.cloud.network.SDNProviderNetworkRule;
+import com.cloud.network.vpc.StaticRoute;
+import com.cloud.network.vpc.Vpc;
+
+import java.util.List;
+
+/**
+ * Interface for Netris Services that provides methods to manage VPCs, networks,
+ * NAT rules, network rules, and static routes in an SDN (Software Defined Networking) environment.
+ */
+
+public interface NetrisService {
+
+    /**
+     * Creates IPAM (IP Address Management) allocations for zone-level public ranges.
+     *
+     * @param zoneId the ID of the zone
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createIPAMAllocationsForZoneLevelPublicRanges(long zoneId);
+
+    /**
+     * Creates a VPC (Virtual Private Cloud) resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcId            the ID of the VPC
+     * @param vpcName          the name of the VPC
+     * @param sourceNatEnabled true if source NAT is enabled
+     * @param cidr             the CIDR of the VPC
+     * @param isVpcNetwork     true if it is a VPC network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createVpcResource(long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled, String cidr, boolean isVpcNetwork);
+
+    /**
+     * Updates an existing VPC resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcId            the ID of the VPC
+     * @param vpcName          the new name of the VPC
+     * @param previousVpcName  the previous name of the VPC
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVpcResource(long zoneId, long accountId, long domainId, Long vpcId, String vpcName, String previousVpcName);
+
+    /**
+     * Deletes a VPC resource.
+     *
+     * @param zoneId   the ID of the zone
+     * @param accountId the ID of the account
+     * @param domainId the ID of the domain
+     * @param vpc      the VPC to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteVpcResource(long zoneId, long accountId, long domainId, Vpc vpc);
+
+    /**
+     * Creates a virtual network (vNet) resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcName          the name of the VPC
+     * @param vpcId            the ID of the VPC
+     * @param networkName      the name of the network
+     * @param networkId        the ID of the network
+     * @param cidr             the CIDR of the network
+     * @param globalRouting    true if global routing is enabled
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr, Boolean globalRouting);
+
+    /**
+     * Updates an existing vNet resource.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the new name of the network
+     * @param networkId       the ID of the network
+     * @param prevNetworkName the previous name of the network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String prevNetworkName);
+
+    /**
+     * Deletes an existing vNet resource.
+     *
+     * @param zoneId       the ID of the zone
+     * @param accountId    the ID of the account
+     * @param domainId     the ID of the domain
+     * @param vpcName      the name of the VPC
+     * @param vpcId        the ID of the VPC
+     * @param networkName  the name of the network
+     * @param networkId    the ID of the network
+     * @param cidr         the CIDR of the network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteVnetResource(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr);
+
+    /**
+     * Creates a source NAT rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param sourceNatIp     the source NAT IP
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createSnatRule(long zoneId, long accountId, long domainId, String vpcName, long vpcId, String networkName, long networkId, boolean isForVpc, String vpcCidr, String sourceNatIp);
+
+    /**
+     * Creates a port forwarding rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param networkRule     the network rule to forward
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createPortForwardingRule(long zoneId, long accountId, long domainId, String vpcName, long vpcId, String networkName, Long networkId, boolean isForVpc, String vpcCidr, SDNProviderNetworkRule networkRule);
+
+    /**
+     * Deletes a port forwarding rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param networkRule     the network rule to remove
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deletePortForwardingRule(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, boolean isForVpc, String vpcCidr, SDNProviderNetworkRule networkRule);
+
+    /**
+     * Updates the source NAT IP for a specified VPC.
+     *
+     * @param vpc     the VPC to updates
+     * @param address the new source NAT IP address
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+
+    /**
+     * Creates a static NAT rule for a specific VM.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if the rule applies to a VPC
+     * @param vpcCidr            the VPC CIDR
+     * @param staticNatIp        the static NAT IP
+     * @param vmIp               the VM's IP address
+     * @param vmId               the ID of the VM
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createStaticNatRule(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String vpcCidr, String staticNatIp, String vmIp, long vmId);
+
+    /**
+     * Deletes a static NAT rule for a specific VM.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if the rule applies to a VPC
+     * @param staticNatIp        the static NAT IP
+     * @param vmId               the ID of the VM
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteStaticNatRule(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String staticNatIp, long vmId);
+
+    /**
+     * Adds firewall rules to a specific network.
+     *
+     * @param network       the target network
+     * @param firewallRules the list of firewall rules to add
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean addFirewallRules(Network network, List<NetrisNetworkRule> firewallRules);
+
+    /**
+     * Deletes firewall rules from a specific network.
+     *
+     * @param network       the target network
+     * @param firewallRules the list of firewall rules to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteFirewallRules(Network network, List<NetrisNetworkRule> firewallRules);
+
+    /**
+     * Adds or updates a static route for a specific network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @param updateRoute        true if the route should be updated
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean addOrUpdateStaticRoute(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId, boolean updateRoute);
+
+    /**
+     * Deletes a specific static route for a network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteStaticRoute(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId);
+
+    /**
+     * Lists static routes for a specific network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @return a list of static routes
+     */
+    List<StaticRoute> listStaticRoutes(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId);
+
+    /**
+     * Releases a NAT IP address.
+     *
+     * @param zoneId   the ID of the zone
+     * @param publicIp the public NAT IP to release
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean releaseNatIp(long zoneId, String publicIp);
+
+    /**
+     * Creates or updates a load balancer (LB) rule.
+     *
+     * @param rule the network rule for the load balancer
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createOrUpdateLbRule(NetrisNetworkRule rule);
+
+    /**
+     * Deletes a load balancer (LB) rule.
+     *
+     * @param rule the network rule to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteLbRule(NetrisNetworkRule rule);
+}

api/src/main/java/com/cloud/network/nsx/NsxService.java

@@ -16,9 +16,10 @@
 // under the License.
 package com.cloud.network.nsx;
 
+import org.apache.cloudstack.framework.config.ConfigKey;
+
 import com.cloud.network.IpAddress;
 import com.cloud.network.vpc.Vpc;
-import org.apache.cloudstack.framework.config.ConfigKey;
 
 public interface NsxService {
 
@@ -33,4 +34,5 @@ public interface NsxService {
 
     boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled);
     boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+    String getSegmentId(long domainId, long accountId, long zoneId, Long vpcId, long networkId);
 }

api/src/main/java/com/cloud/network/vpc/StaticRoute.java

@@ -25,6 +25,7 @@ public interface StaticRoute extends ControlledEntity, Identity, InternalIdentit
         Staged, // route been created but has never got through network rule conflict detection.  Routes in this state can not be sent to VPC virtual router.
         Add,    // Add means the route has been created and has gone through network rule conflict detection.
         Active, // Route has been sent to the VPC router and reported to be active.
+        Update,
         Revoke,  // Revoke means this route has been revoked. If this route has been sent to the VPC router, the route will be deleted from database.
         Deleting // rule has been revoked and is scheduled for deletion
     }
@@ -32,7 +33,9 @@ public interface StaticRoute extends ControlledEntity, Identity, InternalIdentit
     /**
      * @return
      */
-    long getVpcGatewayId();
+    Long getVpcGatewayId();
+
+    String getNextHop();
 
     /**
      * @return

api/src/main/java/com/cloud/network/vpc/StaticRouteProfile.java

@@ -23,7 +23,8 @@ public class StaticRouteProfile implements StaticRoute {
     private String targetCidr;
     private long accountId;
     private long domainId;
-    private long gatewayId;
+    private Long gatewayId;
+    private String nextHop;
     private StaticRoute.State state;
     private long vpcId;
     String vlanTag;
@@ -46,6 +47,18 @@ public class StaticRouteProfile implements StaticRoute {
         ipAddress = gateway.getIp4Address();
     }
 
+    public StaticRouteProfile(StaticRoute staticRoute) {
+        id = staticRoute.getId();
+        uuid = staticRoute.getUuid();
+        targetCidr = staticRoute.getCidr();
+        accountId = staticRoute.getAccountId();
+        domainId = staticRoute.getDomainId();
+        gatewayId = staticRoute.getVpcGatewayId();
+        state = staticRoute.getState();
+        vpcId = staticRoute.getVpcId();
+        gateway = staticRoute.getNextHop();
+    }
+
     @Override
     public long getAccountId() {
         return accountId;
@@ -57,10 +70,15 @@ public class StaticRouteProfile implements StaticRoute {
     }
 
     @Override
-    public long getVpcGatewayId() {
+    public Long getVpcGatewayId() {
         return gatewayId;
     }
 
+    @Override
+    public String getNextHop() {
+        return nextHop;
+    }
+
     @Override
     public String getCidr() {
         return targetCidr;

api/src/main/java/com/cloud/network/vpc/VpcOffering.java

@@ -32,6 +32,8 @@ public interface VpcOffering extends InternalIdentity, Identity {
     public static final String redundantVPCOfferingName = "Redundant VPC offering";
     public static final String DEFAULT_VPC_NAT_NSX_OFFERING_NAME = "VPC offering with NSX - NAT Mode";
     public static final String DEFAULT_VPC_ROUTE_NSX_OFFERING_NAME = "VPC offering with NSX - Route Mode";
+    public static final String DEFAULT_VPC_ROUTE_NETRIS_OFFERING_NAME = "VPC offering with Netris - Route Mode";
+    public static final String DEFAULT_VPC_NAT_NETRIS_OFFERING_NAME = "VPC offering with Netris - NAT Mode";
 
     /**
      *
@@ -56,8 +58,6 @@ public interface VpcOffering extends InternalIdentity, Identity {
      */
     boolean isDefault();
 
-    boolean isForNsx();
-
     NetworkOffering.NetworkMode getNetworkMode();
 
     /**

api/src/main/java/com/cloud/network/vpc/VpcProvisioningService.java

@@ -37,7 +37,7 @@ public interface VpcProvisioningService {
     VpcOffering createVpcOffering(String name, String displayText, List<String> supportedServices,
                                   Map<String, List<String>> serviceProviders,
                                   Map serviceCapabilitystList, NetUtils.InternetProtocol internetProtocol,
-                                  Long serviceOfferingId, Boolean forNsx, NetworkOffering.NetworkMode networkMode,
+                                  Long serviceOfferingId, String externalProvider, NetworkOffering.NetworkMode networkMode,
                                   List<Long> domainIds, List<Long> zoneIds, VpcOffering.State state,
                                   NetworkOffering.RoutingMode routingMode, boolean specifyAsNumber);
 

api/src/main/java/com/cloud/network/vpc/VpcService.java

@@ -238,7 +238,7 @@ public interface VpcService {
      * @param cidr
      * @return
      */
-    StaticRoute createStaticRoute(long gatewayId, String cidr) throws NetworkRuleConflictException;
+    StaticRoute createStaticRoute(Long gatewayId, Long vpcId, String nextHop, String cidr) throws NetworkRuleConflictException;
 
     /**
      * Lists static routes based on parameters passed to the call

api/src/main/java/com/cloud/offering/NetworkOffering.java

@@ -64,6 +64,8 @@ public interface NetworkOffering extends InfrastructureEntity, InternalIdentity,
     public static final String DEFAULT_NAT_NSX_OFFERING_FOR_VPC = "DefaultNATNSXNetworkOfferingForVpc";
     public static final String DEFAULT_NAT_NSX_OFFERING_FOR_VPC_WITH_ILB = "DefaultNATNSXNetworkOfferingForVpcWithInternalLB";
     public static final String DEFAULT_ROUTED_NSX_OFFERING_FOR_VPC = "DefaultRoutedNSXNetworkOfferingForVpc";
+    public static final String DEFAULT_ROUTED_NETRIS_OFFERING_FOR_VPC = "DefaultRoutedNetrisNetworkOfferingForVpc";
+    public static final String DEFAULT_NAT_NETRIS_OFFERING_FOR_VPC = "DefaultNATNetrisNetworkOfferingForVpc";
     public static final String DEFAULT_NAT_NSX_OFFERING = "DefaultNATNSXNetworkOffering";
     public static final String DEFAULT_ROUTED_NSX_OFFERING = "DefaultRoutedNSXNetworkOffering";
     public final static String QuickCloudNoServices = "QuickCloudNoServices";
@@ -102,10 +104,6 @@ public interface NetworkOffering extends InfrastructureEntity, InternalIdentity,
 
     boolean isForVpc();
 
-    boolean isForTungsten();
-
-    boolean isForNsx();
-
     NetworkMode getNetworkMode();
 
     TrafficType getTrafficType();

api/src/main/java/com/cloud/offering/ServiceOffering.java

@@ -142,4 +142,8 @@ public interface ServiceOffering extends InfrastructureEntity, InternalIdentity,
     Boolean getDiskOfferingStrictness();
 
     void setDiskOfferingStrictness(boolean diskOfferingStrictness);
+
+    Long getVgpuProfileId();
+
+    Integer getGpuCount();
 }

api/src/main/java/com/cloud/storage/Storage.java

@@ -30,6 +30,7 @@ public class Storage {
         OVA(true, true, true, "ova"),
         VHDX(true, true, true, "vhdx"),
         BAREMETAL(false, false, false, "BAREMETAL"),
+        EXTERNAL(false, false, false, "EXTERNAL"),
         VMDK(true, true, false, "vmdk"),
         VDI(true, true, false, "vdi"),
         TAR(false, false, false, "tar"),

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -137,7 +137,7 @@ public interface VolumeApiService {
 
     void updateDisplay(Volume volume, Boolean displayVolume);
 
-    Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName) throws ResourceAllocationException;
+    Snapshot allocSnapshotForVm(Long vmId, Long volumeId, String snapshotName, Long vmSnapshotId) throws ResourceAllocationException;
 
     /**
      *  Checks if the storage pool supports the disk offering tags.
@@ -171,6 +171,13 @@ public interface VolumeApiService {
      *   </table>
      */
     boolean doesStoragePoolSupportDiskOffering(StoragePool destPool, DiskOffering diskOffering);
+
+    /**
+     * Checks if the storage pool supports the required disk offering tags
+     * destPool the storage pool to check the disk offering tags
+     * diskOfferingTags the tags that should be supported
+     * return whether the tags are supported in the storage pool
+     */
     boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags);
 
     Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge);

api/src/main/java/com/cloud/template/VirtualMachineTemplate.java

@@ -153,4 +153,6 @@ public interface VirtualMachineTemplate extends ControlledEntity, Identity, Inte
 
     CPU.CPUArch getArch();
 
+    Long getExtensionId();
+
 }

api/src/main/java/com/cloud/user/AccountService.java

@@ -87,6 +87,8 @@ public interface AccountService {
 
     boolean isDomainAdmin(Long accountId);
 
+    boolean isResourceDomainAdmin(Long accountId);
+
     boolean isNormalUser(long accountId);
 
     User getActiveUserByRegistrationToken(String registrationToken);

api/src/main/java/com/cloud/user/ResourceLimitService.java

@@ -50,8 +50,14 @@ public interface ResourceLimitService {
                 "The default maximum number of projects that can be created for an account",false);
     static final ConfigKey<Long> DefaultMaxDomainProjects = new ConfigKey<>("Domain Defaults",Long.class,"max.domain.projects","50",
                         "The default maximum number of projects that can be created for a domain",false);
+    static final ConfigKey<Long> DefaultMaxAccountGpus = new ConfigKey<>("Account Defaults",Long.class,"max.account.gpus","20",
+            "The default maximum number of GPU devices that can be used for an account", false);
+    static final ConfigKey<Long> DefaultMaxDomainGpus = new ConfigKey<>("Domain Defaults",Long.class,"max.domain.gpus","20",
+            "The default maximum number of GPU devices that can be used for a domain", false);
+    static final ConfigKey<Long> DefaultMaxProjectGpus = new ConfigKey<>("Project Defaults",Long.class,"max.project.gpus","20",
+            "The default maximum number of GPU devices that can be used for a project", false);
 
-    static final List<ResourceType> HostTagsSupportingTypes = List.of(ResourceType.user_vm, ResourceType.cpu, ResourceType.memory);
+    static final List<ResourceType> HostTagsSupportingTypes = List.of(ResourceType.user_vm, ResourceType.cpu, ResourceType.memory, ResourceType.gpu);
     static final List<ResourceType> StorageTagsSupportingTypes = List.of(ResourceType.volume, ResourceType.primary_storage);
 
     /**
@@ -284,4 +290,8 @@ public interface ResourceLimitService {
     void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory);
     void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory);
 
+    void checkVmGpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu) throws ResourceAllocationException;
+    void incrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu);
+    void decrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu);
+
 }

api/src/main/java/com/cloud/vm/UserVmService.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.vm;
 
+import com.cloud.storage.Snapshot;
+import com.cloud.storage.Volume;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -222,7 +224,7 @@ public interface UserVmService {
                                                   String userData, Long userDataId, String userDataDetails, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIp, Boolean displayVm, String keyboard,
                                                   List<Long> affinityGroupIdList, Map<String, String> customParameter, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
                                                   Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
-                                                  Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId) throws InsufficientCapacityException,
+                                                  Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, Volume volume, Snapshot snapshot) throws InsufficientCapacityException,
         ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
 
     /**
@@ -298,7 +300,7 @@ public interface UserVmService {
                                                      List<Long> securityGroupIdList, Account owner, String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor,
                                                      HTTPMethod httpmethod, String userData, Long userDataId, String userDataDetails, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard,
                                                      List<Long> affinityGroupIdList, Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap,
-                                                     Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap, Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, String vmType) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
+                                                     Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap, Map<String, String> userVmOVFProperties, boolean dynamicScalingEnabled, Long overrideDiskOfferingId, String vmType, Volume volume, Snapshot snapshot) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
 
     /**
      * Creates a User VM in Advanced Zone (Security Group feature is disabled)
@@ -370,7 +372,7 @@ public interface UserVmService {
                                         String hostName, String displayName, Long diskOfferingId, Long diskSize, String group, HypervisorType hypervisor, HTTPMethod httpmethod, String userData,
                                         Long userDataId, String userDataDetails, List<String> sshKeyPairs, Map<Long, IpAddresses> requestedIps, IpAddresses defaultIps, Boolean displayVm, String keyboard, List<Long> affinityGroupIdList,
                                         Map<String, String> customParameters, String customId, Map<String, Map<Integer, String>> dhcpOptionMap, Map<Long, DiskOffering> dataDiskTemplateToDiskOfferingMap,
-                                        Map<String, String> templateOvfPropertiesMap, boolean dynamicScalingEnabled, String vmType, Long overrideDiskOfferingId)
+                                        Map<String, String> templateOvfPropertiesMap, boolean dynamicScalingEnabled, String vmType, Long overrideDiskOfferingId, Volume volume, Snapshot snapshot)
 
         throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, StorageUnavailableException, ResourceAllocationException;
 

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -114,7 +114,15 @@ public interface VmDetailConstants {
     String GUEST_CPU_MODE = "guest.cpu.mode";
     String GUEST_CPU_MODEL = "guest.cpu.model";
 
+    // Lease related
     String INSTANCE_LEASE_EXPIRY_DATE = "leaseexpirydate";
     String INSTANCE_LEASE_EXPIRY_ACTION = "leaseexpiryaction";
     String INSTANCE_LEASE_EXECUTION = "leaseactionexecution";
+
+    // External orchestrator related
+    String MAC_ADDRESS = "mac_address";
+    String EXPUNGE_EXTERNAL_VM = "expunge.external.vm";
+    String EXTERNAL_DETAIL_PREFIX = "External:";
+    String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details";
+    String CLOUDSTACK_VLAN = "cloudstack.vlan";
 }

api/src/main/java/com/cloud/vm/snapshot/VMSnapshot.java

@@ -31,7 +31,8 @@ public interface VMSnapshot extends ControlledEntity, Identity, InternalIdentity
     enum State {
         Allocated("The VM snapshot is allocated but has not been created yet."), Creating("The VM snapshot is being created."), Ready(
                 "The VM snapshot is ready to be used."), Reverting("The VM snapshot is being used to revert"), Expunging("The volume is being expunging"), Removed(
-                "The volume is destroyed, and can't be recovered."), Error("The volume is in error state, and can't be recovered");
+                "The volume is destroyed, and can't be recovered."), Error("The volume is in error state, and can't be recovered"),
+        Hidden("The VM snapshot is hidden from the user and cannot be recovered.");
 
         String _description;
 
@@ -60,6 +61,8 @@ public interface VMSnapshot extends ControlledEntity, Identity, InternalIdentity
             s_fsm.addTransition(Expunging, Event.ExpungeRequested, Expunging);
             s_fsm.addTransition(Expunging, Event.OperationSucceeded, Removed);
             s_fsm.addTransition(Expunging, Event.OperationFailed, Error);
+            s_fsm.addTransition(Expunging, Event.Hide, Hidden);
+            s_fsm.addTransition(Hidden, Event.ExpungeRequested, Expunging);
         }
     }
 
@@ -68,7 +71,7 @@ public interface VMSnapshot extends ControlledEntity, Identity, InternalIdentity
     }
 
     enum Event {
-        CreateRequested, OperationFailed, OperationSucceeded, RevertRequested, ExpungeRequested,
+        CreateRequested, OperationFailed, OperationSucceeded, RevertRequested, ExpungeRequested, Hide,
     }
 
     @Override

api/src/main/java/org/apache/cloudstack/acl/RoleType.java

@@ -23,8 +23,11 @@ import com.google.common.base.Enums;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
+import java.util.Collection;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 // Enum for default roles in CloudStack
 public enum RoleType {
@@ -100,6 +103,30 @@ public enum RoleType {
         return roleId;
     }
 
+    public static int toCombinedMask(Collection<RoleType> roles) {
+        int combinedMask = 0;
+        if (roles != null) {
+            for (RoleType role : roles) {
+                combinedMask |= role.getMask();
+            }
+        }
+        return combinedMask;
+    }
+
+    public static Set<RoleType> fromCombinedMask(int combinedMask) {
+        Set<RoleType> roles = EnumSet.noneOf(RoleType.class);
+        for (RoleType roleType : RoleType.values()) {
+            if ((combinedMask & roleType.getMask()) != 0) {
+                roles.add(roleType);
+            }
+        }
+        if (roles.isEmpty()) {
+            roles.add(Unknown);
+        }
+        return roles;
+    }
+
+
     /**
      * This method returns the role account type if the role isn't null, else it returns the default account type.
      * */

api/src/main/java/org/apache/cloudstack/alert/AlertService.java

@@ -73,6 +73,7 @@ public interface AlertService {
         public static final AlertType ALERT_TYPE_VM_SNAPSHOT = new AlertType((short)32, "ALERT.VM.SNAPSHOT", true);
         public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true);
         public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true);
+        public static final AlertType ALERT_TYPE_EXTENSION_PATH_NOT_READY = new AlertType((short)33, "ALERT.TYPE.EXTENSION.PATH.NOT.READY", true);
 
         public short getType() {
             return type;

api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java

@@ -87,7 +87,9 @@ public enum ApiCommandResourceType {
     QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class),
     KubernetesCluster(com.cloud.kubernetes.cluster.KubernetesCluster.class),
     KubernetesSupportedVersion(null),
-    SharedFS(org.apache.cloudstack.storage.sharedfs.SharedFS.class);
+    SharedFS(org.apache.cloudstack.storage.sharedfs.SharedFS.class),
+    Extension(org.apache.cloudstack.extension.Extension.class),
+    ExtensionCustomAction(org.apache.cloudstack.extension.ExtensionCustomAction.class);
 
     private final Class<?> clazz;
 

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -32,6 +32,7 @@ public class ApiConstants {
     public static final String ALLOCATED_DATE = "allocateddate";
     public static final String ALLOCATED_ONLY = "allocatedonly";
     public static final String ALLOCATED_TIME = "allocated";
+    public static final String ALLOWED_ROLE_TYPES = "allowedroletypes";
     public static final String ALLOW_USER_FORCE_STOP_VM = "allowuserforcestopvm";
     public static final String ANNOTATION = "annotation";
     public static final String API_KEY = "apikey";
@@ -68,6 +69,7 @@ public class ApiConstants {
     public static final String BOOTABLE = "bootable";
     public static final String BIND_DN = "binddn";
     public static final String BIND_PASSWORD = "bindpass";
+    public static final String BUS_ADDRESS = "busaddress";
     public static final String BYTES_READ_RATE = "bytesreadrate";
     public static final String BYTES_READ_RATE_MAX = "bytesreadratemax";
     public static final String BYTES_READ_RATE_MAX_LENGTH = "bytesreadratemaxlength";
@@ -90,9 +92,11 @@ public class ApiConstants {
     public static final String CONVERT_INSTANCE_HOST_ID = "convertinstancehostid";
     public static final String CONVERT_INSTANCE_STORAGE_POOL_ID = "convertinstancepoolid";
     public static final String ENABLED_REVOCATION_CHECK = "enabledrevocationcheck";
+    public static final String COMBINED_CAPACITY_ORDERING = "COMBINED";
     public static final String CONTROLLER = "controller";
     public static final String CONTROLLER_UNIT = "controllerunit";
     public static final String COPY_IMAGE_TAGS = "copyimagetags";
+    public static final String CPU_OVERCOMMIT_RATIO = "cpuOvercommitRatio";
     public static final String CSR = "csr";
     public static final String PRIVATE_KEY = "privatekey";
     public static final String DATASTORE_HOST = "datastorehost";
@@ -124,6 +128,7 @@ public class ApiConstants {
     public static final String CNI_CONFIG_DETAILS = "cniconfigdetails";
     public static final String CNI_CONFIG_NAME = "cniconfigname";
     public static final String COMPONENT = "component";
+    public static final String CPU = "CPU";
     public static final String CPU_CORE_PER_SOCKET = "cpucorepersocket";
     public static final String CPU_NUMBER = "cpunumber";
     public static final String CPU_SPEED = "cpuspeed";
@@ -137,6 +142,7 @@ public class ApiConstants {
     public static final String CUSTOMIZED = "customized";
     public static final String CUSTOMIZED_IOPS = "customizediops";
     public static final String CUSTOM_ID = "customid";
+    public static final String CUSTOM_ACTION_ID = "customactionid";
     public static final String CUSTOM_JOB_ID = "customjobid";
     public static final String CURRENT_START_IP = "currentstartip";
     public static final String CURRENT_END_IP = "currentendip";
@@ -157,10 +163,12 @@ public class ApiConstants {
     public static final String DESTINATION_ZONE_ID = "destzoneid";
     public static final String DETAILS = "details";
     public static final String DEVICE_ID = "deviceid";
+    public static final String DEVICE_NAME = "devicename";
     public static final String DIRECT_DOWNLOAD = "directdownload";
     public static final String DISK = "disk";
     public static final String DISK_OFFERING_ID = "diskofferingid";
     public static final String NEW_DISK_OFFERING_ID = "newdiskofferingid";
+    public static final String ORCHESTRATOR_REQUIRES_PREPARE_VM = "orchestratorrequirespreparevm";
     public static final String OVERRIDE_DISK_OFFERING_ID = "overridediskofferingid";
     public static final String DISK_KBS_READ = "diskkbsread";
     public static final String DISK_KBS_WRITE = "diskkbswrite";
@@ -202,6 +210,7 @@ public class ApiConstants {
     public static final String END_IPV6 = "endipv6";
     public static final String END_PORT = "endport";
     public static final String ENTRY_TIME = "entrytime";
+    public static final String ERROR_MESSAGE = "errormessage";
     public static final String EVENT_ID = "eventid";
     public static final String EVENT_TYPE = "eventtype";
     public static final String EXPIRES = "expires";
@@ -212,6 +221,12 @@ public class ApiConstants {
     public static final String EXTRA_DHCP_OPTION_VALUE = "extradhcpvalue";
     public static final String EXTERNAL = "external";
     public static final String EXTERNAL_UUID = "externaluuid";
+    public static final String EXTERNAL_DETAILS = "externaldetails";
+    public static final String PARAMETERS = "parameters";
+    public static final String EXTENSION = "extension";
+    public static final String EXTENSION_ID = "extensionid";
+    public static final String EXTENSION_NAME = "extensionname";
+    public static final String EXTENSIONS_PATH = "extensionspath";
     public static final String FENCE = "fence";
     public static final String FETCH_LATEST = "fetchlatest";
     public static final String FILESYSTEM = "filesystem";
@@ -220,9 +235,11 @@ public class ApiConstants {
     public static final String FORCED_DESTROY_LOCAL_STORAGE = "forcedestroylocalstorage";
     public static final String FORCE_DELETE_HOST = "forcedeletehost";
     public static final String FORCE_MS_TO_IMPORT_VM_FILES = "forcemstoimportvmfiles";
+    public static final String FORCE_UPDATE_OS_TYPE = "forceupdateostype";
     public static final String FORMAT = "format";
     public static final String FOR_VIRTUAL_NETWORK = "forvirtualnetwork";
     public static final String FOR_SYSTEM_VMS = "forsystemvms";
+    public static final String FOR_PROVIDER = "forprovider";
     public static final String FULL_PATH = "fullpath";
     public static final String GATEWAY = "gateway";
     public static final String IP6_GATEWAY = "ip6gateway";
@@ -267,6 +284,7 @@ public class ApiConstants {
     public static final String PREVIOUS_OWNER_ID = "previousownerid";
     public static final String PREVIOUS_OWNER_NAME = "previousownername";
     public static final String NEXT_ACL_RULE_ID = "nextaclruleid";
+    public static final String NEXT_HOP = "nexthop";
     public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash";
     public static final String IMAGE_PATH = "imagepath";
     public static final String INSTANCE_CONVERSION_SUPPORTED = "instanceconversionsupported";
@@ -344,9 +362,13 @@ public class ApiConstants {
     public static final String MAX_BACKUPS = "maxbackups";
     public static final String MAX_CPU_NUMBER = "maxcpunumber";
     public static final String MAX_MEMORY = "maxmemory";
+    public static final String MEMORY_OVERCOMMIT_RATIO = "memoryOvercommitRatio";
+    public static final String MESSAGE = "message";
     public static final String MIN_CPU_NUMBER = "mincpunumber";
     public static final String MIN_MEMORY = "minmemory";
     public static final String MIGRATION_TYPE = "migrationtype";
+    public static final String MIGRATION_JOB_ID = "migrationjobid";
+    public static final String MIGRATION_JOB_STATUS = "migrationjobstatus";
     public static final String MIGRATIONS = "migrations";
     public static final String MEMORY = "memory";
     public static final String MODE = "mode";
@@ -368,6 +390,7 @@ public class ApiConstants {
     public static final String NEW_START_IP = "newstartip";
     public static final String NEW_END_IP = "newendip";
     public static final String KUBERNETES_NODE_VERSION = "kubernetesnodeversion";
+    public static final String NUMA_NODE = "numanode";
     public static final String NUM_RETRIES = "numretries";
     public static final String OFFER_HA = "offerha";
     public static final String OS_DISTRIBUTION = "osdistribution";
@@ -384,6 +407,13 @@ public class ApiConstants {
     public static final String OS_TYPE_ID = "ostypeid";
     public static final String OS_DISPLAY_NAME = "osdisplayname";
     public static final String OS_NAME_FOR_HYPERVISOR = "osnameforhypervisor";
+    public static final String GPU_CARD_ID = "gpucardid";
+    public static final String GPU_CARD_NAME = "gpucardname";
+    public static final String GPU_COUNT = "gpucount";
+    public static final String GPU_DISPLAY = "gpudisplay";
+    public static final String GPU_DEVICE_TYPE = "gpudevicetype";
+    public static final String GPU_ENABLED = "gpuenabled";
+    public static final String MAX_VGPU_PER_PHYSICAL_GPU = "maxvgpuperphysicalgpu";
     public static final String GUEST_OS_LIST = "guestoslist";
     public static final String GUEST_OS_COUNT = "guestoscount";
     public static final String OS_MAPPING_CHECK_ENABLED = "osmappingcheckenabled";
@@ -395,14 +425,17 @@ public class ApiConstants {
     public static final String PARENT = "parent";
     public static final String PARENT_ID = "parentid";
     public static final String PARENT_DOMAIN_ID = "parentdomainid";
+    public static final String PARENT_GPU_DEVICE_ID = "parentgpudeviceid";
     public static final String PARENT_SUBNET = "parentsubnet";
     public static final String PARENT_TEMPLATE_ID = "parenttemplateid";
     public static final String PASSWORD = "password";
+    public static final String PCI_ROOT = "pciroot";
     public static final String CURRENT_PASSWORD = "currentpassword";
     public static final String SHOULD_UPDATE_PASSWORD = "update_passwd_on_host";
     public static final String PASSWORD_ENABLED = "passwordenabled";
     public static final String SSHKEY_ENABLED = "sshkeyenabled";
     public static final String PATH = "path";
+    public static final String PATH_READY = "pathready";
     public static final String PAYLOAD = "payload";
     public static final String PAYLOAD_URL = "payloadurl";
     public static final String PEERS = "peers";
@@ -425,6 +458,7 @@ public class ApiConstants {
     public static final String POST_URL = "postURL";
     public static final String POWER_STATE = "powerstate";
     public static final String PRECEDENCE = "precedence";
+    public static final String PREPARE_VM = "preparevm";
     public static final String PRIVATE_INTERFACE = "privateinterface";
     public static final String PRIVATE_IP = "privateip";
     public static final String PRIVATE_PORT = "privateport";
@@ -441,12 +475,14 @@ public class ApiConstants {
     public static final String PUBLIC_END_PORT = "publicendport";
     public static final String PUBLIC_ZONE = "publiczone";
     public static final String PURGE_RESOURCES = "purgeresources";
+    public static final String RAM = "RAM";
     public static final String REBALANCE = "rebalance";
     public static final String RECEIVED_BYTES = "receivedbytes";
     public static final String RECONNECT = "reconnect";
     public static final String RECOVER = "recover";
     public static final String REPAIR = "repair";
     public static final String REQUIRES_HVM = "requireshvm";
+    public static final String RESOURCES = "resources";
     public static final String RESOURCE_COUNT = "resourcecount";
     public static final String RESOURCE_NAME = "resourcename";
     public static final String RESOURCE_TYPE = "resourcetype";
@@ -482,6 +518,7 @@ public class ApiConstants {
     public static final String SIGNATURE = "signature";
     public static final String SIGNATURE_VERSION = "signatureversion";
     public static final String SINCE = "since";
+    public static final String SITE_NAME = "sitename";
     public static final String SIZE = "size";
     public static final String SIZEGB = "sizegb";
     public static final String SNAPSHOT = "snapshot";
@@ -518,6 +555,7 @@ public class ApiConstants {
     public static final String POD_STORAGE_ACCESS_GROUPS = "podstorageaccessgroups";
     public static final String ZONE_STORAGE_ACCESS_GROUPS = "zonestorageaccessgroups";
     public static final String SUCCESS = "success";
+    public static final String SUCCESS_MESSAGE = "successmessage";
     public static final String SUITABLE_FOR_VM = "suitableforvirtualmachine";
     public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot";
     public static final String TARGET_IQN = "targetiqn";
@@ -530,6 +568,7 @@ public class ApiConstants {
     public static final String TIMEOUT = "timeout";
     public static final String TIMEZONE = "timezone";
     public static final String TIMEZONEOFFSET = "timezoneoffset";
+    public static final String TENANT_NAME = "tenantname";
     public static final String TOTAL = "total";
     public static final String TOTAL_SUBNETS = "totalsubnets";
     public static final String TYPE = "type";
@@ -558,7 +597,14 @@ public class ApiConstants {
     public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork";
     public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver";
     public static final String UPDATE_IN_SEQUENCE = "updateinsequence";
+    public static final String VALIDATION_FORMAT = "validationformat";
     public static final String VALUE = "value";
+    public static final String VALUE_OPTIONS = "valueoptions";
+    public static final String VENDOR_ID = "vendorid";
+    public static final String VENDOR_NAME = "vendorname";
+    public static final String VGPU_PROFILE_ID = "vgpuprofileid";
+    public static final String VGPU_PROFILE_NAME = "vgpuprofilename";
+    public static final String VIRTUAL_MACHINE = "virtualmachine";
     public static final String VIRTUAL_MACHINE_ID = "virtualmachineid";
     public static final String VIRTUAL_MACHINE_IDS = "virtualmachineids";
     public static final String VIRTUAL_MACHINE_NAME = "virtualmachinename";
@@ -657,7 +703,7 @@ public class ApiConstants {
     public static final String NETWORK_DEVICE_PARAMETER_LIST = "networkdeviceparameterlist";
     public static final String ZONE_TOKEN = "zonetoken";
     public static final String DHCP_PROVIDER = "dhcpprovider";
-    public static final String RESULT = "success";
+    public static final String RESULT = "result";
     public static final String RESUME = "resume";
     public static final String LUN_ID = "lunId";
     public static final String IQN = "iqn";
@@ -906,6 +952,8 @@ public class ApiConstants {
     public static final String NETWORK = "network";
     public static final String VPC_ID = "vpcid";
     public static final String VPC_NAME = "vpcname";
+    public static final String VPC_GATEWAY_ID = "vpcgatewayid";
+    public static final String VPC_GATEWAY_IP = "vpcgatewayip";
     public static final String GATEWAY_ID = "gatewayid";
     public static final String CAN_USE_FOR_DEPLOY = "canusefordeploy";
     public static final String RESOURCE_IDS = "resourceids";
@@ -1049,6 +1097,7 @@ public class ApiConstants {
 
     public static final String RESOURCE_DETAILS = "resourcedetails";
     public static final String RESOURCE_ICON = "icon";
+    public static final String RESOURCE_MAP = "resourcemap";
     public static final String EXPUNGE = "expunge";
     public static final String FOR_DISPLAY = "fordisplay";
     public static final String PASSIVE = "passive";
@@ -1084,6 +1133,7 @@ public class ApiConstants {
     public static final String OVM3_CLUSTER = "ovm3cluster";
     public static final String OVM3_VIP = "ovm3vip";
     public static final String CLEAN_UP_DETAILS = "cleanupdetails";
+    public static final String CLEAN_UP_PARAMETERS = "cleanupparameters";
     public static final String VIRTUAL_SIZE = "virtualsize";
     public static final String NETSCALER_CONTROLCENTER_ID = "netscalercontrolcenterid";
     public static final String NETSCALER_SERVICEPACKAGE_ID = "netscalerservicepackageid";
@@ -1201,6 +1251,9 @@ public class ApiConstants {
     public static final String SOURCE_NAT_IP_ID = "sourcenatipaddressid";
     public static final String HAS_RULES = "hasrules";
     public static final String NSX_DETAIL_KEY = "forNsx";
+    public static final String NETRIS_DETAIL_KEY = "forNetris";
+    public static final String NETRIS_TAG = "netristag";
+    public static final String NETRIS_VXLAN_ID = "netrisvxlanid";
     public static final String DISK_PATH = "diskpath";
     public static final String IMPORT_SOURCE = "importsource";
     public static final String TEMP_PATH = "temppath";
@@ -1256,6 +1309,22 @@ public class ApiConstants {
 
     public static final String VMWARE_DC = "vmwaredc";
 
+    public static final String CSS = "css";
+
+    public static final String JSON_CONFIGURATION = "jsonconfiguration";
+
+    public static final String COMMON_NAMES = "commonnames";
+
+    public static final String COMMON_NAME = "commonname";
+
+    public static final String DOMAIN_IDS = "domainids";
+
+    public static final String SHOW_PUBLIC = "showpublic";
+
+    public static final String LIST_ONLY_DEFAULT_THEME = "listonlydefaulttheme";
+
+    public static final String RECURSIVE_DOMAINS = "recursivedomains";
+
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
      * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0).
@@ -1308,6 +1377,10 @@ public class ApiConstants {
         all, resource, min;
     }
 
+    public enum ExtensionDetails {
+        all, resource, external, min;
+    }
+
     public enum ApiKeyAccess {
         DISABLED(false),
         ENABLED(true),

api/src/main/java/org/apache/cloudstack/api/BaseCmd.java

@@ -39,6 +39,7 @@ import org.apache.cloudstack.affinity.AffinityGroupService;
 import org.apache.cloudstack.alert.AlertService;
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.gpu.GpuService;
 import org.apache.cloudstack.network.RoutedIpv4Manager;
 import org.apache.cloudstack.network.lb.ApplicationLoadBalancerService;
 import org.apache.cloudstack.network.lb.InternalLoadBalancerVMService;
@@ -94,6 +95,7 @@ import com.cloud.utils.ReflectUtil;
 import com.cloud.utils.db.EntityManager;
 import com.cloud.utils.db.UUIDManager;
 import com.cloud.vm.UserVmService;
+import com.cloud.vm.VmDetailConstants;
 import com.cloud.vm.snapshot.VMSnapshotService;
 
 public abstract class BaseCmd {
@@ -130,6 +132,8 @@ public abstract class BaseCmd {
     @Inject
     public UserVmService _userVmService;
     @Inject
+    public GpuService gpuService;
+    @Inject
     public ManagementService _mgr;
     @Inject
     public StorageService _storageService;
@@ -484,4 +488,14 @@ public abstract class BaseCmd {
         }
         return detailsMap;
     }
+
+    public Map<String, String> convertExternalDetailsToMap(Map externalDetails) {
+        Map<String, String> customparameterMap = convertDetailsToMap(externalDetails);
+        Map<String, String> details = new HashMap<>();
+        for (String key : customparameterMap.keySet()) {
+            String value = customparameterMap.get(key);
+            details.put(VmDetailConstants.EXTERNAL_DETAIL_PREFIX + key, value);
+        }
+        return details;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java

@@ -51,6 +51,10 @@ public abstract class BaseUpdateTemplateOrIsoCmd extends BaseCmd {
                description = "the ID of the OS type that best represents the OS of this image.")
     private Long osTypeId;
 
+    @Parameter(name = ApiConstants.FORCE_UPDATE_OS_TYPE, type = CommandType.BOOLEAN, since = "4.21", description = "Force OS type update. Warning: Updating OS type will " +
+            "update the guest OS configuration for all the existing Instances deployed with this template/iso, which may affect their behavior.")
+    private Boolean forceUpdateOsType;
+
     @Parameter(name = ApiConstants.FORMAT, type = CommandType.STRING, description = "the format for the image")
     private String format;
 
@@ -112,6 +116,10 @@ public abstract class BaseUpdateTemplateOrIsoCmd extends BaseCmd {
         return osTypeId;
     }
 
+    public Boolean getForceUpdateOsType() {
+        return forceUpdateOsType;
+    }
+
     public Boolean getPasswordEnabled() {
         return passwordEnabled;
     }

api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java

@@ -64,6 +64,7 @@ import org.apache.cloudstack.api.response.GuestOSResponse;
 import org.apache.cloudstack.api.response.GuestOsMappingResponse;
 import org.apache.cloudstack.api.response.GuestVlanRangeResponse;
 import org.apache.cloudstack.api.response.GuestVlanResponse;
+import org.apache.cloudstack.api.response.GuiThemeResponse;
 import org.apache.cloudstack.api.response.HostForMigrationResponse;
 import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.HypervisorCapabilitiesResponse;
@@ -150,6 +151,7 @@ import org.apache.cloudstack.config.ConfigurationGroup;
 import org.apache.cloudstack.direct.download.DirectDownloadCertificate;
 import org.apache.cloudstack.direct.download.DirectDownloadCertificateHostMap;
 import org.apache.cloudstack.direct.download.DirectDownloadManager;
+import org.apache.cloudstack.gui.theme.GuiThemeJoin;
 import org.apache.cloudstack.management.ManagementServerHost;
 import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule;
 import org.apache.cloudstack.region.PortableIp;
@@ -579,4 +581,6 @@ public interface ResponseGenerator {
     SharedFSResponse createSharedFSResponse(ResponseView view, SharedFS sharedFS);
 
     void updateTemplateIsoResponsesForIcons(List<TemplateResponse> responses, ResourceTag.ResourceObjectType type);
+
+    GuiThemeResponse createGuiThemeResponse(GuiThemeJoin guiThemeJoin);
 }

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java

@@ -19,21 +19,22 @@ package org.apache.cloudstack.api.command.admin.cluster;
 
 import java.util.ArrayList;
 import java.util.List;
-
-import com.cloud.cpu.CPU;
-import org.apache.cloudstack.api.ApiCommandResourceType;
+import java.util.Map;
 
 import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiErrorCode;
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.api.response.ExtensionResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.PodResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 
+import com.cloud.cpu.CPU;
 import com.cloud.exception.DiscoveryException;
 import com.cloud.exception.ResourceInUseException;
 import com.cloud.org.Cluster;
@@ -43,7 +44,6 @@ import com.cloud.user.Account;
         requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
 public class AddClusterCmd extends BaseCmd {
 
-
     @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING, required = true, description = "the cluster name")
     private String clusterName;
 
@@ -65,7 +65,7 @@ public class AddClusterCmd extends BaseCmd {
     @Parameter(name = ApiConstants.HYPERVISOR,
                type = CommandType.STRING,
                required = true,
-               description = "hypervisor type of the cluster: XenServer,KVM,VMware,Hyperv,BareMetal,Simulator,Ovm3")
+               description = "hypervisor type of the cluster: XenServer,KVM,VMware,Hyperv,BareMetal,Simulator,Ovm3,External")
     private String hypervisor;
 
     @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING,
@@ -118,12 +118,26 @@ public class AddClusterCmd extends BaseCmd {
     private String ovm3cluster;
     @Parameter(name = ApiConstants.OVM3_VIP, type = CommandType.STRING, required = false,  description = "Ovm3 vip to use for pool (and cluster)")
     private String ovm3vip;
+
     @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS,
             type = CommandType.LIST, collectionType = CommandType.STRING,
             description = "comma separated list of storage access groups for the hosts in the cluster",
             since = "4.21.0")
     private List<String> storageAccessGroups;
 
+
+    @Parameter(name = ApiConstants.EXTENSION_ID,
+            type = CommandType.UUID, entityType = ExtensionResponse.class,
+            description = "UUID of the extension",
+            since = "4.21.0")
+    private Long extensionId;
+
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS,
+            type = CommandType.MAP,
+            description = "Details in key/value pairs to be added to the extension-resource mapping. Use the format externaldetails[i].<key>=<value>. Example: externaldetails[0].endpoint.url=https://example.com",
+            since = "4.21.0")
+    protected Map externalDetails;
+
     public String getOvm3Pool() {
          return ovm3pool;
     }
@@ -190,6 +204,10 @@ public class AddClusterCmd extends BaseCmd {
         return hypervisor;
     }
 
+    public Long getExtensionId() {
+        return extensionId;
+    }
+
     public String getClusterType() {
         return clusterType;
     }
@@ -224,6 +242,10 @@ public class AddClusterCmd extends BaseCmd {
         return CPU.CPUArch.fromType(arch);
     }
 
+    public Map<String, String> getExternalDetails() {
+        return convertDetailsToMap(externalDetails);
+    }
+
     @Override
     public void execute() {
         try {

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java

@@ -17,7 +17,11 @@
 package org.apache.cloudstack.api.command.admin.cluster;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+
+import javax.inject.Inject;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@@ -27,9 +31,13 @@ import org.apache.cloudstack.api.response.ClusterResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.PodResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.extension.Extension;
+import org.apache.cloudstack.extension.ExtensionHelper;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.cpu.CPU;
+import com.cloud.hypervisor.Hypervisor;
 import com.cloud.org.Cluster;
 import com.cloud.utils.Pair;
 
@@ -37,6 +45,8 @@ import com.cloud.utils.Pair;
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
 public class ListClustersCmd extends BaseListCmd {
 
+    @Inject
+    ExtensionHelper extensionHelper;
 
     /////////////////////////////////////////////////////
     //////////////// API parameters /////////////////////
@@ -143,21 +153,44 @@ public class ListClustersCmd extends BaseListCmd {
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
 
+    protected void updateClustersExtensions(final List<ClusterResponse> clusterResponses) {
+        if (CollectionUtils.isEmpty(clusterResponses)) {
+            return;
+        }
+        Map<Long, Extension> idExtensionMap = new HashMap<>();
+        for  (ClusterResponse response : clusterResponses) {
+            if (!Hypervisor.HypervisorType.External.getHypervisorDisplayName().equals(response.getHypervisorType())) {
+                continue;
+            }
+            Long extensionId = extensionHelper.getExtensionIdForCluster(response.getInternalId());
+            if (extensionId == null) {
+                continue;
+            }
+            Extension extension = idExtensionMap.computeIfAbsent(extensionId, id -> extensionHelper.getExtension(id));
+            if (extension == null) {
+                continue;
+            }
+            response.setExtensionId(extension.getUuid());
+            response.setExtensionName(extension.getName());
+        }
+    }
+
     protected Pair<List<ClusterResponse>, Integer> getClusterResponses() {
         Pair<List<? extends Cluster>, Integer> result = _mgr.searchForClusters(this);
-        List<ClusterResponse> clusterResponses = new ArrayList<ClusterResponse>();
+        List<ClusterResponse> clusterResponses = new ArrayList<>();
         for (Cluster cluster : result.first()) {
             ClusterResponse clusterResponse = _responseGenerator.createClusterResponse(cluster, showCapacities);
             clusterResponse.setObjectName("cluster");
             clusterResponses.add(clusterResponse);
         }
-        return new Pair<List<ClusterResponse>, Integer>(clusterResponses, result.second());
+        updateClustersExtensions(clusterResponses);
+        return new Pair<>(clusterResponses, result.second());
     }
 
     @Override
     public void execute() {
         Pair<List<ClusterResponse>, Integer> clusterResponses = getClusterResponses();
-        ListResponse<ClusterResponse> response = new ListResponse<ClusterResponse>();
+        ListResponse<ClusterResponse> response = new ListResponse<>();
         response.setResponses(clusterResponses.first(), clusterResponses.second());
         response.setResponseName(getCommandName());
         this.setResponseObject(response);

api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/UpdateClusterCmd.java

@@ -16,6 +16,8 @@
 // under the License.
 package org.apache.cloudstack.api.command.admin.cluster;
 
+import java.util.Map;
+
 import com.cloud.cpu.CPU;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 
@@ -60,6 +62,12 @@ public class UpdateClusterCmd extends BaseCmd {
             since = "4.20")
     private String arch;
 
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS,
+            type = CommandType.MAP,
+            description = "Details in key/value pairs to be added to the extension-resource mapping. Use the format externaldetails[i].<key>=<value>. Example: externaldetails[0].endpoint.url=https://example.com",
+            since = "4.21.0")
+    protected Map externalDetails;
+
     public String getClusterName() {
         return clusterName;
     }
@@ -122,6 +130,10 @@ public class UpdateClusterCmd extends BaseCmd {
         return CPU.CPUArch.fromType(arch);
     }
 
+    public Map<String, String> getExternalDetails() {
+        return convertDetailsToMap(externalDetails);
+    }
+
     @Override
     public void execute() {
         Cluster cluster = _resourceService.getCluster(getId());

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/CreateGpuCardCmd.java

@@ -0,0 +1,122 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.user.Account;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.gpu.GpuCard;
+
+
+@APICommand(name = "createGpuCard", description = "Creates a GPU card definition in the system",
+        responseObject = GpuCardResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.21.0")
+public class CreateGpuCardCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.DEVICE_ID, type = CommandType.STRING, required = true,
+            description = "the device ID of the GPU card")
+    private String deviceId;
+
+    @Parameter(name = ApiConstants.DEVICE_NAME, type = CommandType.STRING, required = true,
+            description = "the device name of the GPU card")
+    private String deviceName;
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true,
+            description = "the display name of the GPU card")
+    private String name;
+
+    @Parameter(name = ApiConstants.VENDOR_NAME, type = CommandType.STRING, required = true,
+            description = "the vendor name of the GPU card")
+    private String vendorName;
+
+    @Parameter(name = ApiConstants.VENDOR_ID, type = CommandType.STRING, required = true,
+            description = "the vendor ID of the GPU card")
+    private String vendorId;
+
+    // Optional parameters for the passthrough vGPU profile display properties
+    @Parameter(name = ApiConstants.VIDEORAM, type = CommandType.LONG,
+            description = "the video RAM size in MB for the passthrough vGPU profile")
+    private Long videoRam;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public String getDeviceId() {
+        return deviceId;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getVendorName() {
+        return vendorName;
+    }
+
+    public String getVendorId() {
+        return vendorId;
+    }
+
+    public Long getVideoRam() {
+        return videoRam;
+    }
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException,
+            ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
+        try {
+            GpuCard gpuCard = gpuService.createGpuCard(this);
+            if (gpuCard != null) {
+                GpuCardResponse response = new GpuCardResponse(gpuCard);
+                response.setResponseName(getCommandName());
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create GPU card");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create GPU card: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/CreateGpuDeviceCmd.java

@@ -0,0 +1,123 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
+import org.apache.cloudstack.gpu.GpuDevice;
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
+
+
+@APICommand(name = "createGpuDevice", description = "Creates a GPU device manually on a host",
+            responseObject = GpuDeviceResponse.class, since = "4.21.0", requestHasSensitiveInfo = false,
+            responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
+public class CreateGpuDeviceCmd extends BaseCmd {
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, required = true,
+               description = "ID of the host where the GPU device is located")
+    private Long hostId;
+
+    @Parameter(name = ApiConstants.BUS_ADDRESS, type = CommandType.STRING, required = true,
+               description = "PCI bus address of the GPU device (e.g., 0000:01:00.0) or UUID for MDEV devices.")
+    private String busAddress;
+
+    @Parameter(name = ApiConstants.GPU_CARD_ID, type = CommandType.UUID, entityType = GpuCardResponse.class,
+               required = true, description = "ID of the GPU card type")
+    private Long gpuCardId;
+
+    @Parameter(name = ApiConstants.VGPU_PROFILE_ID, type = CommandType.UUID, entityType = VgpuProfileResponse.class,
+               required = true, description = "ID of the vGPU profile")
+    private Long vgpuProfileId;
+
+    @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING,
+               description = "Type of GPU device (PCI, MDEV, VGPUOnly). Defaults to PCI.")
+    private String type;
+
+    @Parameter(name = ApiConstants.PARENT_GPU_DEVICE_ID, type = CommandType.UUID, entityType = GpuDeviceResponse.class,
+               description = "ID of the parent GPU device (for virtual GPU devices)")
+    private Long parentGpuDeviceId;
+
+    @Parameter(name = ApiConstants.NUMA_NODE, type = CommandType.STRING,
+            description = "NUMA node of the GPU device (e.g., 0, 1, etc.). This is optional and can be used to "
+                          + "specify the NUMA node for the GPU device which is used during allocation. Defaults to -1")
+    private String numaNode;
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public String getBusAddress() {
+        return busAddress;
+    }
+
+    public Long getGpuCardId() {
+        return gpuCardId;
+    }
+
+    public Long getVgpuProfileId() {
+        return vgpuProfileId;
+    }
+
+    public GpuDevice.DeviceType getType() {
+        GpuDevice.DeviceType deviceType = GpuDevice.DeviceType.PCI;
+        if (StringUtils.isNotBlank(type)) {
+            deviceType = EnumUtils.getEnumIgnoreCase(GpuDevice.DeviceType.class, type);
+            if (deviceType == null) {
+                throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid GPU device type: " + type);
+            }
+        }
+        return deviceType;
+    }
+
+    public Long getParentGpuDeviceId() {
+        return parentGpuDeviceId;
+    }
+
+    public String getNumaNode() {
+        if (StringUtils.isBlank(numaNode)) {
+            return "-1"; // Default value for NUMA node
+        }
+        return numaNode;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            GpuDeviceResponse response = gpuService.createGpuDevice(this);
+            response.setResponseName(getCommandName());
+            setResponseObject(response);
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
+        }
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/CreateVgpuProfileCmd.java

@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
+
+
+@APICommand(name = "createVgpuProfile", description = "Creates a vGPU profile in the system",
+            responseObject = VgpuProfileResponse.class, requestHasSensitiveInfo = false,
+            responseHasSensitiveInfo = false, since = "4.21.0", authorized = {RoleType.Admin})
+public class CreateVgpuProfileCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true,
+               description = "the name of the vGPU profile")
+    private String name;
+
+    @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING,
+               description = "the description of the vGPU profile")
+    private String description;
+
+    @Parameter(name = ApiConstants.GPU_CARD_ID, type = CommandType.UUID, entityType = GpuCardResponse.class,
+               required = true, description = "the GPU card ID associated with this GPU device")
+    private Long cardId;
+
+    @Parameter(name = ApiConstants.MAX_VGPU_PER_PHYSICAL_GPU, type = CommandType.LONG,
+               description = "Max vGPU per physical GPU. This is used to calculate capacity.")
+    private Long maxVgpuPerPgpu;
+
+    @Parameter(name = ApiConstants.VIDEORAM, type = CommandType.LONG,
+               description = "the video RAM size in MB")
+    private Long videoRam;
+
+    @Parameter(name = ApiConstants.MAXHEADS, type = CommandType.LONG,
+               description = "the maximum number of display heads")
+    private Long maxHeads;
+
+    @Parameter(name = ApiConstants.MAXRESOLUTIONX, type = CommandType.LONG,
+               description = "the maximum X resolution")
+    private Long maxResolutionX;
+
+    @Parameter(name = ApiConstants.MAXRESOLUTIONY, type = CommandType.LONG,
+               description = "the maximum Y resolution")
+    private Long maxResolutionY;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public String getName() {
+        return name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public Long getCardId() {
+        return cardId;
+    }
+
+    public Long getMaxVgpuPerPgpu() {
+        return maxVgpuPerPgpu;
+    }
+
+    public Long getVideoRam() {
+        return videoRam;
+    }
+
+    public Long getMaxHeads() {
+        return maxHeads;
+    }
+
+    public Long getMaxResolutionX() {
+        return maxResolutionX;
+    }
+
+    public Long getMaxResolutionY() {
+        return maxResolutionY;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            VgpuProfileResponse response = gpuService.createVgpuProfile(this);
+            if (response != null) {
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create vGPU profile");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR,
+                    "Failed to create vGPU profile: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/DeleteGpuCardCmd.java

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+
+
+@APICommand(name = "deleteGpuCard", description = "Deletes a GPU card definition from the system",
+            responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+            since = "4.21.0", authorized = {RoleType.Admin})
+public class DeleteGpuCardCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GpuCardResponse.class, required = true,
+               description = "the ID of the GPU card")
+    private Long id;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            boolean success = gpuService.deleteGpuCard(this);
+            if (success) {
+                SuccessResponse response = new SuccessResponse(getCommandName());
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete GPU card");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete GPU card: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/DeleteGpuDeviceCmd.java

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+
+import java.util.List;
+
+@APICommand(name = "deleteGpuDevice", description = "Deletes a vGPU profile from the system",
+        responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.21.0", authorized = {RoleType.Admin})
+public class DeleteGpuDeviceCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID,
+            entityType = GpuDeviceResponse.class, required = true,
+            description = "comma separated list of IDs of the GPU device")
+    private List<Long> ids;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public List<Long> getIds() {
+        return ids;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            boolean success = gpuService.deleteGpuDevices(this);
+            if (success) {
+                SuccessResponse response = new SuccessResponse(getCommandName());
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete vGPU profile");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR,
+                    "Failed to delete vGPU profile: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/DeleteVgpuProfileCmd.java

@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
+
+
+@APICommand(name = "deleteVgpuProfile", description = "Deletes a vGPU profile from the system",
+            responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+            since = "4.21.0", authorized = {RoleType.Admin})
+public class DeleteVgpuProfileCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = VgpuProfileResponse.class, required = true,
+               description = "the ID of the vGPU profile")
+    private Long id;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            boolean success = gpuService.deleteVgpuProfile(this);
+            if (success) {
+                SuccessResponse response = new SuccessResponse(getCommandName());
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete vGPU profile");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR,
+                    "Failed to delete vGPU profile: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/DiscoverGpuDevicesCmd.java

@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.context.CallContext;
+
+
+@APICommand(name = "discoverGpuDevices", description = "Discovers available GPU devices on a host",
+            responseObject = GpuDeviceResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+            since = "4.21.0", authorized = {RoleType.Admin})
+public class DiscoverGpuDevicesCmd extends BaseListCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = HostResponse.class, required = true,
+               description = "ID of the host")
+    private Long id;
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation //////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public void execute() {
+        CallContext.current().setEventDetails("Discovering GPU Devices on host id: " + getId());
+        ListResponse<GpuDeviceResponse> response = gpuService.discoverGpuDevices(this);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/ListGpuDevicesCmdByAdmin.java

@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ResponseObject;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+import org.apache.cloudstack.api.command.user.gpu.ListGpuDevicesCmd;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
+
+
+@APICommand(name = "listGpuDevices", description = "Lists all available GPU devices",
+        responseView = ResponseObject.ResponseView.Full,
+        responseObject = GpuDeviceResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.21.0")
+public class ListGpuDevicesCmdByAdmin extends ListGpuDevicesCmd implements AdminCmd {
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GpuDeviceResponse.class,
+            description = "ID of the GPU device")
+    private Long id;
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class,
+            description = "the host ID where the GPU device is attached")
+    private Long hostId;
+
+    @Parameter(name = ApiConstants.GPU_CARD_ID, type = CommandType.UUID, entityType = GpuCardResponse.class,
+            description = "the GPU card ID associated with the GPU device")
+    private Long gpuCardId;
+
+    @Parameter(name = ApiConstants.VGPU_PROFILE_ID, type = CommandType.UUID, entityType = VgpuProfileResponse.class,
+            description = "the vGPU profile ID assigned to the GPU device")
+    private Long vgpuProfileId;
+
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public Long getGpuCardId() {
+        return gpuCardId;
+    }
+
+    public Long getVgpuProfileId() {
+        return vgpuProfileId;
+    }
+
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/ManageGpuDeviceCmd.java

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+
+import java.util.List;
+
+@APICommand(name = "manageGpuDevice", description = "Manages a GPU device", responseObject = SuccessResponse.class,
+            requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.21.0",
+            authorized = {RoleType.Admin})
+public class ManageGpuDeviceCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID,
+               entityType = GpuDeviceResponse.class, required = true,
+               description = "comma separated list of IDs of the GPU device")
+    private List<Long> ids;
+
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public List<Long> getIds() {
+        return ids;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            if (gpuService.enableGpuDevice(this)) {
+                SuccessResponse response = new SuccessResponse();
+                response.setResponseName(getCommandName());
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to enable GPU device");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to enable GPU device: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/UnmanageGpuDeviceCmd.java

@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+
+import java.util.List;
+
+@APICommand(name = "unmanageGpuDevice", description = "Unmanage a GPU device", responseObject = SuccessResponse.class,
+            requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.21.0",
+            authorized = {RoleType.Admin})
+public class UnmanageGpuDeviceCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID,
+               entityType = GpuDeviceResponse.class, required = true,
+               description = "comma separated list of IDs of the GPU device")
+    private List<Long> ids;
+
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public List<Long> getIds() {
+        return ids;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            if (gpuService.disableGpuDevice(this)) {
+                SuccessResponse response = new SuccessResponse();
+                response.setResponseName(getCommandName());
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to disable GPU device");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR,
+                    "Failed to disable GPU device: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/UpdateGpuCardCmd.java

@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.gpu.GpuCard;
+
+
+@APICommand(name = "updateGpuCard", description = "Updates a GPU card definition in the system",
+            responseObject = GpuCardResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+            since = "4.21.0", authorized = {RoleType.Admin})
+public class UpdateGpuCardCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GpuCardResponse.class, required = true,
+               description = "the ID of the GPU card")
+    private Long id;
+
+    @Parameter(name = ApiConstants.DEVICE_NAME, type = CommandType.STRING,
+               description = "the device name of the GPU card")
+    private String deviceName;
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the display name of the GPU card")
+    private String name;
+
+    @Parameter(name = ApiConstants.VENDOR_NAME, type = CommandType.STRING,
+               description = "the vendor name of the GPU card")
+    private String vendorName;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getVendorName() {
+        return vendorName;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            GpuCard gpuCard = gpuService.updateGpuCard(this);
+            if (gpuCard != null) {
+                GpuCardResponse response = new GpuCardResponse(gpuCard);
+                response.setResponseName(getCommandName());
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update GPU card");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update GPU card: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/UpdateGpuDeviceCmd.java

@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
+import org.apache.cloudstack.gpu.GpuDevice;
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
+
+
+@APICommand(name = "updateGpuDevice", description = "Updates an existing GPU device",
+            responseObject = GpuDeviceResponse.class, since = "4.21.0", requestHasSensitiveInfo = false,
+            responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
+public class UpdateGpuDeviceCmd extends BaseCmd {
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GpuDeviceResponse.class, required = true,
+               description = "ID of the GPU device to update")
+    private Long id;
+
+    @Parameter(name = ApiConstants.GPU_CARD_ID, type = CommandType.UUID, entityType = GpuCardResponse.class,
+               description = "New GPU card ID")
+    private Long gpuCardId;
+
+    @Parameter(name = ApiConstants.VGPU_PROFILE_ID, type = CommandType.UUID, entityType = VgpuProfileResponse.class,
+               description = "New vGPU profile ID")
+    private Long vgpuProfileId;
+
+    @Parameter(name = "type", type = CommandType.STRING, description = "New type of GPU device (PCI, MDEV, VGPUOnly)")
+    private String type;
+
+    @Parameter(name = "parentgpudeviceid", type = CommandType.UUID, entityType = GpuDeviceResponse.class,
+               description = "New parent GPU device ID (for virtual GPU devices)")
+    private Long parentGpuDeviceId;
+
+    @Parameter(name = ApiConstants.NUMA_NODE, type = CommandType.STRING,
+               description = "New NUMA node of the GPU device")
+    private String numaNode;
+
+    public Long getId() {
+        return id;
+    }
+
+    public Long getGpuCardId() {
+        return gpuCardId;
+    }
+
+    public Long getVgpuProfileId() {
+        return vgpuProfileId;
+    }
+
+    public GpuDevice.DeviceType getType() {
+        GpuDevice.DeviceType deviceType = null;
+        if (StringUtils.isNotBlank(type)) {
+            deviceType = EnumUtils.getEnumIgnoreCase(GpuDevice.DeviceType.class, type);
+            if (deviceType == null) {
+                throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid GPU device type: " + type);
+            }
+        }
+        return deviceType;
+    }
+
+    public Long getParentGpuDeviceId() {
+        return parentGpuDeviceId;
+    }
+
+    public String getNumaNode() {
+        return numaNode;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            GpuDeviceResponse response = gpuService.updateGpuDevice(this);
+            response.setResponseName(getCommandName());
+            setResponseObject(response);
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
+        }
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/gpu/UpdateVgpuProfileCmd.java

@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.gpu;
+
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
+
+
+@APICommand(name = "updateVgpuProfile", description = "Updates a vGPU profile in the system",
+            responseObject = VgpuProfileResponse.class, requestHasSensitiveInfo = false,
+            responseHasSensitiveInfo = false, since = "4.21.0", authorized = {RoleType.Admin})
+public class UpdateVgpuProfileCmd extends BaseCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = VgpuProfileResponse.class, required = true,
+               description = "the ID of the vGPU profile")
+    private Long id;
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the vGPU profile")
+    private String profileName;
+
+    @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING,
+               description = "the description of the vGPU profile")
+    private String description;
+
+    @Parameter(name = ApiConstants.MAX_VGPU_PER_PHYSICAL_GPU, type = CommandType.LONG,
+               description = "the maximum number of vGPUs per physical GPU")
+    private Long maxVgpuPerPgpu;
+
+    @Parameter(name = ApiConstants.VIDEORAM, type = CommandType.LONG,
+               description = "the video RAM size in MB")
+    private Long videoRam;
+
+    @Parameter(name = ApiConstants.MAXHEADS, type = CommandType.LONG,
+               description = "the maximum number of display heads")
+    private Long maxHeads;
+
+    @Parameter(name = ApiConstants.MAXRESOLUTIONX, type = CommandType.LONG,
+               description = "the maximum X resolution")
+    private Long maxResolutionX;
+
+    @Parameter(name = ApiConstants.MAXRESOLUTIONY, type = CommandType.LONG,
+               description = "the maximum Y resolution")
+    private Long maxResolutionY;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getProfileName() {
+        return profileName;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public Long getMaxVgpuPerPgpu() {
+        return maxVgpuPerPgpu;
+    }
+
+    public Long getVideoRam() {
+        return videoRam;
+    }
+
+    public Long getMaxHeads() {
+        return maxHeads;
+    }
+
+    public Long getMaxResolutionX() {
+        return maxResolutionX;
+    }
+
+    public Long getMaxResolutionY() {
+        return maxResolutionY;
+    }
+
+    @Override
+    public void execute() {
+        try {
+            VgpuProfileResponse response = gpuService.updateVgpuProfile(this);
+            if (response != null) {
+                setResponseObject(response);
+            } else {
+                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update vGPU profile");
+            }
+        } catch (Exception e) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR,
+                    "Failed to update vGPU profile: " + e.getMessage());
+        }
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddHostCmd.java

@@ -18,7 +18,7 @@ package org.apache.cloudstack.api.command.admin.host;
 
 import java.util.ArrayList;
 import java.util.List;
-
+import java.util.Map;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@@ -81,6 +81,12 @@ public class AddHostCmd extends BaseCmd {
             since = "4.21.0")
     private List<String> storageAccessGroups;
 
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS,
+            type = CommandType.MAP,
+            description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].endpoint.url=urlvalue",
+            since = "4.21.0")
+    protected Map externalDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -129,6 +135,10 @@ public class AddHostCmd extends BaseCmd {
         return allocationState;
     }
 
+    public Map<String, String> getExternalDetails() {
+        return convertExternalDetailsToMap(externalDetails);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java

@@ -16,8 +16,9 @@
 // under the License.
 package org.apache.cloudstack.api.command.admin.host;
 
-import com.cloud.host.Host;
-import com.cloud.user.Account;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
@@ -28,7 +29,8 @@ import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.GuestOSCategoryResponse;
 import org.apache.cloudstack.api.response.HostResponse;
 
-import java.util.List;
+import com.cloud.host.Host;
+import com.cloud.user.Account;
 
 @APICommand(name = "updateHost", description = "Updates a host.", responseObject = HostResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -67,6 +69,9 @@ public class UpdateHostCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ANNOTATION, type = CommandType.STRING, description = "Add an annotation to this host", since = "4.11", authorized = {RoleType.Admin})
     private String annotation;
 
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].endpoint.url=urlvalue", since = "4.21.0")
+    protected Map externalDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -103,6 +108,10 @@ public class UpdateHostCmd extends BaseCmd {
         return annotation;
     }
 
+    public Map<String, String> getExternalDetails() {
+        return convertExternalDetailsToMap(externalDetails);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java

@@ -17,6 +17,7 @@
 package org.apache.cloudstack.api.command.admin.network;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -140,12 +141,19 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
             description = "true if network offering is meant to be used for VPC, false otherwise.")
     private Boolean forVpc;
 
+    @Deprecated
     @Parameter(name = ApiConstants.FOR_NSX,
             type = CommandType.BOOLEAN,
             description = "true if network offering is meant to be used for NSX, false otherwise.",
             since = "4.20.0")
     private Boolean forNsx;
 
+    @Parameter(name = ApiConstants.PROVIDER,
+            type = CommandType.STRING,
+            description = "Name of the provider providing the service",
+            since = "4.21.0")
+    private String provider;
+
     @Parameter(name = ApiConstants.NSX_SUPPORT_LB,
             type = CommandType.BOOLEAN,
             description = "true if network offering for NSX network offering supports Load balancer service.",
@@ -257,18 +265,38 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
         return serviceOfferingId;
     }
 
+    public boolean isExternalNetworkProvider() {
+        return Arrays.asList("NSX", "Netris").stream()
+                .anyMatch(s -> provider != null && s.equalsIgnoreCase(provider));
+    }
+
+    public boolean isForNsx() {
+        return provider != null && provider.equalsIgnoreCase("NSX");
+    }
+
+    public boolean isForNetris() {
+        return provider != null && provider.equalsIgnoreCase("Netris");
+    }
+
+    public String getProvider() {
+        return provider;
+    }
+
     public List<String> getSupportedServices() {
-        if (!isForNsx()) {
+        if (!isExternalNetworkProvider()) {
             return supportedServices == null ? new ArrayList<String>() : supportedServices;
         } else {
             List<String> services = new ArrayList<>(List.of(
                     Dhcp.getName(),
                     Dns.getName(),
-                    StaticNat.getName(),
-                    SourceNat.getName(),
-                    PortForwarding.getName(),
                     UserData.getName()
             ));
+            if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode())) {
+                services.addAll(Arrays.asList(
+                        StaticNat.getName(),
+                        SourceNat.getName(),
+                        PortForwarding.getName()));
+            }
             if (getNsxSupportsLbService()) {
                 services.add(Lb.getName());
             }
@@ -308,10 +336,6 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
         return forVpc;
     }
 
-    public boolean isForNsx() {
-        return BooleanUtils.isTrue(forNsx);
-    }
-
     public String getNetworkMode() {
         return networkMode;
     }
@@ -345,7 +369,7 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
 
     public Map<String, List<String>> getServiceProviders() {
         Map<String, List<String>> serviceProviderMap = new HashMap<>();
-        if (serviceProviderList != null && !serviceProviderList.isEmpty() && !isForNsx()) {
+        if (serviceProviderList != null && !serviceProviderList.isEmpty() && !isExternalNetworkProvider()) {
             Collection servicesCollection = serviceProviderList.values();
             Iterator iter = servicesCollection.iterator();
             while (iter.hasNext()) {
@@ -361,17 +385,16 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
                 providerList.add(provider);
                 serviceProviderMap.put(service, providerList);
             }
-        } else if (Boolean.TRUE.equals(forNsx)) {
-            getServiceProviderMapForNsx(serviceProviderMap);
+        } else if (isExternalNetworkProvider()) {
+            getServiceProviderMapForExternalProvider(serviceProviderMap, Network.Provider.getProvider(provider).getName());
         }
         return serviceProviderMap;
     }
 
-    private void getServiceProviderMapForNsx(Map<String, List<String>> serviceProviderMap) {
+    private void getServiceProviderMapForExternalProvider(Map<String, List<String>> serviceProviderMap, String provider) {
         String routerProvider = Boolean.TRUE.equals(getForVpc()) ? VirtualRouterProvider.Type.VPCVirtualRouter.name() :
                 VirtualRouterProvider.Type.VirtualRouter.name();
-        List<String> unsupportedServices = new ArrayList<>(List.of("Vpn", "SecurityGroup", "Connectivity",
-                "Gateway", "BaremetalPxeService"));
+        List<String> unsupportedServices = new ArrayList<>(List.of("Vpn", "Gateway", "SecurityGroup", "Connectivity", "BaremetalPxeService"));
         List<String> routerSupported = List.of("Dhcp", "Dns", "UserData");
         List<String> allServices = Service.listAllServices().stream().map(Service::getName).collect(Collectors.toList());
         if (routerProvider.equals(VirtualRouterProvider.Type.VPCVirtualRouter.name())) {
@@ -384,8 +407,9 @@ public class CreateNetworkOfferingCmd extends BaseCmd {
                 continue;
             if (routerSupported.contains(service))
                 serviceProviderMap.put(service, List.of(routerProvider));
-            else
-                serviceProviderMap.put(service, List.of(Network.Provider.Nsx.getName()));
+            else if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode()) || NetworkACL.getName().equalsIgnoreCase(service)) {
+                    serviceProviderMap.put(service, List.of(provider));
+                }
             if (!getNsxSupportsLbService()) {
                 serviceProviderMap.remove(Lb.getName());
             }

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java

@@ -29,16 +29,17 @@ import org.apache.cloudstack.api.ApiErrorCode;
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.DiskOfferingResponse;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
 import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
-import org.apache.cloudstack.api.response.DiskOfferingResponse;
 import org.apache.cloudstack.vm.lease.VMLeaseManager;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.EnumUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.collections.CollectionUtils;
 
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.offering.ServiceOffering;
@@ -263,6 +264,31 @@ public class CreateServiceOfferingCmd extends BaseCmd {
             description = "Lease expiry action, valid values are STOP and DESTROY")
     private String leaseExpiryAction;
 
+    @Parameter(name = ApiConstants.VGPU_PROFILE_ID,
+            type = CommandType.UUID,
+            entityType = VgpuProfileResponse.class,
+            description = "the ID of the vGPU profile to which service offering should be mapped",
+            since = "4.21")
+    private Long vgpuProfileId;
+
+    @Parameter(name = ApiConstants.GPU_COUNT,
+            type = CommandType.INTEGER,
+            description = "Count of GPUs to be used with this service offering. This is applicable only when passed with vGPU profile.",
+            since = "4.21")
+    private Integer gpuCount;
+
+    @Parameter(name = ApiConstants.GPU_DISPLAY,
+            type = CommandType.BOOLEAN,
+            description = "Whether to enable GPU display for this service offering. This is applicable only when passed with vGPU profile. Defaults to false.",
+            since = "4.21")
+    private Boolean gpuDisplay;
+
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS,
+            type = CommandType.MAP,
+            description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].endpoint.url=urlvalue",
+            since = "4.21.0")
+    private Map externalDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -369,9 +395,15 @@ public class CreateServiceOfferingCmd extends BaseCmd {
                 }
             }
         }
+
+        detailsMap.putAll(getExternalDetails());
         return detailsMap;
     }
 
+    public Map<String, String> getExternalDetails() {
+        return convertExternalDetailsToMap(externalDetails);
+    }
+
     public Long getRootDiskSize() {
         return rootDiskSize;
     }
@@ -517,6 +549,18 @@ public class CreateServiceOfferingCmd extends BaseCmd {
         return Boolean.TRUE.equals(purgeResources);
     }
 
+    public Long getVgpuProfileId() {
+        return vgpuProfileId;
+    }
+
+    public Integer getGpuCount() {
+        return gpuCount;
+    }
+
+    public Boolean getGpuDisplay() {
+        return Boolean.TRUE.equals(gpuDisplay);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java

@@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.admin.offering;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import com.cloud.offering.ServiceOffering.State;
 import org.apache.cloudstack.api.APICommand;
@@ -94,6 +95,12 @@ public class UpdateServiceOfferingCmd extends BaseCmd {
             since="4.20")
     private Boolean purgeResources;
 
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS,
+            type = CommandType.MAP,
+            description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].endpoint.url=urlvalue",
+            since = "4.21.0")
+    private Map externalDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -194,6 +201,10 @@ public class UpdateServiceOfferingCmd extends BaseCmd {
         return Boolean.TRUE.equals(purgeResources);
     }
 
+    public Map<String, String> getExternalDetails() {
+        return convertExternalDetailsToMap(externalDetails);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java

@@ -103,8 +103,8 @@ public class ListPodsByCmd extends BaseListCmd {
     @Override
     public void execute() {
         Pair<List<? extends Pod>, Integer> result = _mgr.searchForPods(this);
-        ListResponse<PodResponse> response = new ListResponse<PodResponse>();
-        List<PodResponse> podResponses = new ArrayList<PodResponse>();
+        ListResponse<PodResponse> response = new ListResponse<>();
+        List<PodResponse> podResponses = new ArrayList<>();
         for (Pod pod : result.first()) {
             PodResponse podResponse = _responseGenerator.createPodResponse(pod, showCapacities);
             podResponse.setObjectName("pod");

api/src/main/java/org/apache/cloudstack/api/command/admin/vlan/CreateVlanIpRangeCmd.java

@@ -16,6 +16,8 @@
 // under the License.
 package org.apache.cloudstack.api.command.admin.vlan;
 
+import com.cloud.configuration.ConfigurationService;
+import com.cloud.network.Network;
 import com.cloud.utils.net.NetUtils;
 
 import org.apache.cloudstack.api.APICommand;
@@ -39,7 +41,6 @@ import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.user.Account;
 
-import java.util.Objects;
 
 @APICommand(name = "createVlanIpRange", description = "Creates a VLAN IP range.", responseObject = VlanIpRangeResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -114,8 +115,8 @@ public class CreateVlanIpRangeCmd extends BaseCmd {
     @Parameter(name = ApiConstants.FOR_SYSTEM_VMS, type = CommandType.BOOLEAN, description = "true if IP range is set to system vms, false if not")
     private Boolean forSystemVms;
 
-    @Parameter(name = ApiConstants.FOR_NSX, type = CommandType.BOOLEAN, description = "true if the IP range is used for NSX resource", since = "4.20.0")
-    private boolean forNsx;
+    @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, description = "Provider name for which the IP range is reserved for", since = "4.21.0")
+    private String provider;
 
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
@@ -157,12 +158,12 @@ public class CreateVlanIpRangeCmd extends BaseCmd {
         return startIp;
     }
 
-    public boolean isForNsx() {
-        return !Objects.isNull(forNsx) && forNsx;
+    public Network.Provider getProvider() {
+        return Network.Provider.getProvider(provider);
     }
 
     public String getVlan() {
-        if ((vlan == null || vlan.isEmpty()) && !isForNsx()) {
+        if ((vlan == null || vlan.isEmpty()) && !ConfigurationService.IsIpRangeForProvider(getProvider())) {
             vlan = "untagged";
         }
         return vlan;

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/MigrateVMCmd.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.api.command.admin.vm;
 
+import com.cloud.hypervisor.Hypervisor;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 
 import org.apache.cloudstack.api.APICommand;
@@ -145,6 +146,10 @@ public class MigrateVMCmd extends BaseAsyncCmd {
             throw new InvalidParameterValueException("Unable to find the VM by id=" + getVirtualMachineId());
         }
 
+        if (Hypervisor.HypervisorType.External.equals(userVm.getHypervisorType())) {
+            throw new InvalidParameterValueException("Migrate VM instance operation is not allowed for External hypervisor type");
+        }
+
         Host destinationHost = null;
         // OfflineMigration performed when this parameter is specified
         StoragePool destStoragePool = null;

api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java

@@ -17,6 +17,7 @@
 package org.apache.cloudstack.api.command.admin.vpc;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -25,10 +26,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.network.Network;
 import com.cloud.network.VirtualRouterProvider;
+import com.cloud.offering.NetworkOffering;
 import org.apache.cloudstack.api.response.DomainResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.commons.collections.CollectionUtils;
@@ -57,6 +60,7 @@ import static com.cloud.network.Network.Service.SourceNat;
 import static com.cloud.network.Network.Service.PortForwarding;
 import static com.cloud.network.Network.Service.NetworkACL;
 import static com.cloud.network.Network.Service.UserData;
+import static com.cloud.network.Network.Service.Gateway;
 
 @APICommand(name = "createVPCOffering", description = "Creates VPC offering", responseObject = VpcOfferingResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -112,12 +116,19 @@ public class CreateVPCOfferingCmd extends BaseAsyncCreateCmd {
             since = "4.13")
     private List<Long> zoneIds;
 
+    @Deprecated
     @Parameter(name = ApiConstants.FOR_NSX,
             type = CommandType.BOOLEAN,
             description = "true if network offering is meant to be used for NSX, false otherwise.",
             since = "4.20.0")
     private Boolean forNsx;
 
+    @Parameter(name = ApiConstants.PROVIDER,
+            type = CommandType.STRING,
+            description = "Name of the provider providing the service",
+            since = "4.21.0")
+    private String provider;
+
     @Parameter(name = ApiConstants.NSX_SUPPORT_LB,
             type = CommandType.BOOLEAN,
             description = "true if network offering for NSX VPC offering supports Load balancer service.",
@@ -158,20 +169,31 @@ public class CreateVPCOfferingCmd extends BaseAsyncCreateCmd {
         return StringUtils.isEmpty(displayText) ? vpcOfferingName : displayText;
     }
 
+    public boolean isExternalNetworkProvider() {
+        return Arrays.asList("NSX", "Netris").stream()
+                .anyMatch(s -> provider != null && s.equalsIgnoreCase(provider));
+    }
+
     public List<String> getSupportedServices() {
-        if (!isForNsx() && CollectionUtils.isEmpty(supportedServices)) {
+        if (!isExternalNetworkProvider() && CollectionUtils.isEmpty(supportedServices)) {
             throw new InvalidParameterValueException("Supported services needs to be provided");
         }
-        if (isForNsx()) {
+        if (isExternalNetworkProvider()) {
             supportedServices = new ArrayList<>(List.of(
                     Dhcp.getName(),
                     Dns.getName(),
-                    StaticNat.getName(),
-                    SourceNat.getName(),
                     NetworkACL.getName(),
-                    PortForwarding.getName(),
                     UserData.getName()
                     ));
+            if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode())) {
+                supportedServices.addAll(Arrays.asList(
+                        StaticNat.getName(),
+                        SourceNat.getName(),
+                        PortForwarding.getName()));
+            }
+            if (NetworkOffering.NetworkMode.ROUTED.name().equalsIgnoreCase(getNetworkMode())) {
+                supportedServices.add(Gateway.getName());
+            }
             if (getNsxSupportsLbService()) {
                 supportedServices.add(Lb.getName());
             }
@@ -179,8 +201,8 @@ public class CreateVPCOfferingCmd extends BaseAsyncCreateCmd {
         return supportedServices;
     }
 
-    public boolean isForNsx() {
-        return BooleanUtils.isTrue(forNsx);
+    public String getProvider() {
+        return provider;
     }
 
     public String getNetworkMode() {
@@ -193,7 +215,7 @@ public class CreateVPCOfferingCmd extends BaseAsyncCreateCmd {
 
     public Map<String, List<String>> getServiceProviders() {
         Map<String, List<String>> serviceProviderMap = new HashMap<>();
-        if (serviceProviderList != null && !serviceProviderList.isEmpty() && !isForNsx()) {
+        if (serviceProviderList != null && !serviceProviderList.isEmpty() && !isExternalNetworkProvider()) {
             Collection<? extends Map<String, String>> servicesCollection = serviceProviderList.values();
             Iterator<? extends Map<String, String>> iter = servicesCollection.iterator();
             while (iter.hasNext()) {
@@ -213,16 +235,18 @@ public class CreateVPCOfferingCmd extends BaseAsyncCreateCmd {
                 providerList.add(provider);
                 serviceProviderMap.put(service, providerList);
             }
-        } else if (Boolean.TRUE.equals(forNsx)) {
-            getServiceProviderMapForNsx(serviceProviderMap);
+        } else if (isExternalNetworkProvider()) {
+            getServiceProviderMapForExternalProvider(serviceProviderMap, Network.Provider.getProvider(provider).getName());
         }
 
         return serviceProviderMap;
     }
 
-    private void getServiceProviderMapForNsx(Map<String, List<String>> serviceProviderMap) {
-        List<String> unsupportedServices = List.of("Vpn", "BaremetalPxeService", "SecurityGroup", "Connectivity",
-                "Gateway", "Firewall");
+    private void getServiceProviderMapForExternalProvider(Map<String, List<String>> serviceProviderMap, String provider) {
+        List<String> unsupportedServices = new ArrayList<>(List.of("Vpn", "BaremetalPxeService", "SecurityGroup", "Connectivity", "Firewall"));
+        if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode())) {
+            unsupportedServices.add("Gateway");
+        }
         List<String> routerSupported = List.of("Dhcp", "Dns", "UserData");
         List<String> allServices = Network.Service.listAllServices().stream().map(Network.Service::getName).collect(Collectors.toList());
         for (String service : allServices) {
@@ -230,8 +254,10 @@ public class CreateVPCOfferingCmd extends BaseAsyncCreateCmd {
                 continue;
             if (routerSupported.contains(service))
                 serviceProviderMap.put(service, List.of(VirtualRouterProvider.Type.VPCVirtualRouter.name()));
-            else
-                serviceProviderMap.put(service, List.of(Network.Provider.Nsx.getName()));
+            else if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode()) ||
+                    Stream.of(NetworkACL.getName(), Gateway.getName()).anyMatch(s -> s.equalsIgnoreCase(service))) {
+                serviceProviderMap.put(service, List.of(provider));
+            }
         }
         if (!getNsxSupportsLbService()) {
             serviceProviderMap.remove(Lb.getName());

api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java

@@ -108,6 +108,9 @@ public class ListPublicIpAddressesCmd extends BaseListRetrieveOnlyResourceCountC
     @Parameter(name = ApiConstants.FOR_SYSTEM_VMS, type = CommandType.BOOLEAN, description = "true if range is dedicated for system VMs", since = "4.20.0")
     private Boolean forSystemVMs;
 
+    @Parameter(name = ApiConstants.FOR_PROVIDER, type = CommandType.BOOLEAN, description = "true if range is dedicated for external network provider", since = "4.21.0")
+    private Boolean forProvider;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -183,6 +186,10 @@ public class ListPublicIpAddressesCmd extends BaseListRetrieveOnlyResourceCountC
         return BooleanUtils.isTrue(forSystemVMs);
     }
 
+    public boolean isForProvider() {
+        return BooleanUtils.isTrue(forProvider);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java

@@ -43,7 +43,7 @@ import com.cloud.utils.exception.CloudRuntimeException;
         description = "Deletes the backup schedule of a VM",
         responseObject = SuccessResponse.class, since = "4.14.0",
         authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
-public class DeleteBackupScheduleCmd  extends BaseCmd {
+public class DeleteBackupScheduleCmd extends BaseCmd {
 
     @Inject
     private BackupManager backupManager;
@@ -52,17 +52,13 @@ public class DeleteBackupScheduleCmd  extends BaseCmd {
     //////////////// API parameters /////////////////////
     /////////////////////////////////////////////////////
 
-    @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID,
-            type = CommandType.UUID,
-            entityType = UserVmResponse.class,
-            description = "ID of the VM")
+    @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class,
+            description = "ID of the VM from which all backup schedules will be deleted.")
     private Long vmId;
 
-    @Parameter(name = ApiConstants.ID,
-            type = CommandType.UUID,
-            entityType = BackupScheduleResponse.class,
-            description = "ID of the schedule",
-            since = "4.20.1")
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = BackupScheduleResponse.class,
+            since = "4.20.1", description = "ID of the backup schedule to be deleted. It has precedence over the 'virtualmachineid' parameter, " +
+            "i.e., when the 'id' parameter is specified, the 'virtualmachineid' parameter will be ignored.")
     private Long id;
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java

@@ -73,6 +73,8 @@ public class ListCapabilitiesCmd extends BaseCmd {
         response.setSharedFsVmMinCpuCount((Integer)capabilities.get(ApiConstants.SHAREDFSVM_MIN_CPU_COUNT));
         response.setSharedFsVmMinRamSize((Integer)capabilities.get(ApiConstants.SHAREDFSVM_MIN_RAM_SIZE));
         response.setInstanceLeaseEnabled((Boolean) capabilities.get(ApiConstants.INSTANCE_LEASE_ENABLED));
+        response.setExtensionsPath((String)capabilities.get(ApiConstants.EXTENSIONS_PATH));
+        response.setDynamicScalingEnabled((Boolean) capabilities.get(ApiConstants.DYNAMIC_SCALING_ENABLED));
         response.setObjectName("capability");
         response.setResponseName(getCommandName());
         this.setResponseObject(response);

api/src/main/java/org/apache/cloudstack/api/command/user/gpu/ListGpuCardsCmd.java

@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.gpu;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+
+@APICommand(name = "listGpuCards", description = "Lists all available GPU cards",
+        responseObject = GpuCardResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.21.0")
+public class ListGpuCardsCmd extends BaseListCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GpuCardResponse.class,
+            description = "ID of the GPU card")
+    private Long id;
+
+    @Parameter(name = ApiConstants.VENDOR_NAME, type = CommandType.STRING,
+            description = "vendor name of the GPU card")
+    private String vendorName;
+
+    @Parameter(name = ApiConstants.VENDOR_ID, type = CommandType.STRING,
+            description = "vendor ID of the GPU card")
+    private String vendorId;
+
+    @Parameter(name = ApiConstants.DEVICE_ID, type = CommandType.STRING,
+            description = "device ID of the GPU card")
+    private String deviceId;
+
+    @Parameter(name = ApiConstants.DEVICE_NAME, type = CommandType.STRING,
+            description = "device name of the GPU card")
+    private String deviceName;
+
+    @Parameter(name = ApiConstants.ACTIVE_ONLY, type = CommandType.BOOLEAN,
+            description = "If true, only GPU cards which have a device will be listed. If false, all GPU cards will be listed.")
+    private Boolean activeOnly;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getVendorName() {
+        return vendorName;
+    }
+
+    public String getVendorId() {
+        return vendorId;
+    }
+
+    public String getDeviceId() {
+        return deviceId;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public boolean getActiveOnly() {
+        return Boolean.TRUE.equals(activeOnly);
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override public void execute() {
+        ListResponse<GpuCardResponse> response = gpuService.listGpuCards(this);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/gpu/ListGpuDevicesCmd.java

@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.gpu;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ResponseObject;
+import org.apache.cloudstack.api.command.user.UserCmd;
+import org.apache.cloudstack.api.response.GpuDeviceResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.context.CallContext;
+
+@APICommand(name = "listGpuDevices", description = "Lists all available GPU devices",
+        responseView = ResponseObject.ResponseView.Restricted,
+        responseObject = GpuDeviceResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.21.0", authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
+public class ListGpuDevicesCmd extends BaseListCmd implements UserCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class,
+            description = "the virtual machine ID to which the GPU device is allocated")
+    private Long vmId;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getVmId() {
+        return vmId;
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation //////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override
+    public void execute() {
+        CallContext.current().setEventDetails("Listing GPU devices");
+        ListResponse<GpuDeviceResponse> response = gpuService.listGpuDevices(this);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+}

api/src/main/java/org/apache/cloudstack/api/command/user/gpu/ListVgpuProfilesCmd.java

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.gpu;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.GpuCardResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
+
+@APICommand(name = "listVgpuProfiles", description = "Lists all available vGPU profiles",
+        responseObject = VgpuProfileResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.21.0")
+public class ListVgpuProfilesCmd extends BaseListCmd {
+
+    /// //////////////////////////////////////////////////
+    /// ///////////// API parameters /////////////////////
+    /// //////////////////////////////////////////////////
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = VgpuProfileResponse.class,
+            description = "ID of the vGPU profile")
+    private Long id;
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the vGPU profile")
+    private String name;
+
+    @Parameter(name = ApiConstants.GPU_CARD_ID, type = CommandType.UUID, entityType = GpuCardResponse.class,
+            description = "the GPU card ID associated with this GPU device")
+    private Long cardId;
+
+    @Parameter(name = ApiConstants.ACTIVE_ONLY, type = CommandType.BOOLEAN,
+            description = "If true, only vGPU profiles which have a device will be listed. If false, all vGPU profiles will be listed.")
+    private Boolean activeOnly;
+
+    /// //////////////////////////////////////////////////
+    /// //////////////// Accessors ///////////////////////
+    /// //////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public Long getCardId() {
+        return cardId;
+    }
+
+    public boolean getActiveOnly() {
+        return Boolean.TRUE.equals(activeOnly);
+    }
+
+    /// //////////////////////////////////////////////////
+    /// //////////// API Implementation///////////////////
+    /// //////////////////////////////////////////////////
+
+    @Override public void execute() {
+        ListResponse<VgpuProfileResponse> response = gpuService.listVgpuProfiles(this);
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/gui/theme/CreateGuiThemeCmd.java

@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.gui.theme;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GuiThemeResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.gui.theme.GuiTheme;
+import org.apache.cloudstack.gui.theme.GuiThemeJoin;
+import org.apache.cloudstack.gui.theme.GuiThemeService;
+
+import javax.inject.Inject;
+
+@APICommand(name = "createGuiTheme", description = "Creates a customized GUI theme for a set of Common Names (fixed or wildcard), a set of domain UUIDs, and/or a set of " +
+        "account UUIDs.", responseObject = GuiThemeResponse.class, entityType = {GuiTheme.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.21.0.0", authorized = {RoleType.Admin})
+public class CreateGuiThemeCmd extends BaseCmd {
+
+    @Inject
+    GuiThemeService guiThemeService;
+
+    @Parameter(name = ApiConstants.NAME, required = true, type = CommandType.STRING, length = 2048, description = "A name to identify the theme.")
+    private String name;
+
+    @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, length = 4096, description = "A description for the theme.")
+    private String description;
+
+    @Parameter(name = ApiConstants.CSS, type = CommandType.STRING, length = 65535, description = "The CSS to be retrieved and imported into the GUI " +
+            "when matching the theme access configurations.")
+    private String css;
+
+    @Parameter(name = ApiConstants.JSON_CONFIGURATION, type = CommandType.STRING, length = 65535, description = "The JSON with the configurations to be " +
+            "retrieved and imported into the GUI when matching the theme access configurations.")
+    private String jsonConfiguration;
+
+    @Parameter(name = ApiConstants.COMMON_NAMES, type = CommandType.STRING, length = 65535, description = "A set of Common Names (CN) (fixed or " +
+            "wildcard) separated by comma that can retrieve the theme; e.g.: *acme.com,acme2.com")
+    private String commonNames;
+
+    @Parameter(name = ApiConstants.DOMAIN_IDS, type = CommandType.STRING, length = 65535, description = "A set of domain UUIDs (also known as ID for " +
+            "the end-user) separated by comma that can retrieve the theme.")
+    private String domainIds;
+
+    @Parameter(name = ApiConstants.ACCOUNT_IDS, type = CommandType.STRING, length = 65535, description = "A set of account UUIDs (also known as ID for" +
+            " the end-user) separated by comma that can retrieve the theme.")
+    private String accountIds;
+
+    @Parameter(name = ApiConstants.IS_PUBLIC, type = CommandType.BOOLEAN, description = "Defines whether a theme can be retrieved by anyone when only " +
+            "the `commonNames` is informed. If the `domainIds` or `accountIds` is informed, it is considered as `false`.")
+    private Boolean isPublic = true;
+
+    @Parameter(name = ApiConstants.RECURSIVE_DOMAINS, type = CommandType.BOOLEAN, description = "Defines whether the subdomains of the informed domains are considered. Default " +
+            "value is false.")
+    private Boolean recursiveDomains = false;
+
+    public String getName() {
+        return name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public String getCss() {
+        return css;
+    }
+
+    public String getJsonConfiguration() {
+        return jsonConfiguration;
+    }
+
+    public String getCommonNames() {
+        return commonNames;
+    }
+
+    public String getDomainIds() {
+        return domainIds;
+    }
+
+    public String getAccountIds() {
+        return accountIds;
+    }
+
+    public Boolean getPublic() {
+        return isPublic;
+    }
+
+    public Boolean getRecursiveDomains() {
+        return recursiveDomains;
+    }
+
+    @Override
+    public void execute() {
+        GuiThemeJoin guiThemeJoin = guiThemeService.createGuiTheme(this);
+
+        if (guiThemeJoin == null) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create the GUI theme.");
+        }
+
+        GuiThemeResponse response = _responseGenerator.createGuiThemeResponse(guiThemeJoin);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/gui/theme/ListGuiThemesCmd.java

@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.gui.theme;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.AccountResponse;
+import org.apache.cloudstack.api.response.DomainResponse;
+import org.apache.cloudstack.api.response.GuiThemeResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.gui.theme.GuiTheme;
+import org.apache.cloudstack.gui.theme.GuiThemeService;
+
+import javax.inject.Inject;
+
+@APICommand(name = "listGuiThemes", description = "Lists GUI themes.", responseObject = GuiThemeResponse.class, entityType = {GuiTheme.class},
+        since = "4.21.0.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin, RoleType.User, RoleType.DomainAdmin,
+        RoleType.ResourceAdmin})
+public class ListGuiThemesCmd extends BaseListCmd {
+
+    @Inject
+    GuiThemeService guiThemeService;
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, description = "The theme ID.", entityType = GuiThemeResponse.class)
+    private Long id;
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the theme.")
+    private String name;
+
+    @Parameter(name = ApiConstants.COMMON_NAME, type = CommandType.STRING, description = "The internet Common Name (CN) to be filtered.")
+    private String commonName;
+
+    @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "The ID of the domain to be filtered.")
+    private Long domainId;
+
+    @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "The ID of the account to be filtered.")
+    private Long accountId;
+
+    @Parameter(name = ApiConstants.LIST_ALL, type = CommandType.BOOLEAN, description = "Whether to list all themes.")
+    private boolean listAll = false;
+
+    @Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Whether to list removed themes.")
+    private boolean showRemoved = false;
+
+    @Parameter(name = ApiConstants.SHOW_PUBLIC, type = CommandType.BOOLEAN, description = "Whether to list public themes.")
+    private Boolean showPublic;
+
+    @Parameter(name = ApiConstants.LIST_ONLY_DEFAULT_THEME, type = CommandType.BOOLEAN, description = "Whether to only list the default theme.")
+    private boolean listOnlyDefaultTheme = false;
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getCommonName() {
+        return commonName;
+    }
+
+    public Long getDomainId() {
+        return domainId;
+    }
+
+    public Long getAccountId() {
+        return accountId;
+    }
+
+    public boolean getListAll() {
+        return listAll;
+    }
+
+    public boolean getShowRemoved() {
+        return showRemoved;
+    }
+
+    public Boolean getShowPublic() {
+        return showPublic;
+    }
+
+    public boolean getListOnlyDefaultTheme() {
+        return listOnlyDefaultTheme;
+    }
+
+    @Override
+    public void execute() {
+        ListResponse<GuiThemeResponse> response = guiThemeService.listGuiThemes(this);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/gui/theme/RemoveGuiThemeCmd.java

@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.gui.theme;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.GuiThemeResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.gui.theme.GuiTheme;
+import org.apache.cloudstack.gui.theme.GuiThemeService;
+
+import javax.inject.Inject;
+
+@APICommand(name = "removeGuiTheme", description = "Removes an existing GUI theme.", responseObject = GuiThemeResponse.class, entityType = {GuiTheme.class},
+        since = "4.21.0.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
+public class RemoveGuiThemeCmd extends BaseCmd {
+
+    @Inject
+    GuiThemeService guiThemeService;
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuiThemeResponse.class, required = true,
+            description = "The unique identifier of the GUI theme to be removed.")
+    private Long id;
+
+    public Long getId() {
+        return id;
+    }
+
+    @Override
+    public void execute() {
+        guiThemeService.removeGuiTheme(this);
+        final SuccessResponse response = new SuccessResponse();
+        response.setResponseName(getCommandName());
+        response.setSuccess(true);
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/gui/theme/UpdateGuiThemeCmd.java

@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.gui.theme;
+
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.GuiThemeResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.cloudstack.gui.theme.GuiTheme;
+import org.apache.cloudstack.gui.theme.GuiThemeJoin;
+import org.apache.cloudstack.gui.theme.GuiThemeService;
+
+import javax.inject.Inject;
+
+
+@APICommand(name = "updateGuiTheme", description = "Updates an existing GUI theme.", responseObject = GuiThemeResponse.class, entityType = {GuiTheme.class},
+        since = "4.21.0.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin})
+public class UpdateGuiThemeCmd extends BaseCmd {
+
+    @Inject
+    GuiThemeService guiThemeService;
+
+    @Parameter(name = ApiConstants.ID, required = true, type = CommandType.UUID, entityType = GuiThemeResponse.class, description = "The ID of the theme to be updated.")
+    private Long id;
+
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, length = 2048, description = "A name to identify the theme.")
+    private String name;
+
+    @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, length = 4096, description = "A description for the theme.")
+    private String description;
+
+    @Parameter(name = ApiConstants.CSS, type = CommandType.STRING, length = 65535, description = "The CSS to be retrieved and imported into the GUI " +
+            "when matching the theme access configurations.")
+    private String css;
+
+    @Parameter(name = ApiConstants.JSON_CONFIGURATION, type = CommandType.STRING, length = 65535, description = "The JSON with the configurations to be " +
+            "retrieved and imported into the GUI when matching the theme access configurations.")
+    private String jsonConfiguration;
+
+    @Parameter(name = ApiConstants.COMMON_NAMES, type = CommandType.STRING, length = 65535, description = "A set of Common Names (CN) (fixed or " +
+            "wildcard) separated by comma that can retrieve the theme; e.g.: *acme.com,acme2.com")
+    private String commonNames;
+
+    @Parameter(name = ApiConstants.DOMAIN_IDS, type = CommandType.STRING, length = 65535, description = "A set of domain UUIDs (also known as ID for " +
+            "the end-user) separated by comma that can retrieve the theme.")
+    private String domainIds;
+
+    @Parameter(name = ApiConstants.RECURSIVE_DOMAINS, type = CommandType.BOOLEAN, description = "Defines whether the subdomains of the informed domains are considered. Default " +
+            "value is false.")
+    private Boolean recursiveDomains = false;
+
+    @Parameter(name = ApiConstants.ACCOUNT_IDS, type = CommandType.STRING, length = 65535, description = "A set of account UUIDs (also known as ID for" +
+            " the end-user) separated by comma that can retrieve the theme.")
+    private String accountIds;
+
+    @Parameter(name = ApiConstants.IS_PUBLIC, type = CommandType.BOOLEAN, description = "Defines whether a theme can be retrieved by anyone when only " +
+            "the `commonNames` is informed. If the `domainIds` or `accountIds` is informed, it is considered as `false`.")
+    private Boolean isPublic = true;
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public String getCss() {
+        return css;
+    }
+
+    public String getJsonConfiguration() {
+        return jsonConfiguration;
+    }
+
+    public String getCommonNames() {
+        return commonNames;
+    }
+
+    public String getDomainIds() {
+        return domainIds;
+    }
+
+    public String getAccountIds() {
+        return accountIds;
+    }
+
+    public Boolean getRecursiveDomains() {
+        return recursiveDomains;
+    }
+
+    public Boolean getIsPublic() {
+        return isPublic;
+    }
+
+    @Override
+    public void execute() {
+        GuiThemeJoin guiThemeJoin = guiThemeService.updateGuiTheme(this);
+
+        if (guiThemeJoin == null) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the GUI theme.");
+        }
+
+        GuiThemeResponse response = _responseGenerator.createGuiThemeResponse(guiThemeJoin);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return CallContext.current().getCallingAccountId();
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java

@@ -26,6 +26,7 @@ import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
 import org.apache.cloudstack.api.response.TemplateResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.api.response.VgpuProfileResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.commons.lang3.EnumUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -110,6 +111,19 @@ public class ListServiceOfferingsCmd extends BaseListProjectAndAccountResourcesC
             since = "4.20.0")
     private Long templateId;
 
+    @Parameter(name = ApiConstants.VGPU_PROFILE_ID,
+            type = CommandType.UUID,
+            entityType = VgpuProfileResponse.class,
+            description = "The ID of the vGPU profile that listed offerings must support",
+            since = "4.21.0")
+    private Long vgpuProfileId;
+
+    @Parameter(name = ApiConstants.GPU_ENABLED,
+            type = CommandType.BOOLEAN,
+            description = "Flag to indicate if the service offering supports GPU. If set to true, only service offerings that support GPU will be returned.",
+            since = "4.21.0")
+    private Boolean gpuEnabled;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -171,6 +185,14 @@ public class ListServiceOfferingsCmd extends BaseListProjectAndAccountResourcesC
         return templateId;
     }
 
+    public Long getVgpuProfileId() {
+        return vgpuProfileId;
+    }
+
+    public Boolean getGpuEnabled() {
+        return gpuEnabled;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotFromVMSnapshotCmd.java

@@ -153,7 +153,7 @@ public class CreateSnapshotFromVMSnapshotCmd extends BaseAsyncCreateCmd {
 
     @Override
     public void create() throws ResourceAllocationException {
-        Snapshot snapshot = this._volumeService.allocSnapshotForVm(getVmId(), getVolumeId(), getSnapshotName());
+        Snapshot snapshot = this._volumeService.allocSnapshotForVm(getVmId(), getVolumeId(), getSnapshotName(), getVMSnapshotId());
         if (snapshot != null) {
             this.setEntityId(snapshot.getId());
             this.setEntityUuid(snapshot.getUuid());

api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java

@@ -20,6 +20,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.cpu.CPU;
 import org.apache.cloudstack.acl.SecurityChecker;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
@@ -148,6 +149,11 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd {
           since = "4.19.0")
     private String accountName;
 
+    @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING,
+            description = "the CPU arch of the template. Valid options are: x86_64, aarch64. Defaults to x86_64",
+            since = "4.20.2")
+    private String arch;
+
     // ///////////////////////////////////////////////////
     // ///////////////// Accessors ///////////////////////
     // ///////////////////////////////////////////////////
@@ -234,6 +240,10 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd {
         return accountName;
     }
 
+    public CPU.CPUArch getArch() {
+        return CPU.CPUArch.fromType(arch);
+    }
+
     // ///////////////////////////////////////////////////
     // ///////////// API Implementation///////////////////
     // ///////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java

@@ -28,6 +28,7 @@ import org.apache.cloudstack.api.BaseListTaggedResourcesCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.command.user.UserCmd;
+import org.apache.cloudstack.api.response.ExtensionResponse;
 import org.apache.cloudstack.api.response.GuestOSCategoryResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.TemplateResponse;
@@ -115,11 +116,16 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User
             since = "4.20")
     private String arch;
 
-    @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType= GuestOSCategoryResponse.class,
+    @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class,
             description = "the ID of the OS category for the template",
             since = "4.21.0")
     private Long osCategoryId;
 
+    @Parameter(name = ApiConstants.EXTENSION_ID, type = CommandType.UUID, entityType = ExtensionResponse.class,
+            description = "ID of the extension for the template",
+            since = "4.21.0")
+    private Long extensionId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -220,6 +226,10 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User
         return osCategoryId;
     }
 
+    public Long getExtensionId() {
+        return extensionId;
+    }
+
     @Override
     public String getCommandName() {
         return s_name;

api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java

@@ -16,15 +16,12 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.template;
 
-import com.cloud.cpu.CPU;
-import com.cloud.hypervisor.Hypervisor;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.hypervisor.HypervisorGuru;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -35,6 +32,7 @@ import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.DomainResponse;
+import org.apache.cloudstack.api.response.ExtensionResponse;
 import org.apache.cloudstack.api.response.GuestOSResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ProjectResponse;
@@ -43,7 +41,10 @@ import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.commons.lang3.StringUtils;
 
+import com.cloud.cpu.CPU;
 import com.cloud.exception.ResourceAllocationException;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.hypervisor.HypervisorGuru;
 import com.cloud.template.VirtualMachineTemplate;
 
 @APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud. ", responseObject = TemplateResponse.class, responseView = ResponseView.Restricted,
@@ -183,6 +184,14 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
             since = "4.20")
     private String arch;
 
+    @Parameter(name = ApiConstants.EXTENSION_ID, type = CommandType.UUID, entityType = ExtensionResponse.class,
+            description = "ID of the extension",
+            since = "4.21.0")
+    private Long extensionId;
+
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].endpoint.url=urlvalue", since = "4.21.0")
+    protected Map externalDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -312,6 +321,14 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd {
         return CPU.CPUArch.fromType(arch);
     }
 
+    public Long getExtensionId() {
+        return extensionId;
+    }
+
+    public Map<String, String> getExternalDetails() {
+        return convertExternalDetailsToMap(externalDetails);
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java

@@ -33,7 +33,7 @@ import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.user.UserData;
 
 @APICommand(name = "registerUserData",
-        description = "Register a new userdata.",
+        description = "Register a new User Data.",
         since = "4.18",
         responseObject = SuccessResponse.class,
         requestHasSensitiveInfo = false,
@@ -49,7 +49,6 @@ public class RegisterUserDataCmd extends BaseRegisterUserDataCmd {
     @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, required = true, description = "User data content", length = 1048576)
     protected String userData;
 
-
     public String getUserData() {
         return userData;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -16,24 +16,19 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.vm;
 
-import com.cloud.agent.api.LogLevel;
-import com.cloud.event.EventTypes;
-import com.cloud.exception.ConcurrentOperationException;
-import com.cloud.exception.InsufficientCapacityException;
-import com.cloud.exception.InsufficientServerCapacityException;
-import com.cloud.exception.InvalidParameterValueException;
-import com.cloud.exception.ResourceAllocationException;
-import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.network.Network;
-import com.cloud.network.Network.IpAddresses;
-import com.cloud.offering.DiskOffering;
-import com.cloud.template.VirtualMachineTemplate;
-import com.cloud.uservm.UserVm;
-import com.cloud.utils.net.Dhcp;
-import com.cloud.utils.net.NetUtils;
-import com.cloud.vm.VirtualMachine;
-import com.cloud.vm.VmDetailConstants;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import javax.annotation.Nonnull;
+
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.affinity.AffinityGroupResponse;
 import org.apache.cloudstack.api.ACL;
@@ -55,9 +50,11 @@ import org.apache.cloudstack.api.response.NetworkResponse;
 import org.apache.cloudstack.api.response.ProjectResponse;
 import org.apache.cloudstack.api.response.SecurityGroupResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
+import org.apache.cloudstack.api.response.SnapshotResponse;
 import org.apache.cloudstack.api.response.TemplateResponse;
 import org.apache.cloudstack.api.response.UserDataResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.api.response.VolumeResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.vm.lease.VMLeaseManager;
@@ -67,15 +64,25 @@ import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.EnumUtils;
 import org.apache.commons.lang3.StringUtils;
 
-import javax.annotation.Nonnull;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
+import com.cloud.agent.api.LogLevel;
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.InsufficientServerCapacityException;
+import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.network.Network;
+import com.cloud.network.Network.IpAddresses;
+import com.cloud.offering.DiskOffering;
+import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.uservm.UserVm;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.net.Dhcp;
+import com.cloud.utils.net.NetUtils;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VmDetailConstants;
 
 @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts a virtual machine based on a service offering, disk offering, and template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class},
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = true)
@@ -95,7 +102,7 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
     private Long serviceOfferingId;
 
     @ACL
-    @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, required = true, description = "the ID of the template for the virtual machine")
+    @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "the ID of the template for the virtual machine")
     private Long templateId;
 
     @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "host name for the virtual machine", validations = {ApiArgValidator.RFCComplianceDomainName})
@@ -286,6 +293,18 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
             description = "Lease expiry action, valid values are STOP and DESTROY")
     private String leaseExpiryAction;
 
+    @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.UUID, entityType = VolumeResponse.class, since = "4.21")
+    private Long volumeId;
+
+    @Parameter(name = ApiConstants.SNAPSHOT_ID, type = CommandType.UUID, entityType = SnapshotResponse.class, since = "4.21")
+    private Long snapshotId;
+
+    @Parameter(name = ApiConstants.EXTERNAL_DETAILS,
+            type = CommandType.MAP,
+            description = "Details in key/value pairs using format externaldetails[i].keyname=keyvalue. Example: externaldetails[0].server.type=typevalue",
+            since = "4.21.0")
+    protected Map externalDetails;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -359,9 +378,16 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
             customparameterMap.put(VmDetailConstants.NIC_PACKED_VIRTQUEUES_ENABLED, BooleanUtils.toStringTrueFalse(nicPackedVirtQueues));
         }
 
+        if (MapUtils.isNotEmpty(externalDetails)) {
+            customparameterMap.putAll(getExternalDetails());
+        }
+
         return customparameterMap;
     }
 
+    public Map<String, String> getExternalDetails() {
+        return convertExternalDetailsToMap(externalDetails);
+    }
 
     public ApiConstants.BootMode getBootMode() {
         if (StringUtils.isNotBlank(bootMode)) {
@@ -744,6 +770,18 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
         }
         return null;
     }
+
+    public Long getVolumeId() {
+        return volumeId;
+    }
+
+    public Long getSnapshotId() {
+        return snapshotId;
+    }
+
+    public boolean isVolumeOrSnapshotProvided() {
+        return volumeId != null || snapshotId != null;
+    }
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -840,6 +878,10 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
 
     @Override
     public void create() throws ResourceAllocationException {
+        if (Stream.of(templateId, snapshotId, volumeId).filter(Objects::nonNull).count() != 1) {
+            throw new CloudRuntimeException("Please provide only one of the following parameters - template ID, volume ID or snapshot ID");
+        }
+
         try {
             UserVm vm = _userVmService.createVirtualMachine(this);
 

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java

@@ -38,6 +38,7 @@ import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.command.user.UserCmd;
 import org.apache.cloudstack.api.response.AutoScaleVmGroupResponse;
 import org.apache.cloudstack.api.response.BackupOfferingResponse;
+import org.apache.cloudstack.api.response.ExtensionResponse;
 import org.apache.cloudstack.api.response.InstanceGroupResponse;
 import org.apache.cloudstack.api.response.IsoVmResponse;
 import org.apache.cloudstack.api.response.ListResponse;
@@ -171,6 +172,17 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
             since = "4.21.0")
     private Boolean onlyLeasedInstances = false;
 
+    @Parameter(name = ApiConstants.GPU_ENABLED,
+            type = CommandType.BOOLEAN,
+            description = "Flag to indicate if the VMs should be filtered by GPU support. If set to true, only VMs that support GPU will be returned.",
+            since = "4.21.0")
+    private Boolean gpuEnabled;
+
+    @Parameter(name = ApiConstants.EXTENSION_ID, type = CommandType.UUID,
+            entityType = ExtensionResponse.class, description = "The ID of the Orchestrator extension for the VM",
+            since = "4.21.0")
+    private Long extensionId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -318,6 +330,14 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements
         return BooleanUtils.toBoolean(onlyLeasedInstances);
     }
 
+    public Boolean getGpuEnabled() {
+        return gpuEnabled;
+    }
+
+    public Long getExtensionId() {
+        return extensionId;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java

@@ -27,6 +27,7 @@ import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.PrivateGatewayResponse;
 import org.apache.cloudstack.api.response.StaticRouteResponse;
+import org.apache.cloudstack.api.response.VpcResponse;
 import org.apache.cloudstack.context.CallContext;
 
 import com.cloud.event.EventTypes;
@@ -45,20 +46,40 @@ public class CreateStaticRouteCmd extends BaseAsyncCreateCmd {
     @Parameter(name = ApiConstants.GATEWAY_ID,
                type = CommandType.UUID,
                entityType = PrivateGatewayResponse.class,
-               required = true,
-               description = "the gateway id we are creating static route for")
+               description = "the gateway id we are creating static route for. Mutually exclusive with the nexthop parameter")
     private Long gatewayId;
 
+    @Parameter(name = ApiConstants.VPC_ID,
+            type = CommandType.UUID,
+            entityType = VpcResponse.class,
+            description = "the vpc id for which the static route is created. This is required for nexthop parameter",
+            since = "4.21.0")
+    private Long vpcId;
+
+    @Parameter(name = ApiConstants.NEXT_HOP,
+            type = CommandType.STRING,
+            description = "the next hop of static route. Mutually exclusive with the gatewayid parameter",
+            since = "4.21.0")
+    private String nextHop;
+
     @Parameter(name = ApiConstants.CIDR, required = true, type = CommandType.STRING, description = "static route cidr")
     private String cidr;
 
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
-    public long getGatewayId() {
+    public Long getGatewayId() {
         return gatewayId;
     }
 
+    public Long getVpcId() {
+        return vpcId;
+    }
+
+    public String getNextHop() {
+        return nextHop;
+    }
+
     public String getCidr() {
         return cidr;
     }
@@ -69,7 +90,7 @@ public class CreateStaticRouteCmd extends BaseAsyncCreateCmd {
     @Override
     public void create() throws ResourceAllocationException {
         try {
-            StaticRoute result = _vpcService.createStaticRoute(getGatewayId(), getCidr());
+            StaticRoute result = _vpcService.createStaticRoute(getGatewayId(), getVpcId(), getNextHop(), getCidr());
             setEntityId(result.getId());
             setEntityUuid(result.getUuid());
         } catch (NetworkRuleConflictException ex) {
@@ -114,11 +135,8 @@ public class CreateStaticRouteCmd extends BaseAsyncCreateCmd {
 
     @Override
     public long getEntityOwnerId() {
-        VpcGateway gateway = _entityMgr.findById(VpcGateway.class, gatewayId);
-        if (gateway == null) {
-            throw new InvalidParameterValueException("Invalid gateway id is specified");
-        }
-        return _entityMgr.findById(Vpc.class, gateway.getVpcId()).getAccountId();
+        Long vpcId = getSyncObjId();
+        return _entityMgr.findById(Vpc.class, vpcId).getAccountId();
     }
 
     @Override
@@ -128,11 +146,20 @@ public class CreateStaticRouteCmd extends BaseAsyncCreateCmd {
 
     @Override
     public Long getSyncObjId() {
-        VpcGateway gateway = _entityMgr.findById(VpcGateway.class, gatewayId);
-        if (gateway == null) {
-            throw new InvalidParameterValueException("Invalid id is specified for the gateway");
+        if (gatewayId != null) {
+            VpcGateway gateway = _entityMgr.findById(VpcGateway.class, gatewayId);
+            if (gateway == null) {
+                throw new InvalidParameterValueException("Invalid id is specified for the gateway");
+            }
+            return gateway.getVpcId();
+        } else if (vpcId != null) {
+            Vpc vpc = _entityMgr.findById(Vpc.class, vpcId);
+            if (vpc == null) {
+                throw new InvalidParameterValueException("Invalid vpc id is specified");
+            }
+            return vpc.getId();
         }
-        return gateway.getVpcId();
+        throw new InvalidParameterValueException("One of vpcId or gatewayId must be specified");
     }
 
     @Override

api/src/main/java/org/apache/cloudstack/api/command/user/vpn/CreateVpnGatewayCmd.java

@@ -28,6 +28,7 @@ import org.apache.cloudstack.api.BaseAsyncCmd;
 import org.apache.cloudstack.api.BaseAsyncCreateCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.IPAddressResponse;
 import org.apache.cloudstack.api.response.Site2SiteVpnGatewayResponse;
 import org.apache.cloudstack.api.response.VpcResponse;
 import org.apache.cloudstack.context.CallContext;
@@ -44,9 +45,16 @@ public class CreateVpnGatewayCmd extends BaseAsyncCreateCmd {
                type = CommandType.UUID,
                entityType = VpcResponse.class,
                required = true,
-               description = "public ip address id of the vpn gateway")
+               description = "id of the vpc")
     private Long vpcId;
 
+    @Parameter(name = ApiConstants.IP_ADDRESS_ID,
+            type = CommandType.UUID,
+            entityType = IPAddressResponse.class,
+            description = "the public IP address ID for which VPN gateway is being enabled. By default the source NAT IP or router IP will be used.",
+            since = "4.21.0")
+    private Long ipAddressId;
+
     @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "an optional field, whether to the display the vpn to the end user or not", since = "4.4", authorized = {RoleType.Admin})
     private Boolean display;
 
@@ -58,6 +66,10 @@ public class CreateVpnGatewayCmd extends BaseAsyncCreateCmd {
         return vpcId;
     }
 
+    public Long getIpAddressId() {
+        return ipAddressId;
+    }
+
     @Deprecated
     public Boolean getDisplay() {
         return display;