Merge branch 'main' into nsx-integration

.github/workflows/build.yml

@@ -23,6 +23,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-22.04

.github/workflows/ci.yml

@@ -23,6 +23,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     if: github.repository == 'apache/cloudstack'
@@ -108,7 +111,8 @@ jobs:
                   smoke/test_reset_configuration_settings
                   smoke/test_reset_vm_on_reboot
                   smoke/test_resource_accounting
-                  smoke/test_resource_detail",
+                  smoke/test_resource_detail
+                  smoke/test_global_acls",
                 "smoke/test_router_dhcphosts
                   smoke/test_router_dns
                   smoke/test_router_dnsservice

.github/workflows/rat.yml

@@ -23,6 +23,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-22.04

.github/workflows/ui.yml

@@ -23,6 +23,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-22.04

PULL_REQUEST_TEMPLATE.md

@@ -10,10 +10,10 @@ This PR...
 <!-- For addressing multiple issues/PRs, use multiple "Fixes: #<id>" -->
 <!-- Fixes: # -->
 
-<!--- ********************************************************************************* -->
-<!--- NOTE: AUTOMATATION USES THE DESCRIPTIONS TO SET LABELS AND PRODUCE DOCUMENTATION. -->
+<!--- ******************************************************************************* -->
+<!--- NOTE: AUTOMATION USES THE DESCRIPTIONS TO SET LABELS AND PRODUCE DOCUMENTATION. -->
 <!--- PLEASE PUT AN 'X' in only **ONE** box -->
-<!--- ********************************************************************************* -->
+<!--- ******************************************************************************* -->
 
 ### Types of changes
 

agent/conf/agent.properties

@@ -407,3 +407,15 @@ iscsi.session.cleanup.enabled=false
 # The path of an executable file/script for host health check for CloudStack to Auto Disable/Enable the host
 # depending on the return value of the file/script
 # agent.health.check.script.path=
+
+# Time interval (in milliseconds) between KVM heartbeats.
+# kvm.heartbeat.update.frequency=60000
+
+# Number of maximum tries to KVM heartbeats.
+# kvm.heartbeat.update.max.tries=5
+
+# Time amount (in milliseconds) for the KVM heartbeat retry sleep.
+# kvm.heartbeat.update.retry.sleep=10000
+
+# Timeout (in milliseconds) of the KVM heartbeat checker.
+# kvm.heartbeat.checker.timeout=360000

agent/src/main/java/com/cloud/agent/dao/impl/PropertiesStorage.java

@@ -92,11 +92,14 @@ public class PropertiesStorage implements StorageComponent {
             file = new File(path);
             try {
                 if (!file.createNewFile()) {
-                    s_logger.error("Unable to create _file: " + file.getAbsolutePath());
+                    s_logger.error(String.format("Unable to create _file: %s", file.getAbsolutePath()));
                     return false;
                 }
             } catch (IOException e) {
-                s_logger.error("Unable to create _file: " + file.getAbsolutePath(), e);
+                s_logger.error(String.format("Unable to create file: %s", file.getAbsolutePath()));
+                if (s_logger.isDebugEnabled()) {
+                    s_logger.debug(String.format("IOException while trying to create file: %s", file.getAbsolutePath()), e);
+                }
                 return false;
             }
         }

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -539,10 +539,10 @@ public class AgentProperties{
     /**
      * Heartbeat update timeout (in ms).<br>
      * Depending on the use case, this timeout might need increasing/decreasing.<br>
-     * Data type: Integer.<br>
-     * Default value: <code>60000</code>
+     * Data type: Long.<br>
+     * Default value: <code>60000L</code>
      */
-    public static final Property<Integer> HEARTBEAT_UPDATE_TIMEOUT = new Property<>("heartbeat.update.timeout", 60000);
+    public static final Property<Long> HEARTBEAT_UPDATE_TIMEOUT = new Property<>("heartbeat.update.timeout", 60000L);
 
     /**
      * The timeout (in seconds) to retrieve the target's domain ID when migrating a VM with KVM. <br>
@@ -740,6 +740,38 @@ public class AgentProperties{
      */
     public static final Property<String> CONTROL_CIDR = new Property<>("control.cidr", "169.254.0.0/16");
 
+    /**
+     * Time interval (in milliseconds) between KVM heartbeats. <br>
+     * This property is for KVM only.
+     * Data type: Long.<br>
+     * Default value: <code>60000l</code>
+     */
+    public static final Property<Long> KVM_HEARTBEAT_UPDATE_FREQUENCY = new Property<>("kvm.heartbeat.update.frequency", 60000L);
+
+    /**
+     * Number of maximum tries to KVM heartbeats. <br>
+     * This property is for KVM only.
+     * Data type: Long.<br>
+     * Default value: <code>5l</code>
+     */
+    public static final Property<Long> KVM_HEARTBEAT_UPDATE_MAX_TRIES = new Property<>("kvm.heartbeat.update.max.tries", 5L);
+
+    /**
+     * Time amount (in milliseconds) for the KVM heartbeat retry sleep. <br>
+     * This property is for KVM only.
+     * Data type: Long.<br>
+     * Default value: <code>10000l</code>
+     */
+    public static final Property<Long> KVM_HEARTBEAT_UPDATE_RETRY_SLEEP = new Property<>("kvm.heartbeat.update.retry.sleep", 10000L);
+
+    /**
+     * Timeout (in milliseconds) of the KVM heartbeat checker. <br>
+     * This property is for KVM only.
+     * Data type: Long.<br>
+     * Default value: <code>360000l</code>
+     */
+    public static final Property<Long> KVM_HEARTBEAT_CHECKER_TIMEOUT = new Property<>("kvm.heartbeat.checker.timeout", 360000L);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java

@@ -52,6 +52,13 @@ public class StoragePoolInfo {
         this.details = details;
     }
 
+    public StoragePoolInfo(String uuid, String host, String hostPath, String localPath, StoragePoolType poolType, long capacityBytes, long availableBytes,
+            Map<String, String> details, String name) {
+        this(uuid, host, hostPath, localPath, poolType, capacityBytes, availableBytes);
+        this.details = details;
+        this.name = name;
+    }
+
     public long getCapacityBytes() {
         return capacityBytes;
     }

api/src/main/java/com/cloud/network/NetworkService.java

@@ -25,6 +25,8 @@ import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd
 import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd;
 import org.apache.cloudstack.api.command.admin.network.ListGuestVlansCmd;
 import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd;
+import org.apache.cloudstack.api.command.user.address.RemoveQuarantinedIpCmd;
+import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd;
 import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd;
 import org.apache.cloudstack.api.command.user.network.CreateNetworkPermissionsCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkPermissionsCmd;
@@ -115,6 +117,8 @@ public interface NetworkService {
 
     IpAddress getIp(long id);
 
+    IpAddress getIp(String ipAddress);
+
     Network updateGuestNetwork(final UpdateNetworkCmd cmd);
 
     /**
@@ -249,4 +253,8 @@ public interface NetworkService {
     boolean resetNetworkPermissions(ResetNetworkPermissionsCmd resetNetworkPermissionsCmd);
 
     void validateIfServiceOfferingIsActiveAndSystemVmTypeIsDomainRouter(final Long serviceOfferingId) throws InvalidParameterValueException;
+
+    PublicIpQuarantine updatePublicIpAddressInQuarantine(UpdateQuarantinedIpCmd cmd);
+
+    void removePublicIpAddressFromQuarantine(RemoveQuarantinedIpCmd cmd);
 }

api/src/main/java/com/cloud/network/PublicIpQuarantine.java

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+import java.util.Date;
+
+public interface PublicIpQuarantine extends InternalIdentity, Identity {
+    Long getPublicIpAddressId();
+
+    Long getPreviousOwnerId();
+
+    Date getEndDate();
+
+    String getRemovalReason();
+
+    Date getRemoved();
+
+    Date getCreated();
+}

api/src/main/java/com/cloud/network/VNF.java

@@ -68,7 +68,6 @@ public class VNF {
     }
 
     public enum VnfDetail {
-        ICON,
         VERSION,
         VENDOR,
         MAINTAINER

api/src/main/java/com/cloud/server/ResourceTag.java

@@ -30,7 +30,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit
     public enum ResourceObjectType {
         UserVm(true, true, true),
         Template(true, true, true),
-        VnfTemplate(true, true, true),
+        VnfTemplate(false, false, true),
         ISO(true, false, true),
         Volume(true, true),
         Snapshot(true, false),

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -224,6 +224,8 @@ public class ApiConstants {
     public static final String INSTANCES_STATS_USER_ONLY = "instancesstatsuseronly";
     public static final String PREFIX = "prefix";
     public static final String PREVIOUS_ACL_RULE_ID = "previousaclruleid";
+    public static final String PREVIOUS_OWNER_ID = "previousownerid";
+    public static final String PREVIOUS_OWNER_NAME = "previousownername";
     public static final String NEXT_ACL_RULE_ID = "nextaclruleid";
     public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash";
     public static final String IMAGE_PATH = "imagepath";
@@ -403,6 +405,7 @@ public class ApiConstants {
     public static final String SHOW_CAPACITIES = "showcapacities";
     public static final String SHOW_REMOVED = "showremoved";
     public static final String SHOW_RESOURCE_ICON = "showicon";
+    public static final String SHOW_INACTIVE = "showinactive";
     public static final String SHOW_UNIQUE = "showunique";
     public static final String SIGNATURE = "signature";
     public static final String SIGNATURE_VERSION = "signatureversion";
@@ -801,6 +804,7 @@ public class ApiConstants {
     public static final String IPSEC_PSK = "ipsecpsk";
     public static final String GUEST_IP = "guestip";
     public static final String REMOVED = "removed";
+    public static final String REMOVAL_REASON = "removalreason";
     public static final String COMPLETED = "completed";
     public static final String IKE_VERSION = "ikeversion";
     public static final String IKE_POLICY = "ikepolicy";
@@ -1075,6 +1079,10 @@ public class ApiConstants {
     public static final String CLIENT_ID = "clientid";
     public static final String REDIRECT_URI = "redirecturi";
 
+    public static final String IS_TAG_A_RULE = "istagarule";
+
+    public static final String PARAMETER_DESCRIPTION_IS_TAG_A_RULE = "Whether the informed tag is a JS interpretable rule or not.";
+
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
      * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0).

api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java

@@ -64,6 +64,7 @@ import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.HypervisorCapabilitiesResponse;
 import org.apache.cloudstack.api.response.HypervisorGuestOsNamesResponse;
 import org.apache.cloudstack.api.response.IPAddressResponse;
+import org.apache.cloudstack.api.response.IpQuarantineResponse;
 import org.apache.cloudstack.api.response.ImageStoreResponse;
 import org.apache.cloudstack.api.response.InstanceGroupResponse;
 import org.apache.cloudstack.api.response.InternalLoadBalancerElementResponse;
@@ -169,6 +170,7 @@ import com.cloud.network.OvsProvider;
 import com.cloud.network.PhysicalNetwork;
 import com.cloud.network.PhysicalNetworkServiceProvider;
 import com.cloud.network.PhysicalNetworkTrafficType;
+import com.cloud.network.PublicIpQuarantine;
 import com.cloud.network.RemoteAccessVpn;
 import com.cloud.network.RouterHealthCheckResult;
 import com.cloud.network.Site2SiteCustomerGateway;
@@ -529,4 +531,6 @@ public interface ResponseGenerator {
     DirectDownloadCertificateHostStatusResponse createDirectDownloadCertificateProvisionResponse(Long certificateId, Long hostId, Pair<Boolean, String> result);
 
     FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl);
+
+    IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp);
 }

api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java

@@ -119,6 +119,10 @@ public class ListHostsCmd extends BaseListCmd {
         return id;
     }
 
+    public void setId(Long id) {
+        this.id = id;
+    }
+
     public String getHostName() {
         return hostName;
     }

api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java

@@ -60,6 +60,9 @@ public class UpdateHostCmd extends BaseCmd {
     @Parameter(name = ApiConstants.HOST_TAGS, type = CommandType.LIST, collectionType = CommandType.STRING, description = "list of tags to be added to the host")
     private List<String> hostTags;
 
+    @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
+    private Boolean isTagARule;
+
     @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the new uri for the secondary storage: nfs://host/path")
     private String url;
 
@@ -90,6 +93,10 @@ public class UpdateHostCmd extends BaseCmd {
         return hostTags;
     }
 
+    public Boolean getIsTagARule() {
+        return isTagARule;
+    }
+
     public String getUrl() {
         return url;
     }

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java

@@ -90,6 +90,9 @@ public class CreateStoragePoolCmd extends BaseCmd {
                description = "hypervisor type of the hosts in zone that will be attached to this storage pool. KVM, VMware supported as of now.")
     private String hypervisor;
 
+    @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
+    private Boolean isTagARule;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -146,6 +149,10 @@ public class CreateStoragePoolCmd extends BaseCmd {
         return hypervisor;
     }
 
+    public Boolean isTagARule() {
+        return this.isTagARule;
+    }
+
     @Override
     public long getEntityOwnerId() {
         return Account.ACCOUNT_ID_SYSTEM;

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java

@@ -24,6 +24,7 @@ import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseListCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.response.ClusterResponse;
+import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.PodResponse;
 import org.apache.cloudstack.api.response.StoragePoolResponse;
@@ -63,16 +64,25 @@ public class ListStoragePoolsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "the ID of the storage pool")
     private Long id;
 
-    @Parameter(name = ApiConstants.SCOPE, type = CommandType.STRING, entityType = StoragePoolResponse.class, description = "the ID of the storage pool")
+    @Parameter(name = ApiConstants.SCOPE, type = CommandType.STRING, entityType = StoragePoolResponse.class, description = "the scope of the storage pool")
     private String scope;
 
+
     @Parameter(name = ApiConstants.STATUS, type = CommandType.STRING, description = "the status of the storage pool")
     private String status;
 
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "host ID of the storage pools")
+    private Long hostId;
+
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
 
+    public Long getHostId() {
+        return hostId;
+    }
+
     public Long getClusterId() {
         return clusterId;
     }
@@ -81,6 +91,10 @@ public class ListStoragePoolsCmd extends BaseListCmd {
         return ipAddress;
     }
 
+    public void setIpAddress(String ipAddress) {
+        this.ipAddress = ipAddress;
+    }
+
     public String getStoragePoolName() {
         return storagePoolName;
     }
@@ -108,6 +122,15 @@ public class ListStoragePoolsCmd extends BaseListCmd {
     public void setId(Long id) {
         this.id = id;
     }
+
+    public String getScope() {
+        return scope;
+    }
+
+    public void setScope(String scope) {
+        this.scope = scope;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -123,8 +146,4 @@ public class ListStoragePoolsCmd extends BaseListCmd {
         response.setResponseName(getCommandName());
         this.setResponseObject(response);
     }
-
-    public String getScope() {
-        return scope;
-    }
 }

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java

@@ -61,6 +61,9 @@ public class UpdateStoragePoolCmd extends BaseCmd {
             " enable it back.")
     private Boolean enabled;
 
+    @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
+    private Boolean isTagARule;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -89,6 +92,10 @@ public class UpdateStoragePoolCmd extends BaseCmd {
         return enabled;
     }
 
+    public Boolean isTagARule() {
+        return isTagARule;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java

@@ -46,10 +46,14 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
     //////////////// API parameters /////////////////////
     /////////////////////////////////////////////////////
 
-    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, required = true, description = "the ID of the public IP address"
-        + " to disassociate")
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, description = "the ID of the public IP address"
+        + " to disassociate. Mutually exclusive with the ipaddress parameter")
     private Long id;
 
+    @Parameter(name=ApiConstants.IP_ADDRESS, type=CommandType.STRING,  since="4.19.0", description="IP Address to be disassociated."
+        +  " Mutually exclusive with the id parameter")
+    private String ipAddress;
+
     // unexposed parameter needed for events logging
     @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, expose = false)
     private Long ownerId;
@@ -59,7 +63,18 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
     /////////////////////////////////////////////////////
 
     public Long getIpAddressId() {
-        return id;
+       if (id != null & ipAddress != null) {
+           throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter");
+       }
+
+       if (id != null) {
+            return id;
+        } else if (ipAddress != null) {
+            IpAddress ip = getIpAddressByIp(ipAddress);
+            return ip.getId();
+        }
+
+        throw new InvalidParameterValueException("Please specify either IP address or IP address ID");
     }
 
     /////////////////////////////////////////////////////
@@ -68,12 +83,13 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
 
     @Override
     public void execute() throws InsufficientAddressCapacityException {
-        CallContext.current().setEventDetails("IP ID: " + getIpAddressId());
+        Long ipAddressId = getIpAddressId();
+        CallContext.current().setEventDetails("IP ID: " + ipAddressId);
         boolean result = false;
-        if (!isPortable(id)) {
-            result = _networkService.releaseIpAddress(getIpAddressId());
+        if (!isPortable()) {
+            result = _networkService.releaseIpAddress(ipAddressId);
         } else {
-            result = _networkService.releasePortableIpAddress(getIpAddressId());
+            result = _networkService.releasePortableIpAddress(ipAddressId);
         }
         if (result) {
             SuccessResponse response = new SuccessResponse(getCommandName());
@@ -85,7 +101,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
 
     @Override
     public String getEventType() {
-        if (!isPortable(id)) {
+        if (!isPortable()) {
             return EventTypes.EVENT_NET_IP_RELEASE;
         } else {
             return EventTypes.EVENT_PORTABLE_IP_RELEASE;
@@ -100,10 +116,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
     @Override
     public long getEntityOwnerId() {
         if (ownerId == null) {
-            IpAddress ip = getIpAddress(id);
-            if (ip == null) {
-                throw new InvalidParameterValueException("Unable to find IP address by ID=" + id);
-            }
+            IpAddress ip = getIpAddress();
             ownerId = ip.getAccountId();
         }
 
@@ -120,11 +133,11 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
 
     @Override
     public Long getSyncObjId() {
-        IpAddress ip = getIpAddress(id);
+        IpAddress ip = getIpAddress();
         return ip.getAssociatedWithNetworkId();
     }
 
-    private IpAddress getIpAddress(long id) {
+    private IpAddress getIpAddressById(Long id) {
         IpAddress ip = _entityMgr.findById(IpAddress.class, id);
 
         if (ip == null) {
@@ -134,6 +147,29 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
         }
     }
 
+    private IpAddress getIpAddressByIp(String ipAddress) {
+        IpAddress ip = _networkService.getIp(ipAddress);
+        if (ip == null) {
+            throw new InvalidParameterValueException("Unable to find IP address by IP address=" + ipAddress);
+        } else {
+            return ip;
+        }
+    }
+
+    private IpAddress getIpAddress() {
+        if (id != null & ipAddress != null) {
+            throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter");
+        }
+
+        if (id != null) {
+            return getIpAddressById(id);
+        } else if (ipAddress != null){
+            return getIpAddressByIp(ipAddress);
+        }
+
+        throw new InvalidParameterValueException("Please specify either IP address or IP address ID");
+    }
+
     @Override
     public ApiCommandResourceType getApiResourceType() {
         return ApiCommandResourceType.IpAddress;
@@ -144,8 +180,8 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd {
         return getIpAddressId();
     }
 
-    private boolean isPortable(long id) {
-        IpAddress ip = getIpAddress(id);
+    private boolean isPortable() {
+        IpAddress ip = getIpAddress();
         return ip.isPortable();
     }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/address/ListQuarantinedIpsCmd.java

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.address;
+
+import com.cloud.network.PublicIpQuarantine;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.IpQuarantineResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+
+@APICommand(name = "listQuarantinedIps", responseObject = IpQuarantineResponse.class, description = "List public IP addresses in quarantine.", since = "4.19",
+        entityType = {PublicIpQuarantine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin, RoleType.DomainAdmin})
+public class ListQuarantinedIpsCmd extends BaseListCmd {
+
+    @Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Show IPs removed from quarantine.")
+    private boolean showRemoved = false;
+
+    @Parameter(name = ApiConstants.SHOW_INACTIVE, type = CommandType.BOOLEAN, description = "Show IPs that are no longer in quarantine.")
+    private boolean showInactive = false;
+
+    public boolean isShowRemoved() {
+        return showRemoved;
+    }
+    public boolean isShowInactive() {
+        return showInactive;
+    }
+
+    @Override
+    public void execute() {
+        ListResponse<IpQuarantineResponse> response = _queryService.listQuarantinedIps(this);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/address/RemoveQuarantinedIpCmd.java

@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.address;
+
+import com.cloud.network.PublicIpQuarantine;
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.response.IpQuarantineResponse;
+import org.apache.cloudstack.api.response.SuccessResponse;
+
+@APICommand(name = "removeQuarantinedIp", responseObject = IpQuarantineResponse.class, description = "Removes a public IP address from quarantine. Only IPs in active " +
+        "quarantine can be removed.",
+        since = "4.19", entityType = {PublicIpQuarantine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.DomainAdmin})
+public class RemoveQuarantinedIpCmd extends BaseCmd {
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IpQuarantineResponse.class, description = "The ID of the public IP address in active quarantine. " +
+            "Either the IP address is informed, or the ID of the IP address in quarantine.")
+    private Long id;
+
+    @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "The public IP address in active quarantine. Either the IP address is informed, or the ID" +
+            " of the IP address in quarantine.")
+    private String ipAddress;
+
+    @Parameter(name = ApiConstants.REMOVAL_REASON, type = CommandType.STRING, required = true, description = "The reason for removing the public IP address from quarantine " +
+            "prematurely.")
+    private String removalReason;
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getIpAddress() {
+        return ipAddress;
+    }
+
+    public String getRemovalReason() {
+        return removalReason;
+    }
+
+    @Override
+    public void execute() {
+        _networkService.removePublicIpAddressFromQuarantine(this);
+        final SuccessResponse response = new SuccessResponse();
+        response.setResponseName(getCommandName());
+        response.setSuccess(true);
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/address/UpdateQuarantinedIpCmd.java

@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.address;
+
+import com.cloud.network.PublicIpQuarantine;
+import com.cloud.user.Account;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.IpQuarantineResponse;
+
+import java.util.Date;
+
+@APICommand(name = "updateQuarantinedIp", responseObject = IpQuarantineResponse.class, description = "Updates the quarantine end date for the given public IP address.",
+        since = "4.19", entityType = {PublicIpQuarantine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.DomainAdmin})
+public class UpdateQuarantinedIpCmd extends BaseCmd {
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IpQuarantineResponse.class, description = "The ID of the public IP address in " +
+            "active quarantine.")
+    private Long id;
+
+    @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "The public IP address in active quarantine. Either the IP address is informed, or the ID" +
+            " of the IP address in quarantine.")
+    private String ipAddress;
+
+    @Parameter(name = ApiConstants.END_DATE, type = BaseCmd.CommandType.DATE, required = true, description = "The date when the quarantine will no longer be active.")
+    private Date endDate;
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getIpAddress() {
+        return ipAddress;
+    }
+
+    public Date getEndDate() {
+        return endDate;
+    }
+
+    @Override
+    public void execute() {
+        PublicIpQuarantine publicIpQuarantine = _networkService.updatePublicIpAddressInQuarantine(this);
+        if (publicIpQuarantine == null) {
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update public IP quarantine.");
+        }
+        IpQuarantineResponse response = _responseGenerator.createQuarantinedIpsResponse(publicIpQuarantine);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java

@@ -177,6 +177,9 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd {
     }
 
     public Long getZoneId() {
+        if (zoneId == null || zoneId == -1) {
+            return null;
+        }
         return zoneId;
     }
 
@@ -220,6 +223,10 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd {
         return directDownload == null ? false : directDownload;
     }
 
+    public void setDirectDownload(Boolean directDownload) {
+        this.directDownload = directDownload;
+    }
+
     public boolean isPasswordEnabled() {
         return passwordEnabled == null ? false : passwordEnabled;
     }

api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.network;
 
+import com.cloud.exception.PermissionDeniedException;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
@@ -26,6 +27,7 @@ import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.response.NetworkACLResponse;
 import org.apache.cloudstack.api.response.VpcResponse;
+import org.apache.cloudstack.context.CallContext;
 import org.apache.log4j.Logger;
 
 import com.cloud.event.EventTypes;
@@ -35,7 +37,8 @@ import com.cloud.network.vpc.NetworkACL;
 import com.cloud.network.vpc.Vpc;
 import com.cloud.user.Account;
 
-@APICommand(name = "createNetworkACLList", description = "Creates a network ACL for the given VPC", responseObject = NetworkACLResponse.class,
+@APICommand(name = "createNetworkACLList", description = "Creates a network ACL. If no VPC is given, then it creates a global ACL that can be used by everyone.",
+        responseObject = NetworkACLResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
 public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd {
     public static final Logger s_logger = Logger.getLogger(CreateNetworkACLListCmd.class.getName());
@@ -53,7 +56,6 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd {
 
     @Parameter(name = ApiConstants.VPC_ID,
                type = CommandType.UUID,
-               required = true,
                entityType = VpcResponse.class,
                description = "ID of the VPC associated with this network ACL list")
     private Long vpcId;
@@ -77,6 +79,10 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd {
         return vpcId;
     }
 
+    public void setVpcId(Long vpcId) {
+        this.vpcId = vpcId;
+    }
+
     @Override
     public boolean isDisplay() {
         if (display != null) {
@@ -92,6 +98,9 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd {
 
     @Override
     public void create() {
+        if (getVpcId() == null) {
+            setVpcId(0L);
+        }
         NetworkACL result = _networkACLService.createNetworkACL(getName(), getDescription(), getVpcId(), isDisplay());
         setEntityId(result.getId());
         setEntityUuid(result.getUuid());
@@ -111,12 +120,21 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd {
 
     @Override
     public long getEntityOwnerId() {
-        Vpc vpc = _entityMgr.findById(Vpc.class, getVpcId());
-        if (vpc == null) {
-            throw new InvalidParameterValueException("Invalid vpcId is given");
-        }
+        Account account;
+        if (isAclAttachedToVpc(this.vpcId)) {
+            Vpc vpc = _entityMgr.findById(Vpc.class, this.vpcId);
+            if (vpc == null) {
+                throw new InvalidParameterValueException(String.format("Invalid VPC ID [%s] provided.", this.vpcId));
+            }
+            account = _accountService.getAccount(vpc.getAccountId());
+        } else {
+            account = CallContext.current().getCallingAccount();
+            if (!Account.Type.ADMIN.equals(account.getType())) {
+                s_logger.warn(String.format("Only Root Admin can create global ACLs. Account [%s] cannot create any global ACL.", account));
+                throw new PermissionDeniedException("Only Root Admin can create global ACLs.");
+            }
 
-        Account account = _accountService.getAccount(vpc.getAccountId());
+        }
         return account.getId();
     }
 
@@ -139,4 +157,8 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd {
     public ApiCommandResourceType getApiResourceType() {
         return ApiCommandResourceType.NetworkAcl;
     }
+
+    public boolean isAclAttachedToVpc(Long aclVpcId) {
+        return aclVpcId != null && aclVpcId != 0;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.offering;
 
+import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd;
 import org.apache.cloudstack.api.response.StoragePoolResponse;
 import org.apache.cloudstack.api.response.VolumeResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
@@ -23,14 +24,14 @@ import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.BaseListDomainResourcesCmd;
 import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.BaseCmd.CommandType;
 import org.apache.cloudstack.api.response.DiskOfferingResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 
 @APICommand(name = "listDiskOfferings", description = "Lists all available disk offerings.", responseObject = DiskOfferingResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class ListDiskOfferingsCmd extends BaseListDomainResourcesCmd {
+public class ListDiskOfferingsCmd extends BaseListProjectAndAccountResourcesCmd {
     public static final Logger s_logger = Logger.getLogger(ListDiskOfferingsCmd.class.getName());
 
 
@@ -60,6 +61,12 @@ public class ListDiskOfferingsCmd extends BaseListDomainResourcesCmd {
     @Parameter(name = ApiConstants.ENCRYPT, type = CommandType.BOOLEAN, description = "listed offerings support disk encryption", since = "4.18")
     private Boolean encrypt;
 
+    @Parameter(name = ApiConstants.STORAGE_TYPE,
+            type = CommandType.STRING,
+            description = "the storage type of the service offering. Values are local and shared.",
+            since = "4.19")
+    private String storageType;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -84,6 +91,10 @@ public class ListDiskOfferingsCmd extends BaseListDomainResourcesCmd {
 
     public Boolean getEncrypt() { return encrypt; }
 
+    public String getStorageType() {
+        return storageType;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java

@@ -16,20 +16,21 @@
 // under the License.
 package org.apache.cloudstack.api.command.user.offering;
 
+import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.BaseListDomainResourcesCmd;
 import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.BaseCmd.CommandType;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ServiceOfferingResponse;
 import org.apache.cloudstack.api.response.UserVmResponse;
 
 @APICommand(name = "listServiceOfferings", description = "Lists all available service offerings.", responseObject = ServiceOfferingResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd {
+public class ListServiceOfferingsCmd extends BaseListProjectAndAccountResourcesCmd {
     public static final Logger s_logger = Logger.getLogger(ListServiceOfferingsCmd.class.getName());
 
 
@@ -88,6 +89,12 @@ public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd {
         since = "4.18")
     private Boolean encryptRoot;
 
+    @Parameter(name = ApiConstants.STORAGE_TYPE,
+            type = CommandType.STRING,
+            description = "the storage type of the service offering. Values are local and shared.",
+            since = "4.19")
+    private String storageType;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -130,6 +137,10 @@ public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd {
 
     public Boolean getEncryptRoot() { return encryptRoot; }
 
+    public String getStorageType() {
+        return storageType;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java

@@ -21,18 +21,6 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.cloudstack.acl.SecurityChecker;
-import org.apache.cloudstack.api.command.user.UserCmd;
-import org.apache.cloudstack.api.response.GuestOSResponse;
-import org.apache.cloudstack.api.response.SnapshotResponse;
-import org.apache.cloudstack.api.response.TemplateResponse;
-import org.apache.cloudstack.api.response.UserVmResponse;
-import org.apache.cloudstack.api.response.VolumeResponse;
-import org.apache.cloudstack.api.response.ProjectResponse;
-
-import org.apache.cloudstack.api.response.ZoneResponse;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Logger;
-
 import org.apache.cloudstack.api.APICommand;
 import org.apache.cloudstack.api.ApiCommandResourceType;
 import org.apache.cloudstack.api.ApiConstants;
@@ -41,13 +29,23 @@ import org.apache.cloudstack.api.BaseAsyncCreateCmd;
 import org.apache.cloudstack.api.Parameter;
 import org.apache.cloudstack.api.ResponseObject.ResponseView;
 import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.command.user.UserCmd;
+import org.apache.cloudstack.api.response.DomainResponse;
+import org.apache.cloudstack.api.response.GuestOSResponse;
+import org.apache.cloudstack.api.response.ProjectResponse;
+import org.apache.cloudstack.api.response.SnapshotResponse;
+import org.apache.cloudstack.api.response.TemplateResponse;
+import org.apache.cloudstack.api.response.UserVmResponse;
+import org.apache.cloudstack.api.response.VolumeResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.context.CallContext;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
 
 import com.cloud.event.EventTypes;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceAllocationException;
-import com.cloud.projects.Project;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.Volume;
 import com.cloud.template.VirtualMachineTemplate;
@@ -139,6 +137,19 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd {
     @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the zone for the template. Can be specified with snapshot only", since = "4.19.0")
     private Long zoneId;
 
+    @Parameter(name = ApiConstants.DOMAIN_ID,
+          type = CommandType.UUID,
+          entityType = DomainResponse.class,
+          description = "an optional domainId. If the account parameter is used, domainId must also be used.",
+          since = "4.19.0")
+    private Long domainId;
+
+    @Parameter(name = ApiConstants.ACCOUNT,
+          type = CommandType.STRING,
+          description = "an optional accountName. Must be used with domainId.",
+          since = "4.19.0")
+    private String accountName;
+
     // ///////////////////////////////////////////////////
     // ///////////////// Accessors ///////////////////////
     // ///////////////////////////////////////////////////
@@ -217,6 +228,14 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd {
         return zoneId;
     }
 
+    public Long getDomainId() {
+        return domainId;
+    }
+
+    public String getAccountName() {
+        return accountName;
+    }
+
     // ///////////////////////////////////////////////////
     // ///////////// API Implementation///////////////////
     // ///////////////////////////////////////////////////
@@ -232,47 +251,12 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd {
 
     @Override
     public long getEntityOwnerId() {
-        Long volumeId = getVolumeId();
-        Long snapshotId = getSnapshotId();
         Account callingAccount = CallContext.current().getCallingAccount();
-        if (volumeId != null) {
-            Volume volume = _entityMgr.findById(Volume.class, volumeId);
-            if (volume != null) {
-                _accountService.checkAccess(callingAccount, SecurityChecker.AccessType.UseEntry, false, volume);
-            } else {
-                throw new InvalidParameterValueException("Unable to find volume by id=" + volumeId);
-            }
-        } else {
-            Snapshot snapshot = _entityMgr.findById(Snapshot.class, snapshotId);
-            if (snapshot != null) {
-                _accountService.checkAccess(callingAccount, SecurityChecker.AccessType.UseEntry, false, snapshot);
-            } else {
-                throw new InvalidParameterValueException("Unable to find snapshot by id=" + snapshotId);
-            }
-        }
-
-        if(projectId != null){
-            final Project project = _projectService.getProject(projectId);
-            if (project != null) {
-                if (project.getState() == Project.State.Active) {
-                    Account projectAccount= _accountService.getAccount(project.getProjectAccountId());
-                    _accountService.checkAccess(callingAccount, SecurityChecker.AccessType.UseEntry, false, projectAccount);
-                    return project.getProjectAccountId();
-                } else {
-                    final PermissionDeniedException ex =
-                            new PermissionDeniedException("Can't add resources to the project with specified projectId in state=" + project.getState() +
-                                    " as it's no longer active");
-                    ex.addProxyObject(project.getUuid(), "projectId");
-                    throw ex;
-                }
-            } else {
-                throw new InvalidParameterValueException("Unable to find project by id");
-            }
-        }
-
-        return callingAccount.getId();
+        ensureAccessCheck(callingAccount);
+        return findAccountIdToUse(callingAccount);
     }
 
+
     @Override
     public String getEventType() {
         return EventTypes.EVENT_TEMPLATE_CREATE;
@@ -330,4 +314,47 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd {
         }
 
     }
+
+    /***
+     * Performs access check on volume and snapshot for given account
+     * @param account
+     */
+    private void ensureAccessCheck(Account account) {
+        if (volumeId != null) {
+            Volume volume = _entityMgr.findById(Volume.class, volumeId);
+            if (volume != null) {
+                _accountService.checkAccess(account, SecurityChecker.AccessType.UseEntry, false, volume);
+            } else {
+                throw new InvalidParameterValueException("Unable to find volume by id=" + volumeId);
+            }
+        } else {
+            Snapshot snapshot = _entityMgr.findById(Snapshot.class, snapshotId);
+            if (snapshot != null) {
+                _accountService.checkAccess(account, SecurityChecker.AccessType.UseEntry, false, snapshot);
+            } else {
+                throw new InvalidParameterValueException("Unable to find snapshot by id=" + snapshotId);
+            }
+        }
+    }
+
+    /***
+     * Find accountId based on accountName and domainId or projectId
+     * if not found, return callingAccountId for further use
+     * @param callingAccount
+     * @return accountId
+     */
+    private Long findAccountIdToUse(Account callingAccount) {
+        Long accountIdToUse = null;
+        try {
+            accountIdToUse = _accountService.finalyzeAccountId(accountName, domainId, projectId, true);
+        } catch (InvalidParameterValueException | PermissionDeniedException ex) {
+            if (s_logger.isDebugEnabled()) {
+                s_logger.debug(String.format("An exception occurred while finalizing account id with accountName, domainId and projectId" +
+                      "using callingAccountId=%s", callingAccount.getUuid()), ex);
+            }
+            s_logger.warn("Unable to find accountId associated with accountName=" + accountName + " and domainId="
+                  + domainId + " or projectId=" + projectId + ", using callingAccountId=" + callingAccount.getUuid());
+        }
+        return accountIdToUse != null ? accountIdToUse : callingAccount.getAccountId();
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java

@@ -124,7 +124,7 @@ public class AcquireIPAddressResponse  extends BaseResponse implements Controlle
  private String networkId;
 
  @SerializedName(ApiConstants.STATE)
- @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing")
+ @Param(description = "State of the ip address. Can be: Allocating, Allocated and Releasing")
  private String state;
 
  @SerializedName(ApiConstants.PHYSICAL_NETWORK_ID)

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -221,6 +221,10 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "comma-separated list of tags for the host")
     private String hostTags;
 
+    @SerializedName(ApiConstants.IS_TAG_A_RULE)
+    @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
+    private Boolean isTagARule;
+
     @SerializedName("hasenoughcapacity")
     @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise")
     private Boolean hasEnoughCapacity;
@@ -732,4 +736,12 @@ public class HostResponse extends BaseResponseWithAnnotations {
     public void setEncryptionSupported(Boolean encryptionSupported) {
         this.encryptionSupported = encryptionSupported;
     }
+
+    public Boolean getIsTagARule() {
+        return isTagARule;
+    }
+
+    public void setIsTagARule(Boolean tagARule) {
+        isTagARule = tagARule;
+    }
 }

api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java

@@ -128,7 +128,7 @@ public class IPAddressResponse extends BaseResponseWithAnnotations implements Co
     private String networkId;
 
     @SerializedName(ApiConstants.STATE)
-    @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing")
+    @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing, Reserved and Free")
     private String state;
 
     @SerializedName(ApiConstants.PHYSICAL_NETWORK_ID)

api/src/main/java/org/apache/cloudstack/api/response/IpQuarantineResponse.java

@@ -0,0 +1,130 @@
+//Licensed to the Apache Software Foundation (ASF) under one
+//or more contributor license agreements.  See the NOTICE file
+//distributed with this work for additional information
+//regarding copyright ownership.  The ASF licenses this file
+//to you under the Apache License, Version 2.0 (the
+//"License"); you may not use this file except in compliance
+//with the License.  You may obtain a copy of the License at
+//
+//http://www.apache.org/licenses/LICENSE-2.0
+//
+//Unless required by applicable law or agreed to in writing,
+//software distributed under the License is distributed on an
+//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+//KIND, either express or implied.  See the License for the
+//specific language governing permissions and limitations
+//under the License.
+package org.apache.cloudstack.api.response;
+
+import com.cloud.network.PublicIpQuarantine;
+import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseResponse;
+import org.apache.cloudstack.api.EntityReference;
+
+import java.util.Date;
+
+@EntityReference(value = {PublicIpQuarantine.class})
+public class IpQuarantineResponse extends BaseResponse {
+
+    @SerializedName(ApiConstants.ID)
+    @Param(description = "ID of the quarantine process.")
+    private String id;
+
+    @SerializedName(ApiConstants.IP_ADDRESS)
+    @Param(description = "The public IP address in quarantine.")
+    private String publicIpAddress;
+
+    @SerializedName(ApiConstants.PREVIOUS_OWNER_ID)
+    @Param(description = "Account ID of the previous public IP address owner.")
+    private String previousOwnerId;
+
+    @SerializedName(ApiConstants.PREVIOUS_OWNER_NAME)
+    @Param(description = "Account name of the previous public IP address owner.")
+    private String previousOwnerName;
+
+    @SerializedName(ApiConstants.CREATED)
+    @Param(description = "When the quarantine was created.")
+    private Date created;
+
+    @SerializedName(ApiConstants.REMOVED)
+    @Param(description = "When the quarantine was removed.")
+    private Date removed;
+
+    @SerializedName(ApiConstants.END_DATE)
+    @Param(description = "End date for the quarantine.")
+    private Date endDate;
+
+    @SerializedName(ApiConstants.REMOVAL_REASON)
+    @Param(description = "The reason for removing the IP from quarantine prematurely.")
+    private String removalReason;
+
+    public IpQuarantineResponse() {
+        super("quarantinedips");
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getPublicIpAddress() {
+        return publicIpAddress;
+    }
+
+    public void setPublicIpAddress(String publicIpAddress) {
+        this.publicIpAddress = publicIpAddress;
+    }
+
+    public String getPreviousOwnerId() {
+        return previousOwnerId;
+    }
+
+    public void setPreviousOwnerId(String previousOwnerId) {
+        this.previousOwnerId = previousOwnerId;
+    }
+
+    public String getPreviousOwnerName() {
+        return previousOwnerName;
+    }
+
+    public void setPreviousOwnerName(String previousOwnerName) {
+        this.previousOwnerName = previousOwnerName;
+    }
+
+    public Date getCreated() {
+        return created;
+    }
+
+    public void setCreated(Date created) {
+        this.created = created;
+    }
+
+    public Date getRemoved() {
+        return removed;
+    }
+
+    public void setRemoved(Date removed) {
+        this.removed = removed;
+    }
+
+    public Date getEndDate() {
+        return endDate;
+    }
+
+    public void setEndDate(Date endDate) {
+        this.endDate = endDate;
+    }
+
+    public String getRemovalReason() {
+        return removalReason;
+    }
+
+    public void setRemovalReason(String removalReason) {
+        this.removalReason = removalReason;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java

@@ -68,7 +68,7 @@ public class PortableIpResponse extends BaseResponse {
     private Date allocated;
 
     @SerializedName(ApiConstants.STATE)
-    @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing")
+    @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing and Free")
     private String state;
 
     public void setRegionId(Integer regionId) {

api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java

@@ -101,6 +101,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
     @Param(description = "the tags for the storage pool")
     private String tags;
 
+    @SerializedName(ApiConstants.IS_TAG_A_RULE)
+    @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
+    private Boolean isTagARule;
+
     @SerializedName(ApiConstants.STATE)
     @Param(description = "the state of the storage pool")
     private StoragePoolStatus state;
@@ -304,6 +308,14 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations {
         this.tags = tags;
     }
 
+    public Boolean getIsTagARule() {
+        return isTagARule;
+    }
+
+    public void setIsTagARule(Boolean tagARule) {
+        isTagARule = tagARule;
+    }
+
     public StoragePoolStatus getState() {
         return state;
     }

api/src/main/java/org/apache/cloudstack/query/QueryService.java

@@ -34,6 +34,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd;
 import org.apache.cloudstack.api.command.admin.user.ListUsersCmd;
 import org.apache.cloudstack.api.command.user.account.ListAccountsCmd;
 import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd;
+import org.apache.cloudstack.api.command.user.address.ListQuarantinedIpsCmd;
 import org.apache.cloudstack.api.command.user.affinitygroup.ListAffinityGroupsCmd;
 import org.apache.cloudstack.api.command.user.event.ListEventsCmd;
 import org.apache.cloudstack.api.command.user.iso.ListIsosCmd;
@@ -64,6 +65,7 @@ import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.HostTagResponse;
 import org.apache.cloudstack.api.response.ImageStoreResponse;
 import org.apache.cloudstack.api.response.InstanceGroupResponse;
+import org.apache.cloudstack.api.response.IpQuarantineResponse;
 import org.apache.cloudstack.api.response.ListResponse;
 import org.apache.cloudstack.api.response.ManagementServerResponse;
 import org.apache.cloudstack.api.response.ProjectAccountResponse;
@@ -183,6 +185,8 @@ public interface QueryService {
 
     List<RouterHealthCheckResultResponse> listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd);
 
+    ListResponse<IpQuarantineResponse> listQuarantinedIps(ListQuarantinedIpsCmd cmd);
+
     ListResponse<SnapshotResponse> listSnapshots(ListSnapshotsCmd cmd);
 
     SnapshotResponse listSnapshot(CopySnapshotCmd cmd);

api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java

@@ -29,4 +29,20 @@ public class CreateTemplateCmdTest {
         ReflectionTestUtils.setField(cmd, "zoneId", id);
         Assert.assertEquals(id, cmd.getZoneId());
     }
+
+    @Test
+    public void testDomainId() {
+        final CreateTemplateCmd cmd = new CreateTemplateCmd();
+        Long id = 2L;
+        ReflectionTestUtils.setField(cmd, "domainId", id);
+        Assert.assertEquals(id, cmd.getDomainId());
+    }
+
+    @Test
+    public void testGetAccountName() {
+        final CreateTemplateCmd cmd = new CreateTemplateCmd();
+        String accountName = "user1";
+        ReflectionTestUtils.setField(cmd, "accountName", accountName);
+        Assert.assertEquals(accountName, cmd.getAccountName());
+    }
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java

@@ -43,6 +43,8 @@ public class PrimaryDataStoreParameters {
     private boolean managed;
     private Long capacityIops;
 
+    private Boolean isTagARule;
+
     /**
      * @return the userInfo
      */
@@ -277,4 +279,12 @@ public class PrimaryDataStoreParameters {
     public void setUsedBytes(long usedBytes) {
         this.usedBytes = usedBytes;
     }
+
+    public Boolean isTagARule() {
+        return isTagARule;
+    }
+
+    public void setIsTagARule(Boolean isTagARule) {
+        this.isTagARule = isTagARule;
+    }
 }

engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.network;
 
+import java.util.Date;
 import java.util.List;
 
 import org.apache.cloudstack.api.response.AcquirePodIpCmdResponse;
@@ -238,5 +239,52 @@ public interface IpAddressManager {
     public static final String MESSAGE_ASSIGN_IPADDR_EVENT = "Message.AssignIpAddr.Event";
     public static final String MESSAGE_RELEASE_IPADDR_EVENT = "Message.ReleaseIpAddr.Event";
 
+
+    /**
+     * Checks if the given public IP address is not in active quarantine.
+     * It returns `true` if:
+     *  <ul>
+     *   <li>The IP was never in quarantine;</li>
+     *   <li>The IP was in quarantine, but the quarantine expired;</li>
+     *   <li>The IP is still in quarantine; however, the new owner is the same as the previous owner, therefore, the IP can be allocated.</li>
+     * </ul>
+     *
+     * It returns `false` if:
+     * <ul>
+     *   <li>The IP is in active quarantine and the new owner is different from the previous owner.</li>
+     * </ul>
+     *
+     * @param ip used to check if it is in active quarantine.
+     * @param account used to identify the new owner of the public IP.
+     * @return true if the IP can be allocated, and false otherwise.
+     */
+    boolean canPublicIpAddressBeAllocated(IpAddress ip, Account account);
+
+    /**
+     * Adds the given public IP address to quarantine for the duration of the global configuration `public.ip.address.quarantine.duration` value.
+     *
+     * @param publicIpAddress to be quarantined.
+     * @param domainId used to retrieve the quarantine duration.
+     * @return the {@link PublicIpQuarantine} persisted in the database.
+     */
+    PublicIpQuarantine addPublicIpAddressToQuarantine(IpAddress publicIpAddress, Long domainId);
+
+    /**
+     * Prematurely removes a public IP address from quarantine. It is required to provide a reason for removing it.
+     *
+     * @param quarantineProcessId the ID of the active quarantine process.
+     * @param removalReason       for prematurely removing the public IP address from quarantine.
+     */
+    void removePublicIpAddressFromQuarantine(Long quarantineProcessId, String removalReason);
+
+    /**
+     * Updates the end date of a public IP address in active quarantine. It can increase and decrease the duration of the quarantine.
+     *
+     * @param quarantineProcessId the ID of the quarantine process.
+     * @param endDate             the new end date for the quarantine.
+     * @return the updated quarantine object.
+     */
+    PublicIpQuarantine updatePublicIpAddressInQuarantine(Long quarantineProcessId, Date endDate);
+
     void updateSourceNatIpAddress(IPAddressVO requestedIp, List<IPAddressVO> userIps) throws Exception;
 }

engine/schema/src/main/java/com/cloud/host/HostTagVO.java

@@ -24,6 +24,7 @@ import javax.persistence.Id;
 import javax.persistence.Table;
 
 import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.commons.lang3.BooleanUtils;
 
 @Entity
 @Table(name = "host_tags")
@@ -39,12 +40,22 @@ public class HostTagVO implements InternalIdentity {
     @Column(name = "tag")
     private String tag;
 
+    @Column(name = "is_tag_a_rule")
+    private boolean isTagARule;
+
     protected HostTagVO() {
     }
 
     public HostTagVO(long hostId, String tag) {
         this.hostId = hostId;
         this.tag = tag;
+        this.isTagARule = false;
+    }
+
+    public HostTagVO(long hostId, String tag, Boolean isTagARule) {
+        this.hostId = hostId;
+        this.tag = tag;
+        this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false);
     }
 
     public long getHostId() {
@@ -59,6 +70,11 @@ public class HostTagVO implements InternalIdentity {
         this.tag = tag;
     }
 
+    public boolean getIsTagARule() {
+        return isTagARule;
+    }
+
+
     @Override
     public long getId() {
         return id;

engine/schema/src/main/java/com/cloud/host/HostVO.java

@@ -39,6 +39,7 @@ import javax.persistence.TemporalType;
 import javax.persistence.Transient;
 
 import com.cloud.agent.api.VgpuTypesInfo;
+import com.cloud.host.dao.HostTagsDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.resource.ResourceState;
@@ -46,7 +47,10 @@ import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.db.GenericDao;
 import java.util.Arrays;
+
+import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper;
 import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+import org.apache.commons.lang.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 
 @Entity
@@ -159,6 +163,14 @@ public class HostVO implements Host {
     @Transient
     List<String> hostTags;
 
+    /**
+     * This is a delayed load value.
+     * If the value is null, then this field has not been loaded yet.
+     * Call host dao to load it.
+     */
+    @Transient
+    Boolean isTagARule;
+
     // This value is only for saving and current cannot be loaded.
     @Transient
     HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails = new HashMap<String, HashMap<String, VgpuTypesInfo>>();
@@ -322,8 +334,13 @@ public class HostVO implements Host {
         return hostTags;
     }
 
-    public void setHostTags(List<String> hostTags) {
+    public void setHostTags(List<String> hostTags, Boolean isTagARule) {
         this.hostTags = hostTags;
+        this.isTagARule = isTagARule;
+    }
+
+    public Boolean getIsTagARule() {
+        return isTagARule;
     }
 
     public  HashMap<String, HashMap<String, VgpuTypesInfo>> getGpuGroupDetails() {
@@ -748,6 +765,11 @@ public class HostVO implements Host {
         if (serviceOffering == null) {
             return false;
         }
+
+        if (BooleanUtils.isTrue(this.getIsTagARule())) {
+            return TagAsRuleHelper.interpretTagAsRule(this.getHostTags().get(0), serviceOffering.getHostTag(), HostTagsDao.hostTagRuleExecutionTimeout.value());
+        }
+
         if (StringUtils.isEmpty(serviceOffering.getHostTag())) {
             return true;
         }

engine/schema/src/main/java/com/cloud/host/dao/HostDao.java

@@ -108,6 +108,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
 
     List<HostVO> listAllHostsByZoneAndHypervisorType(long zoneId, HypervisorType hypervisorType);
 
+    List<HostVO> listAllHostsThatHaveNoRuleTag(Host.Type type, Long clusterId, Long podId, Long dcId);
+
     List<HostVO> listAllHostsByType(Host.Type type);
 
     HostVO findByPublicIp(String publicIp);
@@ -161,4 +163,8 @@ public interface HostDao extends GenericDao<HostVO, Long>, StateDao<Status, Stat
      * @return ordered list of hypervisor versions
      */
     List<String> listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType);
+
+    List<HostVO> findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags);
+
+    List<Long> findClustersThatMatchHostTagRule(String computeOfferingTags);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java

@@ -22,9 +22,11 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.TimeZone;
 import java.util.stream.Collectors;
 
@@ -32,6 +34,8 @@ import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import javax.persistence.TableGenerator;
 
+import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.VgpuTypesInfo;
@@ -80,8 +84,8 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     private static final Logger state_logger = Logger.getLogger(ResourceState.class);
 
     private static final String LIST_HOST_IDS_BY_COMPUTETAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count "
-                                                             + "FROM (SELECT host_id, tag FROM host_tags GROUP BY host_id,tag) AS filtered "
-                                                             + "WHERE tag IN(%s) "
+                                                             + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag) AS filtered "
+                                                             + "WHERE tag IN(%s) AND is_tag_a_rule = 0 "
                                                              + "GROUP BY host_id "
                                                              + "HAVING tag_count = %s ";
     private static final String SEPARATOR = ",";
@@ -148,6 +152,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
     protected GenericSearchBuilder<ClusterVO, Long> AllClustersSearch;
     protected SearchBuilder<HostVO> HostsInClusterSearch;
 
+    protected SearchBuilder<HostVO> searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag;
+
+    protected SearchBuilder<HostTagVO> searchBuilderFindByRuleTag;
+
     protected Attribute _statusAttr;
     protected Attribute _resourceStateAttr;
     protected Attribute _msIdAttr;
@@ -455,6 +463,22 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ);
         HostIdSearch.done();
 
+        searchBuilderFindByRuleTag = _hostTagsDao.createSearchBuilder();
+        searchBuilderFindByRuleTag.and("is_tag_a_rule", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.EQ);
+        searchBuilderFindByRuleTag.or("tagDoesNotExist", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.NULL);
+
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag = createSearchBuilder();
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), Op.EQ);
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("type", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getType(), Op.EQ);
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("cluster_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getClusterId(), Op.EQ);
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("pod_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getPodId(), Op.EQ);
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("data_center_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getDataCenterId(), Op.EQ);
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.join("id", searchBuilderFindByRuleTag, searchBuilderFindByRuleTag.entity().getHostId(),
+                searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), JoinType.LEFTOUTER);
+
+        searchBuilderFindByRuleTag.done();
+        searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.done();
+
         _statusAttr = _allAttributes.get("status");
         _msIdAttr = _allAttributes.get("managementServerId");
         _pingTimeAttr = _allAttributes.get("lastPinged");
@@ -792,9 +816,12 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     @Override
     public List<HostVO> listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag) {
-        SearchBuilder<HostTagVO> hostTagSearch = null;
+        SearchBuilder<HostTagVO> hostTagSearch = _hostTagsDao.createSearchBuilder();
+        hostTagSearch.and();
+        hostTagSearch.op("isTagARule", hostTagSearch.entity().getIsTagARule(), Op.EQ);
+        hostTagSearch.or("tagDoesNotExist", hostTagSearch.entity().getIsTagARule(), Op.NULL);
+        hostTagSearch.cp();
         if (haTag != null && !haTag.isEmpty()) {
-            hostTagSearch = _hostTagsDao.createSearchBuilder();
             hostTagSearch.and().op("tag", hostTagSearch.entity().getTag(), SearchCriteria.Op.NEQ);
             hostTagSearch.or("tagNull", hostTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
             hostTagSearch.cp();
@@ -809,12 +836,14 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         hostSearch.and("status", hostSearch.entity().getStatus(), SearchCriteria.Op.EQ);
         hostSearch.and("resourceState", hostSearch.entity().getResourceState(), SearchCriteria.Op.EQ);
 
-        if (haTag != null && !haTag.isEmpty()) {
-            hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER);
-        }
+
+        hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER);
+
 
         SearchCriteria<HostVO> sc = hostSearch.create();
 
+        sc.setJoinParameters("hostTagSearch", "isTagARule", false);
+
         if (haTag != null && !haTag.isEmpty()) {
             sc.setJoinParameters("hostTagSearch", "tag", haTag);
         }
@@ -846,8 +875,13 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     @Override
     public void loadHostTags(HostVO host) {
-        List<String> hostTags = _hostTagsDao.getHostTags(host.getId());
-        host.setHostTags(hostTags);
+        List<HostTagVO> hostTagVOList = _hostTagsDao.getHostTags(host.getId());
+        if (CollectionUtils.isNotEmpty(hostTagVOList)) {
+            List<String> hostTagList = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList());
+            host.setHostTags(hostTagList, hostTagVOList.get(0).getIsTagARule());
+        } else {
+            host.setHostTags(null, null);
+        }
     }
 
     @DB
@@ -881,10 +915,10 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
 
     protected void saveHostTags(HostVO host) {
         List<String> hostTags = host.getHostTags();
-        if (hostTags == null || (hostTags != null && hostTags.isEmpty())) {
+        if (CollectionUtils.isEmpty(hostTags)) {
             return;
         }
-        _hostTagsDao.persist(host.getId(), hostTags);
+        _hostTagsDao.persist(host.getId(), hostTags, host.getIsTagARule());
     }
 
     protected void saveGpuRecords(HostVO host) {
@@ -1244,6 +1278,26 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
         return listBy(sc);
     }
 
+    @Override
+    public List<HostVO> listAllHostsThatHaveNoRuleTag(Type type, Long clusterId, Long podId, Long dcId) {
+        SearchCriteria<HostVO> sc = searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.create();
+        if (type != null) {
+            sc.setParameters("type", type);
+        }
+        if (clusterId != null) {
+            sc.setParameters("cluster_id", clusterId);
+        }
+        if (podId != null) {
+            sc.setParameters("pod_id", podId);
+        }
+        if (dcId != null) {
+            sc.setParameters("data_center_id", dcId);
+        }
+        sc.setJoinParameters("id", "is_tag_a_rule", false);
+
+        return search(sc, null);
+    }
+
     @Override
     public List<Long> listClustersByHostTag(String computeOfferingTags) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
@@ -1266,9 +1320,6 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
                 result.add(rs.getLong(1));
             }
             pstmt.close();
-            if(result.isEmpty()){
-                throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags);
-            }
             return result;
         } catch (SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + sql, e);
@@ -1293,15 +1344,33 @@ public class HostDaoImpl extends GenericDaoBase<HostVO, Long> implements HostDao
                 result.add(rs.getLong(1));
             }
             pstmt.close();
-            if(result.isEmpty()){
-                throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags);
-            }
             return result;
         } catch (SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + select, e);
         }
     }
 
+    public List<HostVO> findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) {
+        List<HostTagVO> hostTagVOList = _hostTagsDao.findHostRuleTags();
+        List<HostVO> result = new ArrayList<>();
+        for (HostTagVO rule: hostTagVOList) {
+            if (TagAsRuleHelper.interpretTagAsRule(rule.getTag(), computeOfferingTags, HostTagsDao.hostTagRuleExecutionTimeout.value())) {
+                result.add(findById(rule.getHostId()));
+            }
+        }
+
+        return result;
+    }
+
+    public List<Long> findClustersThatMatchHostTagRule(String computeOfferingTags) {
+        Set<Long> result = new HashSet<>();
+        List<HostVO> hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags);
+        for (HostVO host: hosts) {
+            result.add(host.getClusterId());
+        }
+        return new ArrayList<>(result);
+    }
+
     private String getHostIdsByComputeTags(List<String> offeringTags){
         List<String> questionMarks = new ArrayList();
         offeringTags.forEach((tag) -> { questionMarks.add("?"); });

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java

@@ -20,15 +20,21 @@ import java.util.List;
 
 import com.cloud.host.HostTagVO;
 import com.cloud.utils.db.GenericDao;
+import org.apache.cloudstack.framework.config.ConfigKey;
 
 public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
 
-    void persist(long hostId, List<String> hostTags);
+    ConfigKey<Long> hostTagRuleExecutionTimeout = new ConfigKey<>("Advanced", Long.class, "host.tag.rule.execution.timeout", "2000", "The maximum runtime, in milliseconds, " +
+        "to execute a host tag rule; if it is reached, a timeout will happen.", true);
 
-    List<String> getHostTags(long hostId);
+    void persist(long hostId, List<String> hostTags, Boolean isTagARule);
+
+    List<HostTagVO> getHostTags(long hostId);
 
     List<String> getDistinctImplicitHostTags(List<Long> hostIds, String[] implicitHostTags);
 
     void deleteTags(long hostId);
 
+    List<HostTagVO> findHostRuleTags();
+
 }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java

@@ -16,10 +16,10 @@
 // under the License.
 package com.cloud.host.dao;
 
-import java.util.ArrayList;
 import java.util.List;
 
-
+import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.Configurable;
 import org.springframework.stereotype.Component;
 
 import com.cloud.host.HostTagVO;
@@ -31,13 +31,14 @@ import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.SearchCriteria.Func;
 
 @Component
-public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements HostTagsDao {
+public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements HostTagsDao, Configurable {
     protected final SearchBuilder<HostTagVO> HostSearch;
     protected final GenericSearchBuilder<HostTagVO, String> DistinctImplictTagsSearch;
 
     public HostTagsDaoImpl() {
         HostSearch = createSearchBuilder();
         HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        HostSearch.and("isTagARule", HostSearch.entity().getIsTagARule(), SearchCriteria.Op.EQ);
         HostSearch.done();
 
         DistinctImplictTagsSearch = createSearchBuilder(String.class);
@@ -48,17 +49,11 @@ public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements
     }
 
     @Override
-    public List<String> getHostTags(long hostId) {
+    public List<HostTagVO> getHostTags(long hostId) {
         SearchCriteria<HostTagVO> sc = HostSearch.create();
         sc.setParameters("hostId", hostId);
 
-        List<HostTagVO> results = search(sc, null);
-        List<String> hostTags = new ArrayList<String>(results.size());
-        for (HostTagVO result : results) {
-            hostTags.add(result.getTag());
-        }
-
-        return hostTags;
+        return search(sc, null);
     }
 
     @Override
@@ -80,7 +75,15 @@ public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements
     }
 
     @Override
-    public void persist(long hostId, List<String> hostTags) {
+    public List<HostTagVO> findHostRuleTags() {
+        SearchCriteria<HostTagVO> sc = HostSearch.create();
+        sc.setParameters("isTagARule", true);
+
+        return search(sc, null);
+    }
+
+    @Override
+    public void persist(long hostId, List<String> hostTags, Boolean isTagARule) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
 
         txn.start();
@@ -91,10 +94,20 @@ public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements
         for (String tag : hostTags) {
             tag = tag.trim();
             if (tag.length() > 0) {
-                HostTagVO vo = new HostTagVO(hostId, tag);
+                HostTagVO vo = new HostTagVO(hostId, tag, isTagARule);
                 persist(vo);
             }
         }
         txn.commit();
     }
+
+    @Override
+    public ConfigKey<?>[] getConfigKeys() {
+        return new ConfigKey<?>[] {hostTagRuleExecutionTimeout};
+    }
+
+    @Override
+    public String getConfigComponentName() {
+        return HostTagsDaoImpl.class.getSimpleName();
+    }
 }

engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java

@@ -21,6 +21,7 @@ import java.util.List;
 import com.cloud.dc.Vlan.VlanType;
 import com.cloud.network.IpAddress.State;
 import com.cloud.utils.db.GenericDao;
+import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.net.Ip;
 
 public interface IPAddressDao extends GenericDao<IPAddressVO, Long> {
@@ -100,4 +101,6 @@ public interface IPAddressDao extends GenericDao<IPAddressVO, Long> {
     List<IPAddressVO> listByDcIdAndAssociatedNetwork(long dcId);
 
     List<IPAddressVO> listByNetworkId(long networkId);
+
+    void buildQuarantineSearchCriteria(SearchCriteria<IPAddressVO> sc);
 }

engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java

@@ -24,6 +24,7 @@ import java.util.List;
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 
+import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -32,6 +33,7 @@ import com.cloud.dc.Vlan.VlanType;
 import com.cloud.dc.VlanVO;
 import com.cloud.dc.dao.VlanDao;
 import com.cloud.network.IpAddress.State;
+import com.cloud.network.vo.PublicIpQuarantineVO;
 import com.cloud.server.ResourceTag.ResourceObjectType;
 import com.cloud.tags.dao.ResourceTagDao;
 import com.cloud.utils.db.DB;
@@ -69,6 +71,9 @@ public class IPAddressDaoImpl extends GenericDaoBase<IPAddressVO, Long> implemen
     @Inject
     UserIpAddressDetailsDao _detailsDao;
 
+    @Inject
+    PublicIpQuarantineDao publicIpQuarantineDao;
+
     // make it public for JUnit test
     public IPAddressDaoImpl() {
     }
@@ -534,4 +539,19 @@ public class IPAddressDaoImpl extends GenericDaoBase<IPAddressVO, Long> implemen
         sc.setParameters("state", State.Allocated);
         return listBy(sc);
     }
+
+    @Override
+    public void buildQuarantineSearchCriteria(SearchCriteria<IPAddressVO> sc) {
+        long accountId = CallContext.current().getCallingAccount().getAccountId();
+        SearchBuilder<PublicIpQuarantineVO> listAllIpsInQuarantine = publicIpQuarantineDao.createSearchBuilder();
+        listAllIpsInQuarantine.and("quarantineEndDate", listAllIpsInQuarantine.entity().getEndDate(), SearchCriteria.Op.GT);
+        listAllIpsInQuarantine.and("previousOwnerId", listAllIpsInQuarantine.entity().getPreviousOwnerId(), Op.NEQ);
+
+        SearchCriteria<PublicIpQuarantineVO> searchCriteria = listAllIpsInQuarantine.create();
+        searchCriteria.setParameters("quarantineEndDate", new Date());
+        searchCriteria.setParameters("previousOwnerId", accountId);
+        Object[] quarantinedIpsIdsAllowedToUser = publicIpQuarantineDao.search(searchCriteria, null).stream().map(PublicIpQuarantineVO::getPublicIpAddressId).toArray();
+
+        sc.setParametersIfNotNull("quarantinedPublicIpsIdsNIN", quarantinedIpsIdsAllowedToUser);
+    }
 }

engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java

@@ -29,7 +29,6 @@ import javax.persistence.Id;
 import javax.persistence.Table;
 import javax.persistence.Temporal;
 import javax.persistence.TemporalType;
-import javax.persistence.Transient;
 
 import com.cloud.network.IpAddress;
 import com.cloud.utils.db.GenericDao;
@@ -97,14 +96,6 @@ public class IPAddressVO implements IpAddress {
     @Column(name = "is_system")
     private boolean system;
 
-    @Column(name = "account_id")
-    @Transient
-    private Long accountId = null;
-
-    @Transient
-    @Column(name = "domain_id")
-    private Long domainId = null;
-
     @Column(name = "vpc_id")
     private Long vpcId;
 

engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.dao;
+
+import com.cloud.network.vo.PublicIpQuarantineVO;
+import com.cloud.utils.db.GenericDao;
+
+public interface PublicIpQuarantineDao extends GenericDao<PublicIpQuarantineVO, Long> {
+
+    PublicIpQuarantineVO findByPublicIpAddressId(long publicIpAddressId);
+
+    PublicIpQuarantineVO findByIpAddress(String publicIpAddress);
+}

engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java

@@ -0,0 +1,71 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.dao;
+
+import com.cloud.network.vo.PublicIpQuarantineVO;
+import com.cloud.utils.db.Filter;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.JoinBuilder;
+import com.cloud.utils.db.SearchBuilder;
+import com.cloud.utils.db.SearchCriteria;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+
+@Component
+public class PublicIpQuarantineDaoImpl extends GenericDaoBase<PublicIpQuarantineVO, Long> implements PublicIpQuarantineDao {
+    private SearchBuilder<PublicIpQuarantineVO> publicIpAddressByIdSearch;
+
+    private SearchBuilder<IPAddressVO> ipAddressSearchBuilder;
+
+    @Inject
+    IPAddressDao ipAddressDao;
+
+    @PostConstruct
+    public void init() {
+        publicIpAddressByIdSearch = createSearchBuilder();
+        publicIpAddressByIdSearch.and("publicIpAddressId", publicIpAddressByIdSearch.entity().getPublicIpAddressId(), SearchCriteria.Op.EQ);
+
+        ipAddressSearchBuilder = ipAddressDao.createSearchBuilder();
+        ipAddressSearchBuilder.and("ipAddress", ipAddressSearchBuilder.entity().getAddress(), SearchCriteria.Op.EQ);
+        ipAddressSearchBuilder.and("removed", ipAddressSearchBuilder.entity().getRemoved(), SearchCriteria.Op.NULL);
+        publicIpAddressByIdSearch.join("quarantineJoin", ipAddressSearchBuilder, ipAddressSearchBuilder.entity().getId(),
+                publicIpAddressByIdSearch.entity().getPublicIpAddressId(), JoinBuilder.JoinType.INNER);
+
+        ipAddressSearchBuilder.done();
+        publicIpAddressByIdSearch.done();
+    }
+
+    @Override
+    public PublicIpQuarantineVO findByPublicIpAddressId(long publicIpAddressId) {
+        SearchCriteria<PublicIpQuarantineVO> sc = publicIpAddressByIdSearch.create();
+        sc.setParameters("publicIpAddressId", publicIpAddressId);
+        final Filter filter = new Filter(PublicIpQuarantineVO.class, "created", false);
+
+        return findOneBy(sc, filter);
+    }
+
+    @Override
+    public PublicIpQuarantineVO findByIpAddress(String publicIpAddress) {
+        SearchCriteria<PublicIpQuarantineVO> sc = publicIpAddressByIdSearch.create();
+        sc.setJoinParameters("quarantineJoin", "ipAddress", publicIpAddress);
+        final Filter filter = new Filter(PublicIpQuarantineVO.class, "created", false);
+
+        return findOneBy(sc, filter);
+    }
+}

engine/schema/src/main/java/com/cloud/network/vo/PublicIpQuarantineVO.java

@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.vo;
+
+import com.cloud.network.PublicIpQuarantine;
+import com.cloud.utils.db.GenericDao;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
+import java.util.Date;
+import java.util.UUID;
+
+@Entity
+@Table(name = "quarantined_ips")
+public class PublicIpQuarantineVO implements PublicIpQuarantine {
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    @Column(name = "id", nullable = false)
+    private Long id;
+
+    @Column(name = "uuid", nullable = false)
+    private String uuid = UUID.randomUUID().toString();
+
+    @Column(name = "public_ip_address_id", nullable = false)
+    private Long publicIpAddressId;
+
+    @Column(name = "previous_owner_id", nullable = false)
+    private Long previousOwnerId;
+
+    @Column(name = GenericDao.CREATED_COLUMN, nullable = false)
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date created;
+
+    @Column(name = GenericDao.REMOVED_COLUMN)
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date removed = null;
+
+    @Column(name = "end_date", nullable = false)
+    @Temporal(value = TemporalType.TIMESTAMP)
+    private Date endDate;
+
+    @Column(name = "removal_reason")
+    private String removalReason = null;
+
+    public PublicIpQuarantineVO() {
+    }
+
+    public PublicIpQuarantineVO(Long publicIpAddressId, Long previousOwnerId, Date created, Date endDate) {
+        this.publicIpAddressId = publicIpAddressId;
+        this.previousOwnerId = previousOwnerId;
+        this.created = created;
+        this.endDate = endDate;
+    }
+
+    @Override
+    public long getId() {
+        return id;
+    }
+
+    @Override
+    public Long getPublicIpAddressId() {
+        return publicIpAddressId;
+    }
+
+    @Override
+    public Long getPreviousOwnerId() {
+        return previousOwnerId;
+    }
+
+    @Override
+    public Date getEndDate() {
+        return endDate;
+    }
+
+    @Override
+    public String getRemovalReason() {
+        return removalReason;
+    }
+
+    @Override
+    public String getUuid() {
+        return uuid;
+    }
+
+    public void setEndDate(Date endDate) {
+        this.endDate = endDate;
+    }
+
+    public void setRemovalReason(String removalReason) {
+        this.removalReason = removalReason;
+    }
+
+    @Override
+    public Date getRemoved() {
+        return removed;
+    }
+
+    public void setRemoved(Date removed) {
+        this.removed = removed;
+    }
+
+    @Override
+    public Date getCreated() {
+        return created;
+    }
+
+    public void setCreated(Date created) {
+        this.created = created;
+    }
+}

engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java

@@ -17,6 +17,8 @@
 
 package com.cloud.network.vpc;
 
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
+
 import java.util.UUID;
 
 import javax.persistence.Column;
@@ -85,6 +87,11 @@ public class NetworkACLVO implements NetworkACL {
         return name;
     }
 
+    @Override
+    public String toString() {
+        return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "uuid", "name", "vpcId");
+    }
+
     public void setUuid(String uuid) {
         this.uuid = uuid;
     }

engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java

@@ -23,7 +23,9 @@ import javax.persistence.GenerationType;
 import javax.persistence.Id;
 import javax.persistence.Table;
 
+import com.cloud.utils.NumbersUtil;
 import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.commons.lang3.BooleanUtils;
 
 @Entity
 @Table(name = "storage_pool_tags")
@@ -43,9 +45,19 @@ public class StoragePoolTagVO implements InternalIdentity {
     @Column(name = "tag")
     private String tag;
 
+    @Column(name = "is_tag_a_rule")
+    private boolean isTagARule;
+
     public StoragePoolTagVO(long poolId, String tag) {
         this.poolId = poolId;
         this.tag = tag;
+        this.isTagARule = false;
+    }
+
+    public StoragePoolTagVO(long poolId, String tag, Boolean isTagARule) {
+        this.poolId = poolId;
+        this.tag = tag;
+        this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false);
     }
 
     @Override
@@ -61,4 +73,20 @@ public class StoragePoolTagVO implements InternalIdentity {
         return tag;
     }
 
+    public boolean isTagARule() {
+        return this.isTagARule;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof StoragePoolTagVO) {
+            return this.poolId == ((StoragePoolTagVO)obj).getPoolId();
+        }
+        return false;
+    }
+
+    @Override
+    public int hashCode() {
+        return NumbersUtil.hash(id);
+    }
 }

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java

@@ -25,10 +25,14 @@ import com.cloud.utils.db.GenericDao;
 
 public interface StoragePoolTagsDao extends GenericDao<StoragePoolTagVO, Long> {
 
-    void persist(long poolId, List<String> storagePoolTags);
+    void persist(long poolId, List<String> storagePoolTags, Boolean isTagARule);
+
+    void persist(List<StoragePoolTagVO> storagePoolTags);
     List<String> getStoragePoolTags(long poolId);
     void deleteTags(long poolId);
     List<StoragePoolTagVO> searchByIds(Long... stIds);
     StorageTagResponse newStorageTagResponse(StoragePoolTagVO tag);
 
+    List<StoragePoolTagVO> findStoragePoolTags(long poolId);
+
 }

engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java

@@ -21,6 +21,9 @@ import java.util.List;
 
 import javax.inject.Inject;
 
+import com.cloud.utils.db.Transaction;
+import com.cloud.utils.db.TransactionCallbackNoReturn;
+import com.cloud.utils.db.TransactionStatus;
 import org.apache.cloudstack.api.response.StorageTagResponse;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 
@@ -50,7 +53,7 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase<StoragePoolTagVO, Lon
     }
 
     @Override
-    public void persist(long poolId, List<String> storagePoolTags) {
+    public void persist(long poolId, List<String> storagePoolTags, Boolean isTagARule) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
 
         txn.start();
@@ -61,13 +64,23 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase<StoragePoolTagVO, Lon
         for (String tag : storagePoolTags) {
             tag = tag.trim();
             if (tag.length() > 0) {
-                StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag);
+                StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag, isTagARule);
                 persist(vo);
             }
         }
         txn.commit();
     }
 
+    public void persist(List<StoragePoolTagVO> storagePoolTags) {
+        Transaction.execute(TransactionLegacy.CLOUD_DB, new TransactionCallbackNoReturn() {
+            @Override public void doInTransactionWithoutResult(TransactionStatus status) {
+                for (StoragePoolTagVO storagePoolTagVO : storagePoolTags) {
+                    persist(storagePoolTagVO);
+                }
+            }
+        });
+    }
+
     @Override
     public List<String> getStoragePoolTags(long poolId) {
         SearchCriteria<StoragePoolTagVO> sc = StoragePoolSearch.create();
@@ -157,4 +170,12 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase<StoragePoolTagVO, Lon
         return tagResponse;
     }
 
+    @Override
+    public List<StoragePoolTagVO> findStoragePoolTags(long poolId) {
+        SearchCriteria<StoragePoolTagVO> sc = StoragePoolSearch.create();
+        sc.setParameters("poolId", poolId);
+
+        return search(sc, null);
+    }
+
 }

engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java

@@ -22,15 +22,18 @@ import static com.google.common.collect.ObjectArrays.concat;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Date;
+import java.util.List;
 
 import javax.inject.Inject;
 
+import com.cloud.utils.FileUtil;
 import org.apache.cloudstack.utils.CloudStackVersion;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
@@ -123,6 +126,7 @@ import com.google.common.annotations.VisibleForTesting;
 public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
     private static final Logger s_logger = Logger.getLogger(DatabaseUpgradeChecker.class);
     private final DatabaseVersionHierarchy hierarchy;
+    private static final String VIEWS_DIRECTORY = Paths.get("META-INF", "db", "views").toString();
 
     @Inject
     VersionDao _dao;
@@ -363,9 +367,33 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker {
                 txn.close();
             }
         }
+
+        executeViewScripts();
         updateSystemVmTemplates(upgrades);
     }
 
+    protected void executeViewScripts() {
+        s_logger.info(String.format("Executing VIEW scripts that are under resource directory [%s].", VIEWS_DIRECTORY));
+        List<String> filesPathUnderViewsDirectory = FileUtil.getFilesPathsUnderResourceDirectory(VIEWS_DIRECTORY);
+
+        try (TransactionLegacy txn = TransactionLegacy.open("execute-view-scripts")) {
+            Connection conn = txn.getConnection();
+
+            for (String filePath : filesPathUnderViewsDirectory) {
+                s_logger.debug(String.format("Executing VIEW script [%s].", filePath));
+
+                InputStream viewScript = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath);
+                runScript(conn, viewScript);
+            }
+
+            s_logger.info(String.format("Finished execution of VIEW scripts that are under resource directory [%s].", VIEWS_DIRECTORY));
+        } catch (SQLException e) {
+            String message = String.format("Unable to execute VIEW scripts due to [%s].", e.getMessage());
+            s_logger.error(message, e);
+            throw new CloudRuntimeException(message, e);
+        }
+    }
+
     @Override
     public void check() {
         GlobalLock lock = GlobalLock.getInternLock("DatabaseUpgrade");

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java

@@ -53,7 +53,7 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
      */
     void updateCapacityIops(long id, long capacityIops);
 
-    StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags);
+    StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags, Boolean isTagARule);
 
     /**
      * Find pool by name.
@@ -77,7 +77,7 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
      */
     List<StoragePoolVO> findPoolsByDetails(long dcId, long podId, Long clusterId, Map<String, String> details, ScopeType scope);
 
-    List<StoragePoolVO> findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags);
+    List<StoragePoolVO> findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout);
 
     List<StoragePoolVO> findDisabledPoolsByScope(long dcId, Long podId, Long clusterId, ScopeType scope);
 
@@ -112,9 +112,9 @@ public interface PrimaryDataStoreDao extends GenericDao<StoragePoolVO, Long> {
 
     List<StoragePoolVO> listPoolsByCluster(long clusterId);
 
-    List<StoragePoolVO> findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags);
+    List<StoragePoolVO> findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule);
 
-    List<StoragePoolVO> findZoneWideStoragePoolsByTags(long dcId, String[] tags);
+    List<StoragePoolVO> findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule);
 
     List<StoragePoolVO> findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType);
 

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java

@@ -67,11 +67,11 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
 
     protected final String DetailsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_details ON storage_pool.id = storage_pool_details.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and (";
     protected final String DetailsSqlSuffix = ") GROUP BY storage_pool_details.pool_id HAVING COUNT(storage_pool_details.name) >= ?";
-    private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and storage_pool.scope = ? and (";
+    private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and storage_pool.scope = ? and (";
     private final String ZoneWideTagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?";
 
     // Storage tags are now separate from storage_pool_details, leaving only details on that table
-    protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and (";
+    protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and (";
     protected final String TagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?";
 
     private static final String GET_STORAGE_POOLS_OF_VOLUMES_WITHOUT_OR_NOT_HAVING_TAGS = "SELECT s.* " +
@@ -278,7 +278,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
 
     @Override
     @DB
-    public StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags) {
+    public StoragePoolVO persist(StoragePoolVO pool, Map<String, String> details, List<String> tags, Boolean isTagARule) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
         txn.start();
         pool = super.persist(pool);
@@ -289,7 +289,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
             }
         }
         if (CollectionUtils.isNotEmpty(tags)) {
-            _tagsDao.persist(pool.getId(), tags);
+            _tagsDao.persist(pool.getId(), tags, isTagARule);
         }
         txn.commit();
         return pool;
@@ -404,10 +404,15 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
     }
 
     @Override
-    public List<StoragePoolVO> findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags) {
+    public List<StoragePoolVO> findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout) {
         List<StoragePoolVO> storagePools = null;
         if (tags == null || tags.length == 0) {
             storagePools = listBy(dcId, podId, clusterId, ScopeType.CLUSTER);
+
+            if (validateTagRule) {
+                storagePools = getPoolsWithoutTagRule(storagePools);
+            }
+
         } else {
             String sqlValues = getSqlValuesFromStorageTags(tags);
             storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.CLUSTER, sqlValues, ValueType.TAGS, tags.length);
@@ -437,10 +442,14 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
     }
 
     @Override
-    public List<StoragePoolVO> findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags) {
+    public List<StoragePoolVO> findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule) {
         List<StoragePoolVO> storagePools = null;
         if (tags == null || tags.length == 0) {
             storagePools = listBy(dcId, podId, clusterId, ScopeType.HOST);
+
+            if (validateTagRule) {
+                storagePools = getPoolsWithoutTagRule(storagePools);
+            }
         } else {
             String sqlValues = getSqlValuesFromStorageTags(tags);
             storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.HOST, sqlValues, ValueType.TAGS, tags.length);
@@ -483,13 +492,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
     }
 
     @Override
-    public List<StoragePoolVO> findZoneWideStoragePoolsByTags(long dcId, String[] tags) {
+    public List<StoragePoolVO> findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule) {
         if (tags == null || tags.length == 0) {
             QueryBuilder<StoragePoolVO> sc = QueryBuilder.create(StoragePoolVO.class);
             sc.and(sc.entity().getDataCenterId(), Op.EQ, dcId);
             sc.and(sc.entity().getStatus(), Op.EQ, Status.Up);
             sc.and(sc.entity().getScope(), Op.EQ, ScopeType.ZONE);
-            return sc.list();
+
+            List<StoragePoolVO> storagePools = sc.list();
+
+            if (validateTagRule) {
+                storagePools = getPoolsWithoutTagRule(storagePools);
+            }
+
+            return storagePools;
         } else {
             String sqlValues = getSqlValuesFromStorageTags(tags);
             String sql = getSqlPreparedStatement(ZoneWideTagsSqlPrefix, ZoneWideTagsSqlSuffix, sqlValues, null);
@@ -497,6 +513,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase<StoragePoolVO, Long>
         }
     }
 
+    protected List<StoragePoolVO> getPoolsWithoutTagRule(List<StoragePoolVO> storagePools) {
+        List<StoragePoolVO> storagePoolsToReturn = new ArrayList<>();
+        for (StoragePoolVO storagePool : storagePools) {
+
+            List<StoragePoolTagVO> poolTags = _tagsDao.findStoragePoolTags(storagePool.getId());
+
+            if (CollectionUtils.isEmpty(poolTags) || !poolTags.get(0).isTagARule()) {
+                storagePoolsToReturn.add(storagePool);
+            }
+        }
+
+        return storagePoolsToReturn;
+    }
+
     @Override
     public List<String> searchForStoragePoolTags(long poolId) {
         return _tagsDao.getStoragePoolTags(poolId);

engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -277,6 +277,7 @@
   <bean id="UserVmDeployAsIsDetailsDaoImpl" class="com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDaoImpl" />
   <bean id="NetworkPermissionDaoImpl" class="org.apache.cloudstack.network.dao.NetworkPermissionDaoImpl" />
   <bean id="PassphraseDaoImpl" class="org.apache.cloudstack.secret.dao.PassphraseDaoImpl" />
+  <bean id="publicIpQuarantineDaoImpl" class="com.cloud.network.dao.PublicIpQuarantineDaoImpl" />
   <bean id="VMScheduleDaoImpl" class="org.apache.cloudstack.vm.schedule.dao.VMScheduleDaoImpl" />
   <bean id="VMScheduledJobDaoImpl" class="org.apache.cloudstack.vm.schedule.dao.VMScheduledJobDaoImpl" />
   <bean id="vnfTemplateDetailsDaoImpl" class="com.cloud.storage.dao.VnfTemplateDetailsDaoImpl" />

engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql

@@ -21,121 +21,26 @@
 
 ALTER TABLE `cloud`.`mshost` MODIFY COLUMN `state` varchar(25);
 
-DROP VIEW IF EXISTS `cloud`.`async_job_view`;
-CREATE VIEW `cloud`.`async_job_view` AS
-    select
-        account.id account_id,
-        account.uuid account_uuid,
-        account.account_name account_name,
-        account.type account_type,
-        domain.id domain_id,
-        domain.uuid domain_uuid,
-        domain.name domain_name,
-        domain.path domain_path,
-        user.id user_id,
-        user.uuid user_uuid,
-        async_job.id,
-        async_job.uuid,
-        async_job.job_cmd,
-        async_job.job_status,
-        async_job.job_process_status,
-        async_job.job_result_code,
-        async_job.job_result,
-        async_job.created,
-        async_job.removed,
-        async_job.instance_type,
-        async_job.instance_id,
-        async_job.job_executing_msid,
-        CASE
-            WHEN async_job.instance_type = 'Volume' THEN volumes.uuid
-            WHEN
-                async_job.instance_type = 'Template'
-                    or async_job.instance_type = 'Iso'
-            THEN
-                vm_template.uuid
-            WHEN
-                async_job.instance_type = 'VirtualMachine'
-                    or async_job.instance_type = 'ConsoleProxy'
-                    or async_job.instance_type = 'SystemVm'
-                    or async_job.instance_type = 'DomainRouter'
-            THEN
-                vm_instance.uuid
-            WHEN async_job.instance_type = 'Snapshot' THEN snapshots.uuid
-            WHEN async_job.instance_type = 'Host' THEN host.uuid
-            WHEN async_job.instance_type = 'StoragePool' THEN storage_pool.uuid
-            WHEN async_job.instance_type = 'IpAddress' THEN user_ip_address.uuid
-            WHEN async_job.instance_type = 'SecurityGroup' THEN security_group.uuid
-            WHEN async_job.instance_type = 'PhysicalNetwork' THEN physical_network.uuid
-            WHEN async_job.instance_type = 'TrafficType' THEN physical_network_traffic_types.uuid
-            WHEN async_job.instance_type = 'PhysicalNetworkServiceProvider' THEN physical_network_service_providers.uuid
-            WHEN async_job.instance_type = 'FirewallRule' THEN firewall_rules.uuid
-            WHEN async_job.instance_type = 'Account' THEN acct.uuid
-            WHEN async_job.instance_type = 'User' THEN us.uuid
-            WHEN async_job.instance_type = 'StaticRoute' THEN static_routes.uuid
-            WHEN async_job.instance_type = 'PrivateGateway' THEN vpc_gateways.uuid
-            WHEN async_job.instance_type = 'Counter' THEN counter.uuid
-            WHEN async_job.instance_type = 'Condition' THEN conditions.uuid
-            WHEN async_job.instance_type = 'AutoScalePolicy' THEN autoscale_policies.uuid
-            WHEN async_job.instance_type = 'AutoScaleVmProfile' THEN autoscale_vmprofiles.uuid
-            WHEN async_job.instance_type = 'AutoScaleVmGroup' THEN autoscale_vmgroups.uuid
-            ELSE null
-        END instance_uuid
-    from
-        `cloud`.`async_job`
-            left join
-        `cloud`.`account` ON async_job.account_id = account.id
-            left join
-        `cloud`.`domain` ON domain.id = account.domain_id
-            left join
-        `cloud`.`user` ON async_job.user_id = user.id
-            left join
-        `cloud`.`volumes` ON async_job.instance_id = volumes.id
-            left join
-        `cloud`.`vm_template` ON async_job.instance_id = vm_template.id
-            left join
-        `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id
-            left join
-        `cloud`.`snapshots` ON async_job.instance_id = snapshots.id
-            left join
-        `cloud`.`host` ON async_job.instance_id = host.id
-            left join
-        `cloud`.`storage_pool` ON async_job.instance_id = storage_pool.id
-            left join
-        `cloud`.`user_ip_address` ON async_job.instance_id = user_ip_address.id
-            left join
-        `cloud`.`security_group` ON async_job.instance_id = security_group.id
-            left join
-        `cloud`.`physical_network` ON async_job.instance_id = physical_network.id
-            left join
-        `cloud`.`physical_network_traffic_types` ON async_job.instance_id = physical_network_traffic_types.id
-            left join
-        `cloud`.`physical_network_service_providers` ON async_job.instance_id = physical_network_service_providers.id
-            left join
-        `cloud`.`firewall_rules` ON async_job.instance_id = firewall_rules.id
-            left join
-        `cloud`.`account` acct ON async_job.instance_id = acct.id
-            left join
-        `cloud`.`user` us ON async_job.instance_id = us.id
-            left join
-        `cloud`.`static_routes` ON async_job.instance_id = static_routes.id
-            left join
-        `cloud`.`vpc_gateways` ON async_job.instance_id = vpc_gateways.id
-            left join
-        `cloud`.`counter` ON async_job.instance_id = counter.id
-            left join
-        `cloud`.`conditions` ON async_job.instance_id = conditions.id
-            left join
-        `cloud`.`autoscale_policies` ON async_job.instance_id = autoscale_policies.id
-            left join
-        `cloud`.`autoscale_vmprofiles` ON async_job.instance_id = autoscale_vmprofiles.id
-            left join
-        `cloud`.`autoscale_vmgroups` ON async_job.instance_id = autoscale_vmgroups.id;
-
 -- Invalidate existing console_session records
 UPDATE `cloud`.`console_session` SET removed=now();
 -- Modify acquired column in console_session to datetime type
 ALTER TABLE `cloud`.`console_session` DROP `acquired`, ADD `acquired` datetime COMMENT 'When the session was acquired' AFTER `host_id`;
 
+-- IP quarantine PR#7378
+CREATE TABLE IF NOT EXISTS `cloud`.`quarantined_ips` (
+  `id` bigint(20) unsigned NOT NULL auto_increment,
+  `uuid` varchar(255) UNIQUE,
+  `public_ip_address_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the quarantined public IP address, foreign key to `user_ip_address` table',
+  `previous_owner_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the previous owner of the public IP address, foreign key to `account` table',
+  `created` datetime NOT NULL,
+  `removed` datetime DEFAULT NULL,
+  `end_date` datetime NOT NULL,
+  `removal_reason` VARCHAR(255) DEFAULT NULL,
+  PRIMARY KEY (`id`),
+  CONSTRAINT `fk_quarantined_ips__public_ip_address_id` FOREIGN KEY(`public_ip_address_id`) REFERENCES `cloud`.`user_ip_address`(`id`),
+  CONSTRAINT `fk_quarantined_ips__previous_owner_id` FOREIGN KEY(`previous_owner_id`) REFERENCES `cloud`.`account`(`id`)
+);
+
 -- create_public_parameter_on_roles. #6960
 ALTER TABLE `cloud`.`roles` ADD COLUMN `public_role` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Indicates whether the role will be visible to all users (public) or only to root admins (private). If this parameter is not specified during the creation of the role its value will be defaulted to true (public).';
 
@@ -337,203 +242,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`vnf_template_details` (
     CONSTRAINT `fk_vnf_template_details__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template` (`id`) ON DELETE CASCADE
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
-DROP VIEW IF EXISTS `cloud`.`user_vm_view`;
-CREATE
-    VIEW `user_vm_view` AS
-SELECT
-    `vm_instance`.`id` AS `id`,
-    `vm_instance`.`name` AS `name`,
-    `user_vm`.`display_name` AS `display_name`,
-    `user_vm`.`user_data` AS `user_data`,
-    `account`.`id` AS `account_id`,
-    `account`.`uuid` AS `account_uuid`,
-    `account`.`account_name` AS `account_name`,
-    `account`.`type` AS `account_type`,
-    `domain`.`id` AS `domain_id`,
-    `domain`.`uuid` AS `domain_uuid`,
-    `domain`.`name` AS `domain_name`,
-    `domain`.`path` AS `domain_path`,
-    `projects`.`id` AS `project_id`,
-    `projects`.`uuid` AS `project_uuid`,
-    `projects`.`name` AS `project_name`,
-    `instance_group`.`id` AS `instance_group_id`,
-    `instance_group`.`uuid` AS `instance_group_uuid`,
-    `instance_group`.`name` AS `instance_group_name`,
-    `vm_instance`.`uuid` AS `uuid`,
-    `vm_instance`.`user_id` AS `user_id`,
-    `vm_instance`.`last_host_id` AS `last_host_id`,
-    `vm_instance`.`vm_type` AS `type`,
-    `vm_instance`.`limit_cpu_use` AS `limit_cpu_use`,
-    `vm_instance`.`created` AS `created`,
-    `vm_instance`.`state` AS `state`,
-    `vm_instance`.`update_time` AS `update_time`,
-    `vm_instance`.`removed` AS `removed`,
-    `vm_instance`.`ha_enabled` AS `ha_enabled`,
-    `vm_instance`.`hypervisor_type` AS `hypervisor_type`,
-    `vm_instance`.`instance_name` AS `instance_name`,
-    `vm_instance`.`guest_os_id` AS `guest_os_id`,
-    `vm_instance`.`display_vm` AS `display_vm`,
-    `guest_os`.`uuid` AS `guest_os_uuid`,
-    `vm_instance`.`pod_id` AS `pod_id`,
-    `host_pod_ref`.`uuid` AS `pod_uuid`,
-    `vm_instance`.`private_ip_address` AS `private_ip_address`,
-    `vm_instance`.`private_mac_address` AS `private_mac_address`,
-    `vm_instance`.`vm_type` AS `vm_type`,
-    `data_center`.`id` AS `data_center_id`,
-    `data_center`.`uuid` AS `data_center_uuid`,
-    `data_center`.`name` AS `data_center_name`,
-    `data_center`.`is_security_group_enabled` AS `security_group_enabled`,
-    `data_center`.`networktype` AS `data_center_network_type`,
-    `host`.`id` AS `host_id`,
-    `host`.`uuid` AS `host_uuid`,
-    `host`.`name` AS `host_name`,
-    `host`.`cluster_id` AS `cluster_id`,
-    `host`.`status` AS `host_status`,
-    `host`.`resource_state` AS `host_resource_state`,
-    `vm_template`.`id` AS `template_id`,
-    `vm_template`.`uuid` AS `template_uuid`,
-    `vm_template`.`name` AS `template_name`,
-    `vm_template`.`type` AS `template_type`,
-    `vm_template`.`display_text` AS `template_display_text`,
-    `vm_template`.`enable_password` AS `password_enabled`,
-    `iso`.`id` AS `iso_id`,
-    `iso`.`uuid` AS `iso_uuid`,
-    `iso`.`name` AS `iso_name`,
-    `iso`.`display_text` AS `iso_display_text`,
-    `service_offering`.`id` AS `service_offering_id`,
-    `service_offering`.`uuid` AS `service_offering_uuid`,
-    `disk_offering`.`uuid` AS `disk_offering_uuid`,
-    `disk_offering`.`id` AS `disk_offering_id`,
-    (CASE
-         WHEN ISNULL(`service_offering`.`cpu`) THEN `custom_cpu`.`value`
-         ELSE `service_offering`.`cpu`
-        END) AS `cpu`,
-    (CASE
-         WHEN ISNULL(`service_offering`.`speed`) THEN `custom_speed`.`value`
-         ELSE `service_offering`.`speed`
-        END) AS `speed`,
-    (CASE
-         WHEN ISNULL(`service_offering`.`ram_size`) THEN `custom_ram_size`.`value`
-         ELSE `service_offering`.`ram_size`
-        END) AS `ram_size`,
-    `backup_offering`.`uuid` AS `backup_offering_uuid`,
-    `backup_offering`.`id` AS `backup_offering_id`,
-    `service_offering`.`name` AS `service_offering_name`,
-    `disk_offering`.`name` AS `disk_offering_name`,
-    `backup_offering`.`name` AS `backup_offering_name`,
-    `storage_pool`.`id` AS `pool_id`,
-    `storage_pool`.`uuid` AS `pool_uuid`,
-    `storage_pool`.`pool_type` AS `pool_type`,
-    `volumes`.`id` AS `volume_id`,
-    `volumes`.`uuid` AS `volume_uuid`,
-    `volumes`.`device_id` AS `volume_device_id`,
-    `volumes`.`volume_type` AS `volume_type`,
-    `security_group`.`id` AS `security_group_id`,
-    `security_group`.`uuid` AS `security_group_uuid`,
-    `security_group`.`name` AS `security_group_name`,
-    `security_group`.`description` AS `security_group_description`,
-    `nics`.`id` AS `nic_id`,
-    `nics`.`uuid` AS `nic_uuid`,
-    `nics`.`device_id` AS `nic_device_id`,
-    `nics`.`network_id` AS `network_id`,
-    `nics`.`ip4_address` AS `ip_address`,
-    `nics`.`ip6_address` AS `ip6_address`,
-    `nics`.`ip6_gateway` AS `ip6_gateway`,
-    `nics`.`ip6_cidr` AS `ip6_cidr`,
-    `nics`.`default_nic` AS `is_default_nic`,
-    `nics`.`gateway` AS `gateway`,
-    `nics`.`netmask` AS `netmask`,
-    `nics`.`mac_address` AS `mac_address`,
-    `nics`.`broadcast_uri` AS `broadcast_uri`,
-    `nics`.`isolation_uri` AS `isolation_uri`,
-    `vpc`.`id` AS `vpc_id`,
-    `vpc`.`uuid` AS `vpc_uuid`,
-    `networks`.`uuid` AS `network_uuid`,
-    `networks`.`name` AS `network_name`,
-    `networks`.`traffic_type` AS `traffic_type`,
-    `networks`.`guest_type` AS `guest_type`,
-    `user_ip_address`.`id` AS `public_ip_id`,
-    `user_ip_address`.`uuid` AS `public_ip_uuid`,
-    `user_ip_address`.`public_ip_address` AS `public_ip_address`,
-    `ssh_details`.`value` AS `keypair_names`,
-    `resource_tags`.`id` AS `tag_id`,
-    `resource_tags`.`uuid` AS `tag_uuid`,
-    `resource_tags`.`key` AS `tag_key`,
-    `resource_tags`.`value` AS `tag_value`,
-    `resource_tags`.`domain_id` AS `tag_domain_id`,
-    `domain`.`uuid` AS `tag_domain_uuid`,
-    `domain`.`name` AS `tag_domain_name`,
-    `resource_tags`.`account_id` AS `tag_account_id`,
-    `account`.`account_name` AS `tag_account_name`,
-    `resource_tags`.`resource_id` AS `tag_resource_id`,
-    `resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
-    `resource_tags`.`resource_type` AS `tag_resource_type`,
-    `resource_tags`.`customer` AS `tag_customer`,
-    `async_job`.`id` AS `job_id`,
-    `async_job`.`uuid` AS `job_uuid`,
-    `async_job`.`job_status` AS `job_status`,
-    `async_job`.`account_id` AS `job_account_id`,
-    `affinity_group`.`id` AS `affinity_group_id`,
-    `affinity_group`.`uuid` AS `affinity_group_uuid`,
-    `affinity_group`.`name` AS `affinity_group_name`,
-    `affinity_group`.`description` AS `affinity_group_description`,
-    `autoscale_vmgroups`.`id` AS `autoscale_vmgroup_id`,
-    `autoscale_vmgroups`.`uuid` AS `autoscale_vmgroup_uuid`,
-    `autoscale_vmgroups`.`name` AS `autoscale_vmgroup_name`,
-    `vm_instance`.`dynamically_scalable` AS `dynamically_scalable`,
-    `user_data`.`id` AS `user_data_id`,
-    `user_data`.`uuid` AS `user_data_uuid`,
-    `user_data`.`name` AS `user_data_name`,
-    `user_vm`.`user_data_details` AS `user_data_details`,
-    `vm_template`.`user_data_link_policy` AS `user_data_policy`
-FROM
-    (((((((((((((((((((((((((((((((((((`user_vm`
-        JOIN `vm_instance` ON (((`vm_instance`.`id` = `user_vm`.`id`)
-            AND ISNULL(`vm_instance`.`removed`))))
-        JOIN `account` ON ((`vm_instance`.`account_id` = `account`.`id`)))
-        JOIN `domain` ON ((`vm_instance`.`domain_id` = `domain`.`id`)))
-        LEFT JOIN `guest_os` ON ((`vm_instance`.`guest_os_id` = `guest_os`.`id`)))
-        LEFT JOIN `host_pod_ref` ON ((`vm_instance`.`pod_id` = `host_pod_ref`.`id`)))
-        LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
-        LEFT JOIN `instance_group_vm_map` ON ((`vm_instance`.`id` = `instance_group_vm_map`.`instance_id`)))
-        LEFT JOIN `instance_group` ON ((`instance_group_vm_map`.`group_id` = `instance_group`.`id`)))
-        LEFT JOIN `data_center` ON ((`vm_instance`.`data_center_id` = `data_center`.`id`)))
-        LEFT JOIN `host` ON ((`vm_instance`.`host_id` = `host`.`id`)))
-        LEFT JOIN `vm_template` ON ((`vm_instance`.`vm_template_id` = `vm_template`.`id`)))
-        LEFT JOIN `vm_template` `iso` ON ((`iso`.`id` = `user_vm`.`iso_id`)))
-        LEFT JOIN `volumes` ON ((`vm_instance`.`id` = `volumes`.`instance_id`)))
-        LEFT JOIN `service_offering` ON ((`vm_instance`.`service_offering_id` = `service_offering`.`id`)))
-        LEFT JOIN `disk_offering` `svc_disk_offering` ON ((`volumes`.`disk_offering_id` = `svc_disk_offering`.`id`)))
-        LEFT JOIN `disk_offering` ON ((`volumes`.`disk_offering_id` = `disk_offering`.`id`)))
-        LEFT JOIN `backup_offering` ON ((`vm_instance`.`backup_offering_id` = `backup_offering`.`id`)))
-        LEFT JOIN `storage_pool` ON ((`volumes`.`pool_id` = `storage_pool`.`id`)))
-        LEFT JOIN `security_group_vm_map` ON ((`vm_instance`.`id` = `security_group_vm_map`.`instance_id`)))
-        LEFT JOIN `security_group` ON ((`security_group_vm_map`.`security_group_id` = `security_group`.`id`)))
-        LEFT JOIN `user_data` ON ((`user_data`.`id` = `user_vm`.`user_data_id`)))
-        LEFT JOIN `nics` ON (((`vm_instance`.`id` = `nics`.`instance_id`)
-        AND ISNULL(`nics`.`removed`))))
-        LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`)))
-        LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`)
-        AND ISNULL(`vpc`.`removed`))))
-        LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`)))
-        LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`)
-        AND (`ssh_details`.`name` = 'SSH.KeyPairNames'))))
-        LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`)
-        AND (`resource_tags`.`resource_type` = 'UserVm'))))
-        LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `vm_instance`.`id`)
-        AND (`async_job`.`instance_type` = 'VirtualMachine')
-        AND (`async_job`.`job_status` = 0))))
-        LEFT JOIN `affinity_group_vm_map` ON ((`vm_instance`.`id` = `affinity_group_vm_map`.`instance_id`)))
-        LEFT JOIN `affinity_group` ON ((`affinity_group_vm_map`.`affinity_group_id` = `affinity_group`.`id`)))
-        LEFT JOIN `autoscale_vmgroup_vm_map` ON ((`autoscale_vmgroup_vm_map`.`instance_id` = `vm_instance`.`id`)))
-        LEFT JOIN `autoscale_vmgroups` ON ((`autoscale_vmgroup_vm_map`.`vmgroup_id` = `autoscale_vmgroups`.`id`)))
-        LEFT JOIN `user_vm_details` `custom_cpu` ON (((`custom_cpu`.`vm_id` = `vm_instance`.`id`)
-        AND (`custom_cpu`.`name` = 'CpuNumber'))))
-        LEFT JOIN `user_vm_details` `custom_speed` ON (((`custom_speed`.`vm_id` = `vm_instance`.`id`)
-        AND (`custom_speed`.`name` = 'CpuSpeed'))))
-        LEFT JOIN `user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `vm_instance`.`id`)
-        AND (`custom_ram_size`.`name` = 'memory'))));
-
 -- Add tables for Cluster DRS
 DROP TABLE IF EXISTS `cloud`.`cluster_drs_plan`;
 CREATE TABLE `cloud`.`cluster_drs_plan` (
@@ -754,3 +462,12 @@ CREATE TABLE `cloud`.`oauth_provider` (
   `removed` datetime COMMENT 'date removed if not null',
   PRIMARY KEY (`id`)
   ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+-- Flexible tags
+ALTER TABLE `cloud`.`storage_pool_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0;
+
+ALTER TABLE `cloud`.`storage_pool_tags` MODIFY tag text NOT NULL;
+
+ALTER TABLE `cloud`.`host_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0;
+
+ALTER TABLE `cloud`.`host_tags` MODIFY tag text NOT NULL;

engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql

@@ -0,0 +1,129 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`async_job_view`;
+
+DROP VIEW IF EXISTS `cloud`.`async_job_view`;
+
+CREATE VIEW `cloud`.`async_job_view` AS
+select
+    account.id account_id,
+    account.uuid account_uuid,
+    account.account_name account_name,
+    account.type account_type,
+    domain.id domain_id,
+    domain.uuid domain_uuid,
+    domain.name domain_name,
+    domain.path domain_path,
+    user.id user_id,
+    user.uuid user_uuid,
+    async_job.id,
+    async_job.uuid,
+    async_job.job_cmd,
+    async_job.job_status,
+    async_job.job_process_status,
+    async_job.job_result_code,
+    async_job.job_result,
+    async_job.created,
+    async_job.removed,
+    async_job.instance_type,
+    async_job.instance_id,
+    async_job.job_executing_msid,
+    CASE
+        WHEN async_job.instance_type = 'Volume' THEN volumes.uuid
+        WHEN
+                    async_job.instance_type = 'Template'
+                or async_job.instance_type = 'Iso'
+            THEN
+            vm_template.uuid
+        WHEN
+                    async_job.instance_type = 'VirtualMachine'
+                or async_job.instance_type = 'ConsoleProxy'
+                or async_job.instance_type = 'SystemVm'
+                or async_job.instance_type = 'DomainRouter'
+            THEN
+            vm_instance.uuid
+        WHEN async_job.instance_type = 'Snapshot' THEN snapshots.uuid
+        WHEN async_job.instance_type = 'Host' THEN host.uuid
+        WHEN async_job.instance_type = 'StoragePool' THEN storage_pool.uuid
+        WHEN async_job.instance_type = 'IpAddress' THEN user_ip_address.uuid
+        WHEN async_job.instance_type = 'SecurityGroup' THEN security_group.uuid
+        WHEN async_job.instance_type = 'PhysicalNetwork' THEN physical_network.uuid
+        WHEN async_job.instance_type = 'TrafficType' THEN physical_network_traffic_types.uuid
+        WHEN async_job.instance_type = 'PhysicalNetworkServiceProvider' THEN physical_network_service_providers.uuid
+        WHEN async_job.instance_type = 'FirewallRule' THEN firewall_rules.uuid
+        WHEN async_job.instance_type = 'Account' THEN acct.uuid
+        WHEN async_job.instance_type = 'User' THEN us.uuid
+        WHEN async_job.instance_type = 'StaticRoute' THEN static_routes.uuid
+        WHEN async_job.instance_type = 'PrivateGateway' THEN vpc_gateways.uuid
+        WHEN async_job.instance_type = 'Counter' THEN counter.uuid
+        WHEN async_job.instance_type = 'Condition' THEN conditions.uuid
+        WHEN async_job.instance_type = 'AutoScalePolicy' THEN autoscale_policies.uuid
+        WHEN async_job.instance_type = 'AutoScaleVmProfile' THEN autoscale_vmprofiles.uuid
+        WHEN async_job.instance_type = 'AutoScaleVmGroup' THEN autoscale_vmgroups.uuid
+        ELSE null
+        END instance_uuid
+from
+    `cloud`.`async_job`
+        left join
+    `cloud`.`account` ON async_job.account_id = account.id
+        left join
+    `cloud`.`domain` ON domain.id = account.domain_id
+        left join
+    `cloud`.`user` ON async_job.user_id = user.id
+        left join
+    `cloud`.`volumes` ON async_job.instance_id = volumes.id
+        left join
+    `cloud`.`vm_template` ON async_job.instance_id = vm_template.id
+        left join
+    `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id
+        left join
+    `cloud`.`snapshots` ON async_job.instance_id = snapshots.id
+        left join
+    `cloud`.`host` ON async_job.instance_id = host.id
+        left join
+    `cloud`.`storage_pool` ON async_job.instance_id = storage_pool.id
+        left join
+    `cloud`.`user_ip_address` ON async_job.instance_id = user_ip_address.id
+        left join
+    `cloud`.`security_group` ON async_job.instance_id = security_group.id
+        left join
+    `cloud`.`physical_network` ON async_job.instance_id = physical_network.id
+        left join
+    `cloud`.`physical_network_traffic_types` ON async_job.instance_id = physical_network_traffic_types.id
+        left join
+    `cloud`.`physical_network_service_providers` ON async_job.instance_id = physical_network_service_providers.id
+        left join
+    `cloud`.`firewall_rules` ON async_job.instance_id = firewall_rules.id
+        left join
+    `cloud`.`account` acct ON async_job.instance_id = acct.id
+        left join
+    `cloud`.`user` us ON async_job.instance_id = us.id
+        left join
+    `cloud`.`static_routes` ON async_job.instance_id = static_routes.id
+        left join
+    `cloud`.`vpc_gateways` ON async_job.instance_id = vpc_gateways.id
+        left join
+    `cloud`.`counter` ON async_job.instance_id = counter.id
+        left join
+    `cloud`.`conditions` ON async_job.instance_id = conditions.id
+        left join
+    `cloud`.`autoscale_policies` ON async_job.instance_id = autoscale_policies.id
+        left join
+    `cloud`.`autoscale_vmprofiles` ON async_job.instance_id = autoscale_vmprofiles.id
+        left join
+    `cloud`.`autoscale_vmgroups` ON async_job.instance_id = autoscale_vmgroups.id;

engine/schema/src/main/resources/META-INF/db/views/cloud.data_center_view.sql

@@ -0,0 +1,59 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`data_center_view`;
+
+DROP VIEW IF EXISTS `cloud`.`data_center_view`;
+
+CREATE VIEW `cloud`.`data_center_view` AS
+select
+    data_center.id,
+    data_center.uuid,
+    data_center.name,
+    data_center.is_security_group_enabled,
+    data_center.is_local_storage_enabled,
+    data_center.description,
+    data_center.dns1,
+    data_center.dns2,
+    data_center.ip6_dns1,
+    data_center.ip6_dns2,
+    data_center.internal_dns1,
+    data_center.internal_dns2,
+    data_center.guest_network_cidr,
+    data_center.domain,
+    data_center.networktype,
+    data_center.allocation_state,
+    data_center.zone_token,
+    data_center.dhcp_provider,
+    data_center.type,
+    data_center.removed,
+    data_center.sort_key,
+    domain.id domain_id,
+    domain.uuid domain_uuid,
+    domain.name domain_name,
+    domain.path domain_path,
+    dedicated_resources.affinity_group_id,
+    dedicated_resources.account_id,
+    affinity_group.uuid affinity_group_uuid
+from
+    `cloud`.`data_center`
+        left join
+    `cloud`.`domain` ON data_center.domain_id = domain.id
+        left join
+    `cloud`.`dedicated_resources` ON data_center.id = dedicated_resources.data_center_id
+        left join
+    `cloud`.`affinity_group` ON dedicated_resources.affinity_group_id = affinity_group.id;

engine/schema/src/main/resources/META-INF/db/views/cloud.disk_offering_view.sql

@@ -0,0 +1,82 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`disk_offering_view`;
+
+DROP VIEW IF EXISTS `cloud`.`disk_offering_view`;
+
+CREATE VIEW `cloud`.`disk_offering_view` AS
+SELECT
+    `disk_offering`.`id` AS `id`,
+    `disk_offering`.`uuid` AS `uuid`,
+    `disk_offering`.`name` AS `name`,
+    `disk_offering`.`display_text` AS `display_text`,
+    `disk_offering`.`provisioning_type` AS `provisioning_type`,
+    `disk_offering`.`disk_size` AS `disk_size`,
+    `disk_offering`.`min_iops` AS `min_iops`,
+    `disk_offering`.`max_iops` AS `max_iops`,
+    `disk_offering`.`created` AS `created`,
+    `disk_offering`.`tags` AS `tags`,
+    `disk_offering`.`customized` AS `customized`,
+    `disk_offering`.`customized_iops` AS `customized_iops`,
+    `disk_offering`.`removed` AS `removed`,
+    `disk_offering`.`use_local_storage` AS `use_local_storage`,
+    `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`,
+    `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`,
+    `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`,
+    `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`,
+    `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`,
+    `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`,
+    `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`,
+    `disk_offering`.`iops_read_rate` AS `iops_read_rate`,
+    `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`,
+    `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`,
+    `disk_offering`.`iops_write_rate` AS `iops_write_rate`,
+    `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`,
+    `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`,
+    `disk_offering`.`cache_mode` AS `cache_mode`,
+    `disk_offering`.`sort_key` AS `sort_key`,
+    `disk_offering`.`compute_only` AS `compute_only`,
+    `disk_offering`.`display_offering` AS `display_offering`,
+    `disk_offering`.`state` AS `state`,
+    `disk_offering`.`disk_size_strictness` AS `disk_size_strictness`,
+    `vsphere_storage_policy`.`value` AS `vsphere_storage_policy`,
+    `disk_offering`.`encrypt` AS `encrypt`,
+    GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id,
+    GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid,
+    GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name,
+    GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path,
+    GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id,
+    GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid,
+    GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name
+FROM
+    `cloud`.`disk_offering`
+        LEFT JOIN
+    `cloud`.`disk_offering_details` AS `domain_details` ON `domain_details`.`offering_id` = `disk_offering`.`id` AND `domain_details`.`name`='domainid'
+        LEFT JOIN
+    `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`)
+        LEFT JOIN
+    `cloud`.`disk_offering_details` AS `zone_details` ON `zone_details`.`offering_id` = `disk_offering`.`id` AND `zone_details`.`name`='zoneid'
+        LEFT JOIN
+    `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`)
+        LEFT JOIN
+    `cloud`.`disk_offering_details` AS `vsphere_storage_policy` ON `vsphere_storage_policy`.`offering_id` = `disk_offering`.`id`
+        AND `vsphere_storage_policy`.`name` = 'storagepolicy'
+WHERE
+        `disk_offering`.`state`='Active'
+GROUP BY
+    `disk_offering`.`id`;

engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql

@@ -0,0 +1,126 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`domain_router_view`;
+
+DROP VIEW IF EXISTS `cloud`.`domain_router_view`;
+
+CREATE VIEW `cloud`.`domain_router_view` AS
+select
+    vm_instance.id id,
+    vm_instance.name name,
+    account.id account_id,
+    account.uuid account_uuid,
+    account.account_name account_name,
+    account.type account_type,
+    domain.id domain_id,
+    domain.uuid domain_uuid,
+    domain.name domain_name,
+    domain.path domain_path,
+    projects.id project_id,
+    projects.uuid project_uuid,
+    projects.name project_name,
+    vm_instance.uuid uuid,
+    vm_instance.created created,
+    vm_instance.state state,
+    vm_instance.removed removed,
+    vm_instance.pod_id pod_id,
+    vm_instance.instance_name instance_name,
+    host_pod_ref.uuid pod_uuid,
+    data_center.id data_center_id,
+    data_center.uuid data_center_uuid,
+    data_center.name data_center_name,
+    data_center.networktype data_center_network_type,
+    data_center.dns1 dns1,
+    data_center.dns2 dns2,
+    data_center.ip6_dns1 ip6_dns1,
+    data_center.ip6_dns2 ip6_dns2,
+    host.id host_id,
+    host.uuid host_uuid,
+    host.name host_name,
+    host.hypervisor_type,
+    host.cluster_id cluster_id,
+    host.status host_status,
+    host.resource_state host_resource_state,
+    vm_template.id template_id,
+    vm_template.uuid template_uuid,
+    service_offering.id service_offering_id,
+    service_offering.uuid service_offering_uuid,
+    service_offering.name service_offering_name,
+    nics.id nic_id,
+    nics.uuid nic_uuid,
+    nics.network_id network_id,
+    nics.ip4_address ip_address,
+    nics.ip6_address ip6_address,
+    nics.ip6_gateway ip6_gateway,
+    nics.ip6_cidr ip6_cidr,
+    nics.default_nic is_default_nic,
+    nics.gateway gateway,
+    nics.netmask netmask,
+    nics.mac_address mac_address,
+    nics.broadcast_uri broadcast_uri,
+    nics.isolation_uri isolation_uri,
+    nics.mtu mtu,
+    vpc.id vpc_id,
+    vpc.uuid vpc_uuid,
+    vpc.name vpc_name,
+    networks.uuid network_uuid,
+    networks.name network_name,
+    networks.network_domain network_domain,
+    networks.traffic_type traffic_type,
+    networks.guest_type guest_type,
+    async_job.id job_id,
+    async_job.uuid job_uuid,
+    async_job.job_status job_status,
+    async_job.account_id job_account_id,
+    domain_router.template_version template_version,
+    domain_router.scripts_version scripts_version,
+    domain_router.is_redundant_router is_redundant_router,
+    domain_router.redundant_state redundant_state,
+    domain_router.stop_pending stop_pending,
+    domain_router.role role,
+    domain_router.software_version software_version
+from
+    `cloud`.`domain_router`
+        inner join
+    `cloud`.`vm_instance` ON vm_instance.id = domain_router.id
+        inner join
+    `cloud`.`account` ON vm_instance.account_id = account.id
+        inner join
+    `cloud`.`domain` ON vm_instance.domain_id = domain.id
+        left join
+    `cloud`.`host_pod_ref` ON vm_instance.pod_id = host_pod_ref.id
+        left join
+    `cloud`.`projects` ON projects.project_account_id = account.id
+        left join
+    `cloud`.`data_center` ON vm_instance.data_center_id = data_center.id
+        left join
+    `cloud`.`host` ON vm_instance.host_id = host.id
+        left join
+    `cloud`.`vm_template` ON vm_instance.vm_template_id = vm_template.id
+        left join
+    `cloud`.`service_offering` ON vm_instance.service_offering_id = service_offering.id
+        left join
+    `cloud`.`nics` ON vm_instance.id = nics.instance_id and nics.removed is null
+        left join
+    `cloud`.`networks` ON nics.network_id = networks.id
+        left join
+    `cloud`.`vpc` ON domain_router.vpc_id = vpc.id and vpc.removed is null
+        left join
+    `cloud`.`async_job` ON async_job.instance_id = vm_instance.id
+        and async_job.instance_type = 'DomainRouter'
+        and async_job.job_status = 0;

engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql

@@ -0,0 +1,111 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`host_view`;
+
+DROP VIEW IF EXISTS `cloud`.`host_view`;
+
+CREATE VIEW `cloud`.`host_view` AS
+SELECT
+    host.id,
+    host.uuid,
+    host.name,
+    host.status,
+    host.disconnected,
+    host.type,
+    host.private_ip_address,
+    host.version,
+    host.hypervisor_type,
+    host.hypervisor_version,
+    host.capabilities,
+    host.last_ping,
+    host.created,
+    host.removed,
+    host.resource_state,
+    host.mgmt_server_id,
+    host.cpu_sockets,
+    host.cpus,
+    host.speed,
+    host.ram,
+    cluster.id cluster_id,
+    cluster.uuid cluster_uuid,
+    cluster.name cluster_name,
+    cluster.cluster_type,
+    data_center.id data_center_id,
+    data_center.uuid data_center_uuid,
+    data_center.name data_center_name,
+    data_center.networktype data_center_type,
+    host_pod_ref.id pod_id,
+    host_pod_ref.uuid pod_uuid,
+    host_pod_ref.name pod_name,
+    GROUP_CONCAT(DISTINCT(host_tags.tag)) AS tag,
+    `host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`,
+    guest_os_category.id guest_os_category_id,
+    guest_os_category.uuid guest_os_category_uuid,
+    guest_os_category.name guest_os_category_name,
+    mem_caps.used_capacity memory_used_capacity,
+    mem_caps.reserved_capacity memory_reserved_capacity,
+    cpu_caps.used_capacity cpu_used_capacity,
+    cpu_caps.reserved_capacity cpu_reserved_capacity,
+    async_job.id job_id,
+    async_job.uuid job_uuid,
+    async_job.job_status job_status,
+    async_job.account_id job_account_id,
+    oobm.enabled AS `oobm_enabled`,
+    oobm.power_state AS `oobm_power_state`,
+    ha_config.enabled AS `ha_enabled`,
+    ha_config.ha_state AS `ha_state`,
+    ha_config.provider AS `ha_provider`,
+    `last_annotation_view`.`annotation` AS `annotation`,
+    `last_annotation_view`.`created` AS `last_annotated`,
+    `user`.`username` AS `username`
+FROM
+    `cloud`.`host`
+        LEFT JOIN
+    `cloud`.`cluster` ON host.cluster_id = cluster.id
+        LEFT JOIN
+    `cloud`.`data_center` ON host.data_center_id = data_center.id
+        LEFT JOIN
+    `cloud`.`host_pod_ref` ON host.pod_id = host_pod_ref.id
+        LEFT JOIN
+    `cloud`.`host_details` ON host.id = host_details.host_id
+        AND host_details.name = 'guest.os.category.id'
+        LEFT JOIN
+    `cloud`.`guest_os_category` ON guest_os_category.id = CONVERT ( host_details.value, UNSIGNED )
+        LEFT JOIN
+    `cloud`.`host_tags` ON host_tags.host_id = host.id
+        LEFT JOIN
+    `cloud`.`op_host_capacity` mem_caps ON host.id = mem_caps.host_id
+        AND mem_caps.capacity_type = 0
+        LEFT JOIN
+    `cloud`.`op_host_capacity` cpu_caps ON host.id = cpu_caps.host_id
+        AND cpu_caps.capacity_type = 1
+        LEFT JOIN
+    `cloud`.`async_job` ON async_job.instance_id = host.id
+        AND async_job.instance_type = 'Host'
+        AND async_job.job_status = 0
+        LEFT JOIN
+    `cloud`.`oobm` ON oobm.host_id = host.id
+        left join
+    `cloud`.`ha_config` ON ha_config.resource_id=host.id
+        and ha_config.resource_type='Host'
+        LEFT JOIN
+    `cloud`.`last_annotation_view` ON `last_annotation_view`.`entity_uuid` = `host`.`uuid`
+        LEFT JOIN
+    `cloud`.`user` ON `user`.`uuid` = `last_annotation_view`.`user_uuid`
+GROUP BY
+    `host`.`id`;

engine/schema/src/main/resources/META-INF/db/views/cloud.network_offering_view.sql

@@ -0,0 +1,85 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`network_offering_view`;
+
+DROP VIEW IF EXISTS `cloud`.`network_offering_view`;
+
+CREATE VIEW `cloud`.`network_offering_view` AS
+SELECT
+    `network_offerings`.`id` AS `id`,
+    `network_offerings`.`uuid` AS `uuid`,
+    `network_offerings`.`name` AS `name`,
+    `network_offerings`.`unique_name` AS `unique_name`,
+    `network_offerings`.`display_text` AS `display_text`,
+    `network_offerings`.`nw_rate` AS `nw_rate`,
+    `network_offerings`.`mc_rate` AS `mc_rate`,
+    `network_offerings`.`traffic_type` AS `traffic_type`,
+    `network_offerings`.`tags` AS `tags`,
+    `network_offerings`.`system_only` AS `system_only`,
+    `network_offerings`.`specify_vlan` AS `specify_vlan`,
+    `network_offerings`.`service_offering_id` AS `service_offering_id`,
+    `network_offerings`.`conserve_mode` AS `conserve_mode`,
+    `network_offerings`.`created` AS `created`,
+    `network_offerings`.`removed` AS `removed`,
+    `network_offerings`.`default` AS `default`,
+    `network_offerings`.`availability` AS `availability`,
+    `network_offerings`.`dedicated_lb_service` AS `dedicated_lb_service`,
+    `network_offerings`.`shared_source_nat_service` AS `shared_source_nat_service`,
+    `network_offerings`.`sort_key` AS `sort_key`,
+    `network_offerings`.`redundant_router_service` AS `redundant_router_service`,
+    `network_offerings`.`state` AS `state`,
+    `network_offerings`.`guest_type` AS `guest_type`,
+    `network_offerings`.`elastic_ip_service` AS `elastic_ip_service`,
+    `network_offerings`.`eip_associate_public_ip` AS `eip_associate_public_ip`,
+    `network_offerings`.`elastic_lb_service` AS `elastic_lb_service`,
+    `network_offerings`.`specify_ip_ranges` AS `specify_ip_ranges`,
+    `network_offerings`.`inline` AS `inline`,
+    `network_offerings`.`is_persistent` AS `is_persistent`,
+    `network_offerings`.`internal_lb` AS `internal_lb`,
+    `network_offerings`.`public_lb` AS `public_lb`,
+    `network_offerings`.`egress_default_policy` AS `egress_default_policy`,
+    `network_offerings`.`concurrent_connections` AS `concurrent_connections`,
+    `network_offerings`.`keep_alive_enabled` AS `keep_alive_enabled`,
+    `network_offerings`.`supports_streched_l2` AS `supports_streched_l2`,
+    `network_offerings`.`supports_public_access` AS `supports_public_access`,
+    `network_offerings`.`supports_vm_autoscaling` AS `supports_vm_autoscaling`,
+    `network_offerings`.`for_vpc` AS `for_vpc`,
+    `network_offerings`.`for_tungsten` AS `for_tungsten`,
+    `network_offerings`.`service_package_id` AS `service_package_id`,
+    GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id,
+    GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid,
+    GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name,
+    GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path,
+    GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id,
+    GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid,
+    GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name,
+    `offering_details`.value AS internet_protocol
+FROM
+    `cloud`.`network_offerings`
+        LEFT JOIN
+    `cloud`.`network_offering_details` AS `domain_details` ON `domain_details`.`network_offering_id` = `network_offerings`.`id` AND `domain_details`.`name`='domainid'
+        LEFT JOIN
+    `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`)
+        LEFT JOIN
+    `cloud`.`network_offering_details` AS `zone_details` ON `zone_details`.`network_offering_id` = `network_offerings`.`id` AND `zone_details`.`name`='zoneid'
+        LEFT JOIN
+    `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`)
+        LEFT JOIN
+    `cloud`.`network_offering_details` AS `offering_details` ON `offering_details`.`network_offering_id` = `network_offerings`.`id` AND `offering_details`.`name`='internetProtocol'
+GROUP BY
+    `network_offerings`.`id`;

engine/schema/src/main/resources/META-INF/db/views/cloud.service_offering_view.sql

@@ -0,0 +1,114 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`service_offering_view`;
+
+DROP VIEW IF EXISTS `cloud`.`service_offering_view`;
+
+CREATE VIEW `cloud`.`service_offering_view` AS
+SELECT
+    `service_offering`.`id` AS `id`,
+    `service_offering`.`uuid` AS `uuid`,
+    `service_offering`.`name` AS `name`,
+    `service_offering`.`display_text` AS `display_text`,
+    `disk_offering`.`provisioning_type` AS `provisioning_type`,
+    `service_offering`.`created` AS `created`,
+    `disk_offering`.`tags` AS `tags`,
+    `service_offering`.`removed` AS `removed`,
+    `disk_offering`.`use_local_storage` AS `use_local_storage`,
+    `service_offering`.`system_use` AS `system_use`,
+    `disk_offering`.`id` AS `disk_offering_id`,
+    `disk_offering`.`name` AS `disk_offering_name`,
+    `disk_offering`.`uuid` AS `disk_offering_uuid`,
+    `disk_offering`.`display_text` AS `disk_offering_display_text`,
+    `disk_offering`.`customized_iops` AS `customized_iops`,
+    `disk_offering`.`min_iops` AS `min_iops`,
+    `disk_offering`.`max_iops` AS `max_iops`,
+    `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`,
+    `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`,
+    `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`,
+    `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`,
+    `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`,
+    `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`,
+    `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`,
+    `disk_offering`.`iops_read_rate` AS `iops_read_rate`,
+    `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`,
+    `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`,
+    `disk_offering`.`iops_write_rate` AS `iops_write_rate`,
+    `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`,
+    `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`,
+    `disk_offering`.`cache_mode` AS `cache_mode`,
+    `disk_offering`.`disk_size` AS `root_disk_size`,
+    `disk_offering`.`encrypt` AS `encrypt_root`,
+    `service_offering`.`cpu` AS `cpu`,
+    `service_offering`.`speed` AS `speed`,
+    `service_offering`.`ram_size` AS `ram_size`,
+    `service_offering`.`nw_rate` AS `nw_rate`,
+    `service_offering`.`mc_rate` AS `mc_rate`,
+    `service_offering`.`ha_enabled` AS `ha_enabled`,
+    `service_offering`.`limit_cpu_use` AS `limit_cpu_use`,
+    `service_offering`.`host_tag` AS `host_tag`,
+    `service_offering`.`default_use` AS `default_use`,
+    `service_offering`.`vm_type` AS `vm_type`,
+    `service_offering`.`sort_key` AS `sort_key`,
+    `service_offering`.`is_volatile` AS `is_volatile`,
+    `service_offering`.`deployment_planner` AS `deployment_planner`,
+    `service_offering`.`dynamic_scaling_enabled` AS `dynamic_scaling_enabled`,
+    `service_offering`.`disk_offering_strictness` AS `disk_offering_strictness`,
+    `vsphere_storage_policy`.`value` AS `vsphere_storage_policy`,
+    GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id,
+    GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid,
+    GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name,
+    GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path,
+    GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id,
+    GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid,
+    GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name,
+    IFNULL(`min_compute_details`.`value`, `cpu`) AS min_cpu,
+    IFNULL(`max_compute_details`.`value`, `cpu`) AS max_cpu,
+    IFNULL(`min_memory_details`.`value`, `ram_size`) AS min_memory,
+    IFNULL(`max_memory_details`.`value`, `ram_size`) AS max_memory
+FROM
+    `cloud`.`service_offering`
+        INNER JOIN
+    `cloud`.`disk_offering_view` AS `disk_offering` ON service_offering.disk_offering_id = disk_offering.id
+        LEFT JOIN
+    `cloud`.`service_offering_details` AS `domain_details` ON `domain_details`.`service_offering_id` = `service_offering`.`id` AND `domain_details`.`name`='domainid'
+        LEFT JOIN
+    `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`)
+        LEFT JOIN
+    `cloud`.`service_offering_details` AS `zone_details` ON `zone_details`.`service_offering_id` = `service_offering`.`id` AND `zone_details`.`name`='zoneid'
+        LEFT JOIN
+    `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`)
+        LEFT JOIN
+    `cloud`.`service_offering_details` AS `min_compute_details` ON `min_compute_details`.`service_offering_id` = `service_offering`.`id`
+        AND `min_compute_details`.`name` = 'mincpunumber'
+        LEFT JOIN
+    `cloud`.`service_offering_details` AS `max_compute_details` ON `max_compute_details`.`service_offering_id` = `service_offering`.`id`
+        AND `max_compute_details`.`name` = 'maxcpunumber'
+        LEFT JOIN
+    `cloud`.`service_offering_details` AS `min_memory_details` ON `min_memory_details`.`service_offering_id` = `service_offering`.`id`
+        AND `min_memory_details`.`name` = 'minmemory'
+        LEFT JOIN
+    `cloud`.`service_offering_details` AS `max_memory_details` ON `max_memory_details`.`service_offering_id` = `service_offering`.`id`
+        AND `max_memory_details`.`name` = 'maxmemory'
+        LEFT JOIN
+    `cloud`.`service_offering_details` AS `vsphere_storage_policy` ON `vsphere_storage_policy`.`service_offering_id` = `service_offering`.`id`
+        AND `vsphere_storage_policy`.`name` = 'storagepolicy'
+WHERE
+        `service_offering`.`state`='Active'
+GROUP BY
+    `service_offering`.`id`;

engine/schema/src/main/resources/META-INF/db/views/cloud.snapshot_view.sql

@@ -0,0 +1,107 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`snapshot_view`;
+
+DROP VIEW IF EXISTS `cloud`.`snapshot_view`;
+
+CREATE VIEW `cloud`.`snapshot_view` AS
+SELECT
+    `snapshots`.`id` AS `id`,
+    `snapshots`.`uuid` AS `uuid`,
+    `snapshots`.`name` AS `name`,
+    `snapshots`.`status` AS `status`,
+    `snapshots`.`disk_offering_id` AS `disk_offering_id`,
+    `snapshots`.`snapshot_type` AS `snapshot_type`,
+    `snapshots`.`type_description` AS `type_description`,
+    `snapshots`.`size` AS `size`,
+    `snapshots`.`created` AS `created`,
+    `snapshots`.`removed` AS `removed`,
+    `snapshots`.`location_type` AS `location_type`,
+    `snapshots`.`hypervisor_type` AS `hypervisor_type`,
+    `account`.`id` AS `account_id`,
+    `account`.`uuid` AS `account_uuid`,
+    `account`.`account_name` AS `account_name`,
+    `account`.`type` AS `account_type`,
+    `domain`.`id` AS `domain_id`,
+    `domain`.`uuid` AS `domain_uuid`,
+    `domain`.`name` AS `domain_name`,
+    `domain`.`path` AS `domain_path`,
+    `projects`.`id` AS `project_id`,
+    `projects`.`uuid` AS `project_uuid`,
+    `projects`.`name` AS `project_name`,
+    `volumes`.`id` AS `volume_id`,
+    `volumes`.`uuid` AS `volume_uuid`,
+    `volumes`.`name` AS `volume_name`,
+    `volumes`.`volume_type` AS `volume_type`,
+    `volumes`.`size` AS `volume_size`,
+    `data_center`.`id` AS `data_center_id`,
+    `data_center`.`uuid` AS `data_center_uuid`,
+    `data_center`.`name` AS `data_center_name`,
+    `snapshot_store_ref`.`store_id` AS `store_id`,
+    IFNULL(`image_store`.`uuid`, `storage_pool`.`uuid`) AS `store_uuid`,
+    IFNULL(`image_store`.`name`, `storage_pool`.`name`) AS `store_name`,
+    `snapshot_store_ref`.`store_role` AS `store_role`,
+    `snapshot_store_ref`.`state` AS `store_state`,
+    `snapshot_store_ref`.`download_state` AS `download_state`,
+    `snapshot_store_ref`.`download_pct` AS `download_pct`,
+    `snapshot_store_ref`.`error_str` AS `error_str`,
+    `snapshot_store_ref`.`size` AS `store_size`,
+    `snapshot_store_ref`.`created` AS `created_on_store`,
+    `resource_tags`.`id` AS `tag_id`,
+    `resource_tags`.`uuid` AS `tag_uuid`,
+    `resource_tags`.`key` AS `tag_key`,
+    `resource_tags`.`value` AS `tag_value`,
+    `resource_tags`.`domain_id` AS `tag_domain_id`,
+    `domain`.`uuid` AS `tag_domain_uuid`,
+    `domain`.`name` AS `tag_domain_name`,
+    `resource_tags`.`account_id` AS `tag_account_id`,
+    `account`.`account_name` AS `tag_account_name`,
+    `resource_tags`.`resource_id` AS `tag_resource_id`,
+    `resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
+    `resource_tags`.`resource_type` AS `tag_resource_type`,
+    `resource_tags`.`customer` AS `tag_customer`,
+    CONCAT(`snapshots`.`id`,
+           '_',
+           IFNULL(`snapshot_store_ref`.`store_role`, 'UNKNOWN'),
+           '_',
+           IFNULL(`snapshot_store_ref`.`store_id`, 0)) AS `snapshot_store_pair`
+FROM
+    ((((((((((`snapshots`
+        JOIN `account` ON ((`account`.`id` = `snapshots`.`account_id`)))
+        JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`)))
+        LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
+        LEFT JOIN `volumes` ON ((`volumes`.`id` = `snapshots`.`volume_id`)))
+        LEFT JOIN `snapshot_store_ref` ON (((`snapshot_store_ref`.`snapshot_id` = `snapshots`.`id`)
+        AND (`snapshot_store_ref`.`state` != 'Destroyed')
+        AND (`snapshot_store_ref`.`display` = 1))))
+        LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`)
+        AND (`snapshot_store_ref`.`store_role` = 'Image')
+        AND (`snapshot_store_ref`.`store_id` IS NOT NULL)
+        AND (`image_store`.`id` = `snapshot_store_ref`.`store_id`))))
+        LEFT JOIN `storage_pool` ON ((ISNULL(`storage_pool`.`removed`)
+        AND (`snapshot_store_ref`.`store_role` = 'Primary')
+        AND (`snapshot_store_ref`.`store_id` IS NOT NULL)
+        AND (`storage_pool`.`id` = `snapshot_store_ref`.`store_id`))))
+        LEFT JOIN `snapshot_zone_ref` ON (((`snapshot_zone_ref`.`snapshot_id` = `snapshots`.`id`)
+        AND ISNULL(`snapshot_store_ref`.`store_id`)
+        AND ISNULL(`snapshot_zone_ref`.`removed`))))
+        LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`)
+        OR (`storage_pool`.`data_center_id` = `data_center`.`id`)
+        OR (`snapshot_zone_ref`.`zone_id` = `data_center`.`id`))))
+        LEFT JOIN `resource_tags` ON ((`resource_tags`.`resource_id` = `snapshots`.`id`)
+        AND (`resource_tags`.`resource_type` = 'Snapshot')));

engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql

@@ -0,0 +1,68 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`storage_pool_view`;
+
+DROP VIEW IF EXISTS `cloud`.`storage_pool_view`;
+
+CREATE VIEW `cloud`.`storage_pool_view` AS
+SELECT
+    `storage_pool`.`id` AS `id`,
+    `storage_pool`.`uuid` AS `uuid`,
+    `storage_pool`.`name` AS `name`,
+    `storage_pool`.`status` AS `status`,
+    `storage_pool`.`path` AS `path`,
+    `storage_pool`.`pool_type` AS `pool_type`,
+    `storage_pool`.`host_address` AS `host_address`,
+    `storage_pool`.`created` AS `created`,
+    `storage_pool`.`removed` AS `removed`,
+    `storage_pool`.`capacity_bytes` AS `capacity_bytes`,
+    `storage_pool`.`capacity_iops` AS `capacity_iops`,
+    `storage_pool`.`scope` AS `scope`,
+    `storage_pool`.`hypervisor` AS `hypervisor`,
+    `storage_pool`.`storage_provider_name` AS `storage_provider_name`,
+    `storage_pool`.`parent` AS `parent`,
+    `cluster`.`id` AS `cluster_id`,
+    `cluster`.`uuid` AS `cluster_uuid`,
+    `cluster`.`name` AS `cluster_name`,
+    `cluster`.`cluster_type` AS `cluster_type`,
+    `data_center`.`id` AS `data_center_id`,
+    `data_center`.`uuid` AS `data_center_uuid`,
+    `data_center`.`name` AS `data_center_name`,
+    `data_center`.`networktype` AS `data_center_type`,
+    `host_pod_ref`.`id` AS `pod_id`,
+    `host_pod_ref`.`uuid` AS `pod_uuid`,
+    `host_pod_ref`.`name` AS `pod_name`,
+    `storage_pool_tags`.`tag` AS `tag`,
+    `storage_pool_tags`.`is_tag_a_rule` AS `is_tag_a_rule`,
+    `op_host_capacity`.`used_capacity` AS `disk_used_capacity`,
+    `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`,
+    `async_job`.`id` AS `job_id`,
+    `async_job`.`uuid` AS `job_uuid`,
+    `async_job`.`job_status` AS `job_status`,
+    `async_job`.`account_id` AS `job_account_id`
+FROM
+    ((((((`cloud`.`storage_pool`
+        LEFT JOIN `cloud`.`cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`)))
+        LEFT JOIN `cloud`.`data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`)))
+        LEFT JOIN `cloud`.`host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`)))
+        LEFT JOIN `cloud`.`storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`))))
+        LEFT JOIN `cloud`.`op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`)
+        AND (`op_host_capacity`.`capacity_type` IN (3 , 9)))))
+        LEFT JOIN `cloud`.`async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`)
+        AND (`async_job`.`instance_type` = 'StoragePool')
+        AND (`async_job`.`job_status` = 0))));

engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql

@@ -0,0 +1,131 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`template_view`;
+
+DROP VIEW IF EXISTS `cloud`.`template_view`;
+
+CREATE VIEW `cloud`.`template_view` AS
+SELECT
+    `vm_template`.`id` AS `id`,
+    `vm_template`.`uuid` AS `uuid`,
+    `vm_template`.`unique_name` AS `unique_name`,
+    `vm_template`.`name` AS `name`,
+    `vm_template`.`public` AS `public`,
+    `vm_template`.`featured` AS `featured`,
+    `vm_template`.`type` AS `type`,
+    `vm_template`.`hvm` AS `hvm`,
+    `vm_template`.`bits` AS `bits`,
+    `vm_template`.`url` AS `url`,
+    `vm_template`.`format` AS `format`,
+    `vm_template`.`created` AS `created`,
+    `vm_template`.`checksum` AS `checksum`,
+    `vm_template`.`display_text` AS `display_text`,
+    `vm_template`.`enable_password` AS `enable_password`,
+    `vm_template`.`dynamically_scalable` AS `dynamically_scalable`,
+    `vm_template`.`state` AS `template_state`,
+    `vm_template`.`guest_os_id` AS `guest_os_id`,
+    `guest_os`.`uuid` AS `guest_os_uuid`,
+    `guest_os`.`display_name` AS `guest_os_name`,
+    `vm_template`.`bootable` AS `bootable`,
+    `vm_template`.`prepopulate` AS `prepopulate`,
+    `vm_template`.`cross_zones` AS `cross_zones`,
+    `vm_template`.`hypervisor_type` AS `hypervisor_type`,
+    `vm_template`.`extractable` AS `extractable`,
+    `vm_template`.`template_tag` AS `template_tag`,
+    `vm_template`.`sort_key` AS `sort_key`,
+    `vm_template`.`removed` AS `removed`,
+    `vm_template`.`enable_sshkey` AS `enable_sshkey`,
+    `parent_template`.`id` AS `parent_template_id`,
+    `parent_template`.`uuid` AS `parent_template_uuid`,
+    `source_template`.`id` AS `source_template_id`,
+    `source_template`.`uuid` AS `source_template_uuid`,
+    `account`.`id` AS `account_id`,
+    `account`.`uuid` AS `account_uuid`,
+    `account`.`account_name` AS `account_name`,
+    `account`.`type` AS `account_type`,
+    `domain`.`id` AS `domain_id`,
+    `domain`.`uuid` AS `domain_uuid`,
+    `domain`.`name` AS `domain_name`,
+    `domain`.`path` AS `domain_path`,
+    `projects`.`id` AS `project_id`,
+    `projects`.`uuid` AS `project_uuid`,
+    `projects`.`name` AS `project_name`,
+    `data_center`.`id` AS `data_center_id`,
+    `data_center`.`uuid` AS `data_center_uuid`,
+    `data_center`.`name` AS `data_center_name`,
+    `launch_permission`.`account_id` AS `lp_account_id`,
+    `template_store_ref`.`store_id` AS `store_id`,
+    `image_store`.`scope` AS `store_scope`,
+    `template_store_ref`.`state` AS `state`,
+    `template_store_ref`.`download_state` AS `download_state`,
+    `template_store_ref`.`download_pct` AS `download_pct`,
+    `template_store_ref`.`error_str` AS `error_str`,
+    `template_store_ref`.`size` AS `size`,
+    `template_store_ref`.physical_size AS `physical_size`,
+    `template_store_ref`.`destroyed` AS `destroyed`,
+    `template_store_ref`.`created` AS `created_on_store`,
+    `vm_template_details`.`name` AS `detail_name`,
+    `vm_template_details`.`value` AS `detail_value`,
+    `resource_tags`.`id` AS `tag_id`,
+    `resource_tags`.`uuid` AS `tag_uuid`,
+    `resource_tags`.`key` AS `tag_key`,
+    `resource_tags`.`value` AS `tag_value`,
+    `resource_tags`.`domain_id` AS `tag_domain_id`,
+    `domain`.`uuid` AS `tag_domain_uuid`,
+    `domain`.`name` AS `tag_domain_name`,
+    `resource_tags`.`account_id` AS `tag_account_id`,
+    `account`.`account_name` AS `tag_account_name`,
+    `resource_tags`.`resource_id` AS `tag_resource_id`,
+    `resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
+    `resource_tags`.`resource_type` AS `tag_resource_type`,
+    `resource_tags`.`customer` AS `tag_customer`,
+    CONCAT(`vm_template`.`id`,
+           '_',
+           IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`,
+    `vm_template`.`direct_download` AS `direct_download`,
+    `vm_template`.`deploy_as_is` AS `deploy_as_is`,
+    `user_data`.`id` AS `user_data_id`,
+    `user_data`.`uuid` AS `user_data_uuid`,
+    `user_data`.`name` AS `user_data_name`,
+    `user_data`.`params` AS `user_data_params`,
+    `vm_template`.`user_data_link_policy` AS `user_data_policy`
+FROM
+    (((((((((((((`vm_template`
+        JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`)))
+        JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`)))
+        JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`)))
+        LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
+        LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`)))
+        LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`)))
+        LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`)
+            AND (`template_store_ref`.`store_role` = 'Image')
+            AND (`template_store_ref`.`destroyed` = 0))))
+        LEFT JOIN `vm_template` `parent_template` ON ((`parent_template`.`id` = `vm_template`.`parent_template_id`)))
+        LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`)
+            AND (`template_store_ref`.`store_id` IS NOT NULL)
+            AND (`image_store`.`id` = `template_store_ref`.`store_id`))))
+        LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`)
+            AND ISNULL(`template_store_ref`.`store_id`)
+            AND ISNULL(`template_zone_ref`.`removed`))))
+        LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`)
+            OR (`template_zone_ref`.`zone_id` = `data_center`.`id`))))
+        LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`)))
+        LEFT JOIN `user_data` ON ((`user_data`.`id` = `vm_template`.`user_data_id`))
+        LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`)
+        AND ((`resource_tags`.`resource_type` = 'Template')
+            OR (`resource_tags`.`resource_type` = 'ISO')))));

engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql

@@ -0,0 +1,65 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`user_view`;
+
+DROP VIEW IF EXISTS `cloud`.`user_view`;
+
+CREATE VIEW `cloud`.`user_view` AS
+select
+    user.id,
+    user.uuid,
+    user.username,
+    user.password,
+    user.firstname,
+    user.lastname,
+    user.email,
+    user.state,
+    user.api_key,
+    user.secret_key,
+    user.created,
+    user.removed,
+    user.timezone,
+    user.registration_token,
+    user.is_registered,
+    user.incorrect_login_attempts,
+    user.source,
+    user.default,
+    account.id account_id,
+    account.uuid account_uuid,
+    account.account_name account_name,
+    account.type account_type,
+    account.role_id account_role_id,
+    domain.id domain_id,
+    domain.uuid domain_uuid,
+    domain.name domain_name,
+    domain.path domain_path,
+    async_job.id job_id,
+    async_job.uuid job_uuid,
+    async_job.job_status job_status,
+    async_job.account_id job_account_id,
+    user.is_user_2fa_enabled is_user_2fa_enabled
+from
+    `cloud`.`user`
+        inner join
+    `cloud`.`account` ON user.account_id = account.id
+        inner join
+    `cloud`.`domain` ON account.domain_id = domain.id
+        left join
+    `cloud`.`async_job` ON async_job.instance_id = user.id
+        and async_job.instance_type = 'User'
+        and async_job.job_status = 0;

engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql

@@ -0,0 +1,215 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- VIEW `cloud`.`user_vm_view`;
+
+DROP VIEW IF EXISTS `cloud`.`user_vm_view`;
+
+CREATE VIEW `user_vm_view` AS
+SELECT
+    `vm_instance`.`id` AS `id`,
+    `vm_instance`.`name` AS `name`,
+    `user_vm`.`display_name` AS `display_name`,
+    `user_vm`.`user_data` AS `user_data`,
+    `account`.`id` AS `account_id`,
+    `account`.`uuid` AS `account_uuid`,
+    `account`.`account_name` AS `account_name`,
+    `account`.`type` AS `account_type`,
+    `domain`.`id` AS `domain_id`,
+    `domain`.`uuid` AS `domain_uuid`,
+    `domain`.`name` AS `domain_name`,
+    `domain`.`path` AS `domain_path`,
+    `projects`.`id` AS `project_id`,
+    `projects`.`uuid` AS `project_uuid`,
+    `projects`.`name` AS `project_name`,
+    `instance_group`.`id` AS `instance_group_id`,
+    `instance_group`.`uuid` AS `instance_group_uuid`,
+    `instance_group`.`name` AS `instance_group_name`,
+    `vm_instance`.`uuid` AS `uuid`,
+    `vm_instance`.`user_id` AS `user_id`,
+    `vm_instance`.`last_host_id` AS `last_host_id`,
+    `vm_instance`.`vm_type` AS `type`,
+    `vm_instance`.`limit_cpu_use` AS `limit_cpu_use`,
+    `vm_instance`.`created` AS `created`,
+    `vm_instance`.`state` AS `state`,
+    `vm_instance`.`update_time` AS `update_time`,
+    `vm_instance`.`removed` AS `removed`,
+    `vm_instance`.`ha_enabled` AS `ha_enabled`,
+    `vm_instance`.`hypervisor_type` AS `hypervisor_type`,
+    `vm_instance`.`instance_name` AS `instance_name`,
+    `vm_instance`.`guest_os_id` AS `guest_os_id`,
+    `vm_instance`.`display_vm` AS `display_vm`,
+    `guest_os`.`uuid` AS `guest_os_uuid`,
+    `vm_instance`.`pod_id` AS `pod_id`,
+    `host_pod_ref`.`uuid` AS `pod_uuid`,
+    `vm_instance`.`private_ip_address` AS `private_ip_address`,
+    `vm_instance`.`private_mac_address` AS `private_mac_address`,
+    `vm_instance`.`vm_type` AS `vm_type`,
+    `data_center`.`id` AS `data_center_id`,
+    `data_center`.`uuid` AS `data_center_uuid`,
+    `data_center`.`name` AS `data_center_name`,
+    `data_center`.`is_security_group_enabled` AS `security_group_enabled`,
+    `data_center`.`networktype` AS `data_center_network_type`,
+    `host`.`id` AS `host_id`,
+    `host`.`uuid` AS `host_uuid`,
+    `host`.`name` AS `host_name`,
+    `host`.`cluster_id` AS `cluster_id`,
+    `host`.`status` AS `host_status`,
+    `host`.`resource_state` AS `host_resource_state`,
+    `vm_template`.`id` AS `template_id`,
+    `vm_template`.`uuid` AS `template_uuid`,
+    `vm_template`.`name` AS `template_name`,
+    `vm_template`.`type` AS `template_type`,
+    `vm_template`.`display_text` AS `template_display_text`,
+    `vm_template`.`enable_password` AS `password_enabled`,
+    `iso`.`id` AS `iso_id`,
+    `iso`.`uuid` AS `iso_uuid`,
+    `iso`.`name` AS `iso_name`,
+    `iso`.`display_text` AS `iso_display_text`,
+    `service_offering`.`id` AS `service_offering_id`,
+    `service_offering`.`uuid` AS `service_offering_uuid`,
+    `disk_offering`.`uuid` AS `disk_offering_uuid`,
+    `disk_offering`.`id` AS `disk_offering_id`,
+    (CASE
+         WHEN ISNULL(`service_offering`.`cpu`) THEN `custom_cpu`.`value`
+         ELSE `service_offering`.`cpu`
+        END) AS `cpu`,
+    (CASE
+         WHEN ISNULL(`service_offering`.`speed`) THEN `custom_speed`.`value`
+         ELSE `service_offering`.`speed`
+        END) AS `speed`,
+    (CASE
+         WHEN ISNULL(`service_offering`.`ram_size`) THEN `custom_ram_size`.`value`
+         ELSE `service_offering`.`ram_size`
+        END) AS `ram_size`,
+    `backup_offering`.`uuid` AS `backup_offering_uuid`,
+    `backup_offering`.`id` AS `backup_offering_id`,
+    `service_offering`.`name` AS `service_offering_name`,
+    `disk_offering`.`name` AS `disk_offering_name`,
+    `backup_offering`.`name` AS `backup_offering_name`,
+    `storage_pool`.`id` AS `pool_id`,
+    `storage_pool`.`uuid` AS `pool_uuid`,
+    `storage_pool`.`pool_type` AS `pool_type`,
+    `volumes`.`id` AS `volume_id`,
+    `volumes`.`uuid` AS `volume_uuid`,
+    `volumes`.`device_id` AS `volume_device_id`,
+    `volumes`.`volume_type` AS `volume_type`,
+    `security_group`.`id` AS `security_group_id`,
+    `security_group`.`uuid` AS `security_group_uuid`,
+    `security_group`.`name` AS `security_group_name`,
+    `security_group`.`description` AS `security_group_description`,
+    `nics`.`id` AS `nic_id`,
+    `nics`.`uuid` AS `nic_uuid`,
+    `nics`.`device_id` AS `nic_device_id`,
+    `nics`.`network_id` AS `network_id`,
+    `nics`.`ip4_address` AS `ip_address`,
+    `nics`.`ip6_address` AS `ip6_address`,
+    `nics`.`ip6_gateway` AS `ip6_gateway`,
+    `nics`.`ip6_cidr` AS `ip6_cidr`,
+    `nics`.`default_nic` AS `is_default_nic`,
+    `nics`.`gateway` AS `gateway`,
+    `nics`.`netmask` AS `netmask`,
+    `nics`.`mac_address` AS `mac_address`,
+    `nics`.`broadcast_uri` AS `broadcast_uri`,
+    `nics`.`isolation_uri` AS `isolation_uri`,
+    `vpc`.`id` AS `vpc_id`,
+    `vpc`.`uuid` AS `vpc_uuid`,
+    `networks`.`uuid` AS `network_uuid`,
+    `networks`.`name` AS `network_name`,
+    `networks`.`traffic_type` AS `traffic_type`,
+    `networks`.`guest_type` AS `guest_type`,
+    `user_ip_address`.`id` AS `public_ip_id`,
+    `user_ip_address`.`uuid` AS `public_ip_uuid`,
+    `user_ip_address`.`public_ip_address` AS `public_ip_address`,
+    `ssh_details`.`value` AS `keypair_names`,
+    `resource_tags`.`id` AS `tag_id`,
+    `resource_tags`.`uuid` AS `tag_uuid`,
+    `resource_tags`.`key` AS `tag_key`,
+    `resource_tags`.`value` AS `tag_value`,
+    `resource_tags`.`domain_id` AS `tag_domain_id`,
+    `domain`.`uuid` AS `tag_domain_uuid`,
+    `domain`.`name` AS `tag_domain_name`,
+    `resource_tags`.`account_id` AS `tag_account_id`,
+    `account`.`account_name` AS `tag_account_name`,
+    `resource_tags`.`resource_id` AS `tag_resource_id`,
+    `resource_tags`.`resource_uuid` AS `tag_resource_uuid`,
+    `resource_tags`.`resource_type` AS `tag_resource_type`,
+    `resource_tags`.`customer` AS `tag_customer`,
+    `async_job`.`id` AS `job_id`,
+    `async_job`.`uuid` AS `job_uuid`,
+    `async_job`.`job_status` AS `job_status`,
+    `async_job`.`account_id` AS `job_account_id`,
+    `affinity_group`.`id` AS `affinity_group_id`,
+    `affinity_group`.`uuid` AS `affinity_group_uuid`,
+    `affinity_group`.`name` AS `affinity_group_name`,
+    `affinity_group`.`description` AS `affinity_group_description`,
+    `autoscale_vmgroups`.`id` AS `autoscale_vmgroup_id`,
+    `autoscale_vmgroups`.`uuid` AS `autoscale_vmgroup_uuid`,
+    `autoscale_vmgroups`.`name` AS `autoscale_vmgroup_name`,
+    `vm_instance`.`dynamically_scalable` AS `dynamically_scalable`,
+    `user_data`.`id` AS `user_data_id`,
+    `user_data`.`uuid` AS `user_data_uuid`,
+    `user_data`.`name` AS `user_data_name`,
+    `user_vm`.`user_data_details` AS `user_data_details`,
+    `vm_template`.`user_data_link_policy` AS `user_data_policy`
+FROM
+    (((((((((((((((((((((((((((((((((((`user_vm`
+        JOIN `vm_instance` ON (((`vm_instance`.`id` = `user_vm`.`id`)
+            AND ISNULL(`vm_instance`.`removed`))))
+        JOIN `account` ON ((`vm_instance`.`account_id` = `account`.`id`)))
+        JOIN `domain` ON ((`vm_instance`.`domain_id` = `domain`.`id`)))
+        LEFT JOIN `guest_os` ON ((`vm_instance`.`guest_os_id` = `guest_os`.`id`)))
+        LEFT JOIN `host_pod_ref` ON ((`vm_instance`.`pod_id` = `host_pod_ref`.`id`)))
+        LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`)))
+        LEFT JOIN `instance_group_vm_map` ON ((`vm_instance`.`id` = `instance_group_vm_map`.`instance_id`)))
+        LEFT JOIN `instance_group` ON ((`instance_group_vm_map`.`group_id` = `instance_group`.`id`)))
+        LEFT JOIN `data_center` ON ((`vm_instance`.`data_center_id` = `data_center`.`id`)))
+        LEFT JOIN `host` ON ((`vm_instance`.`host_id` = `host`.`id`)))
+        LEFT JOIN `vm_template` ON ((`vm_instance`.`vm_template_id` = `vm_template`.`id`)))
+        LEFT JOIN `vm_template` `iso` ON ((`iso`.`id` = `user_vm`.`iso_id`)))
+        LEFT JOIN `volumes` ON ((`vm_instance`.`id` = `volumes`.`instance_id`)))
+        LEFT JOIN `service_offering` ON ((`vm_instance`.`service_offering_id` = `service_offering`.`id`)))
+        LEFT JOIN `disk_offering` `svc_disk_offering` ON ((`volumes`.`disk_offering_id` = `svc_disk_offering`.`id`)))
+        LEFT JOIN `disk_offering` ON ((`volumes`.`disk_offering_id` = `disk_offering`.`id`)))
+        LEFT JOIN `backup_offering` ON ((`vm_instance`.`backup_offering_id` = `backup_offering`.`id`)))
+        LEFT JOIN `storage_pool` ON ((`volumes`.`pool_id` = `storage_pool`.`id`)))
+        LEFT JOIN `security_group_vm_map` ON ((`vm_instance`.`id` = `security_group_vm_map`.`instance_id`)))
+        LEFT JOIN `security_group` ON ((`security_group_vm_map`.`security_group_id` = `security_group`.`id`)))
+        LEFT JOIN `user_data` ON ((`user_data`.`id` = `user_vm`.`user_data_id`)))
+        LEFT JOIN `nics` ON (((`vm_instance`.`id` = `nics`.`instance_id`)
+        AND ISNULL(`nics`.`removed`))))
+        LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`)))
+        LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`)
+        AND ISNULL(`vpc`.`removed`))))
+        LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`)))
+        LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`)
+        AND (`ssh_details`.`name` = 'SSH.KeyPairNames'))))
+        LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`)
+        AND (`resource_tags`.`resource_type` = 'UserVm'))))
+        LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `vm_instance`.`id`)
+        AND (`async_job`.`instance_type` = 'VirtualMachine')
+        AND (`async_job`.`job_status` = 0))))
+        LEFT JOIN `affinity_group_vm_map` ON ((`vm_instance`.`id` = `affinity_group_vm_map`.`instance_id`)))
+        LEFT JOIN `affinity_group` ON ((`affinity_group_vm_map`.`affinity_group_id` = `affinity_group`.`id`)))
+        LEFT JOIN `autoscale_vmgroup_vm_map` ON ((`autoscale_vmgroup_vm_map`.`instance_id` = `vm_instance`.`id`)))
+        LEFT JOIN `autoscale_vmgroups` ON ((`autoscale_vmgroup_vm_map`.`vmgroup_id` = `autoscale_vmgroups`.`id`)))
+        LEFT JOIN `user_vm_details` `custom_cpu` ON (((`custom_cpu`.`vm_id` = `vm_instance`.`id`)
+        AND (`custom_cpu`.`name` = 'CpuNumber'))))
+        LEFT JOIN `user_vm_details` `custom_speed` ON (((`custom_speed`.`vm_id` = `vm_instance`.`id`)
+        AND (`custom_speed`.`name` = 'CpuSpeed'))))
+        LEFT JOIN `user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `vm_instance`.`id`)
+        AND (`custom_ram_size`.`name` = 'memory'))));

engine/schema/src/test/java/com/cloud/host/HostVOTest.java

@@ -1,61 +1,84 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package com.cloud.host;
-
-import com.cloud.service.ServiceOfferingVO;
-import com.cloud.vm.VirtualMachine;
-import java.util.Arrays;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import org.junit.Test;
-import org.junit.Before;
-
-public class HostVOTest {
-    HostVO host;
-    ServiceOfferingVO offering;
-
-    @Before
-    public void setUp() throws Exception {
-        host = new HostVO();
-        offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0,
-                false, "TestSO", false,VirtualMachine.Type.User,false);
-    }
-
-    @Test
-    public void testNoSO() {
-        assertFalse(host.checkHostServiceOfferingTags(null));
-    }
-
-    @Test
-    public void testNoTag() {
-        assertTrue(host.checkHostServiceOfferingTags(offering));
-    }
-
-    @Test
-    public void testRightTag() {
-        host.setHostTags(Arrays.asList("tag1","tag2"));
-        offering.setHostTag("tag2,tag1");
-        assertTrue(host.checkHostServiceOfferingTags(offering));
-    }
-
-    @Test
-    public void testWrongTag() {
-        host.setHostTags(Arrays.asList("tag1","tag2"));
-        offering.setHostTag("tag2,tag4");
-        assertFalse(host.checkHostServiceOfferingTags(offering));
-    }
-}
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.host;
+
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.vm.VirtualMachine;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.Test;
+import org.junit.Before;
+
+public class HostVOTest {
+    HostVO host;
+    ServiceOfferingVO offering;
+
+    @Before
+    public void setUp() throws Exception {
+        host = new HostVO();
+        offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0,
+                false, "TestSO", false,VirtualMachine.Type.User,false);
+    }
+
+    @Test
+    public void testNoSO() {
+        assertFalse(host.checkHostServiceOfferingTags(null));
+    }
+
+    @Test
+    public void testNoTag() {
+        assertTrue(host.checkHostServiceOfferingTags(offering));
+    }
+
+    @Test
+    public void testRightTag() {
+        host.setHostTags(Arrays.asList("tag1","tag2"), false);
+        offering.setHostTag("tag2,tag1");
+        assertTrue(host.checkHostServiceOfferingTags(offering));
+    }
+
+    @Test
+    public void testWrongTag() {
+        host.setHostTags(Arrays.asList("tag1","tag2"), false);
+        offering.setHostTag("tag2,tag4");
+        assertFalse(host.checkHostServiceOfferingTags(offering));
+    }
+
+    @Test
+    public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatMatches() {
+        host.setHostTags(List.of("tags[0] == 'A'"), true);
+        offering.setHostTag("A");
+        assertTrue(host.checkHostServiceOfferingTags(offering));
+    }
+
+    @Test
+    public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatDoesNotMatch() {
+        host.setHostTags(List.of("tags[0] == 'A'"), true);
+        offering.setHostTag("B");
+        assertFalse(host.checkHostServiceOfferingTags(offering));
+    }
+
+    @Test
+    public void checkHostServiceOfferingTagsTestRuleTagWithNullServiceTag() {
+        host.setHostTags(List.of("tags[0] == 'A'"), true);
+        offering.setHostTag(null);
+        assertFalse(host.checkHostServiceOfferingTags(offering));
+    }
+}

engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java

@@ -17,6 +17,7 @@
 package com.cloud.network.as;
 
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -42,14 +43,14 @@ public class AutoScaleVmProfileVOTest {
     public void testCounterParamsForUpdate() {
         AutoScaleVmProfileVO profile = new AutoScaleVmProfileVO();
 
-        Map<String, HashMap<String, String>> counterParamList = new HashMap<>();
-        counterParamList.put("0", new HashMap<>() {{ put("name", "snmpcommunity"); put("value", "public"); }});
-        counterParamList.put("1", new HashMap<>() {{ put("name", "snmpport"); put("value", "161"); }});
+        Map<String, LinkedHashMap<String, String>> counterParamList = new LinkedHashMap<>();
+        counterParamList.put("0", new LinkedHashMap<>() {{ put("name", "snmpcommunity"); put("value", "public"); }});
+        counterParamList.put("1", new LinkedHashMap<>() {{ put("name", "snmpport"); put("value", "161"); }});
 
         profile.setCounterParamsForUpdate(counterParamList);
         Assert.assertEquals("snmpcommunity=public&snmpport=161", profile.getCounterParamsString());
+        List<Pair<String, String>> counterParams = profile.getCounterParams();
 
-        List<Pair<String, String>>  counterParams = profile.getCounterParams();
         Assert.assertEquals(2, counterParams.size());
         Assert.assertEquals("snmpcommunity", counterParams.get(0).first());
         Assert.assertEquals("public", counterParams.get(0).second());
@@ -69,10 +70,17 @@ public class AutoScaleVmProfileVOTest {
 
         List<Pair<String, String>> otherDeployParamsList = profile.getOtherDeployParamsList();
         Assert.assertEquals(2, otherDeployParamsList.size());
-        Assert.assertEquals("serviceofferingid", otherDeployParamsList.get(0).first());
-        Assert.assertEquals("a7fb50f6-01d9-11ed-8bc1-77f8f0228926", otherDeployParamsList.get(0).second());
-        Assert.assertEquals("rootdisksize", otherDeployParamsList.get(1).first());
-        Assert.assertEquals("10", otherDeployParamsList.get(1).second());
+        Assert.assertTrue(containsPair(otherDeployParamsList, "serviceofferingid", "a7fb50f6-01d9-11ed-8bc1-77f8f0228926"));
+        Assert.assertTrue(containsPair(otherDeployParamsList, "rootdisksize", "10"));
+    }
+
+    private boolean containsPair(List<Pair<String, String>> list, String key, String value) {
+        for (Pair<String, String> pair : list) {
+            if (key.equals(pair.first()) && value.equals(pair.second())) {
+                return true;
+            }
+        }
+        return false;
     }
 
     @Test

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java

@@ -20,11 +20,10 @@ package org.apache.cloudstack.storage.image;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import javax.inject.Inject;
 
-import com.cloud.hypervisor.Hypervisor;
-import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.cloudstack.direct.download.DirectDownloadManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@@ -36,18 +35,21 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.VMTemplateStoragePoolVO;
 import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.utils.exception.CloudRuntimeException;
 
 @Component
 public class TemplateDataFactoryImpl implements TemplateDataFactory {
@@ -203,12 +205,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
      * Given existing spool refs, return one pool id existing on pools and refs
      */
     private Long getOneMatchingPoolIdFromRefs(List<VMTemplateStoragePoolVO> existingRefs, List<StoragePoolVO> pools) {
-        if (pools.isEmpty()) {
-            throw new CloudRuntimeException("No storage pools found");
-        }
-        if (existingRefs.isEmpty()) {
-            return pools.get(0).getId();
-        } else {
+        if (!existingRefs.isEmpty()) {
             for (VMTemplateStoragePoolVO ref : existingRefs) {
                 for (StoragePoolVO p : pools) {
                     if (ref.getPoolId() == p.getId()) {
@@ -217,45 +214,51 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
                 }
             }
         }
-        return null;
+        return pools.get(0).getId();
     }
 
     /**
-     * Retrieve storage pools with scope = cluster or zone matching clusterId or dataCenterId depending on their scope
+     * Retrieve storage pools with scope = cluster or zone or local matching clusterId or dataCenterId or hostId depending on their scope
      */
-    private List<StoragePoolVO> getStoragePoolsFromClusterOrZone(Long clusterId, long dataCenterId, Hypervisor.HypervisorType hypervisorType) {
+    private List<StoragePoolVO> getStoragePoolsForScope(long dataCenterId, Long clusterId, long hostId, Hypervisor.HypervisorType hypervisorType) {
         List<StoragePoolVO> pools = new ArrayList<>();
         if (clusterId != null) {
             List<StoragePoolVO> clusterPools = primaryDataStoreDao.listPoolsByCluster(clusterId);
+            clusterPools = clusterPools.stream().filter(p -> !p.isLocal()).collect(Collectors.toList());
             pools.addAll(clusterPools);
         }
         List<StoragePoolVO> zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisor(dataCenterId, hypervisorType);
         pools.addAll(zonePools);
+        List<StoragePoolVO> localPools = primaryDataStoreDao.findLocalStoragePoolsByHostAndTags(hostId, null);
+        pools.addAll(localPools);
         return pools;
     }
 
+    protected Long getBypassedTemplateExistingOrNewPoolId(VMTemplateVO templateVO, Long hostId) {
+        HostVO host = hostDao.findById(hostId);
+        List<StoragePoolVO> pools = getStoragePoolsForScope(host.getDataCenterId(), host.getClusterId(), hostId, host.getHypervisorType());
+        if (CollectionUtils.isEmpty(pools)) {
+            throw new CloudRuntimeException(String.format("No storage pool found to download template: %s", templateVO.getName()));
+        }
+        List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateVO.getId());
+        return getOneMatchingPoolIdFromRefs(existingRefs, pools);
+    }
+
     @Override
     public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId) {
         VMTemplateVO templateVO = imageDataDao.findById(templateId);
         if (templateVO == null || !templateVO.isDirectDownload()) {
             return null;
         }
-        Long pool = poolId;
+        Long templatePoolId = poolId;
         if (poolId == null) {
-            //Get ISO from existing pool ref
-            HostVO host = hostDao.findById(hostId);
-            List<StoragePoolVO> pools = getStoragePoolsFromClusterOrZone(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType());
-            List<VMTemplateStoragePoolVO> existingRefs = templatePoolDao.listByTemplateId(templateId);
-            pool = getOneMatchingPoolIdFromRefs(existingRefs, pools);
+            templatePoolId = getBypassedTemplateExistingOrNewPoolId(templateVO, hostId);
         }
-        if (pool == null) {
-            throw new CloudRuntimeException("No storage pool found where to download template: " + templateId);
-        }
-        VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId, null);
+        VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(templatePoolId, templateId, null);
         if (spoolRef == null) {
-            directDownloadManager.downloadTemplate(templateId, pool, hostId);
+            directDownloadManager.downloadTemplate(templateId, templatePoolId, hostId);
         }
-        DataStore store = storeMgr.getDataStore(pool, DataStoreRole.Primary);
+        DataStore store = storeMgr.getDataStore(templatePoolId, DataStoreRole.Primary);
         return this.getTemplate(templateId, store);
     }
 

engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java

@@ -28,6 +28,7 @@ import java.util.Map;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.api.query.dao.StoragePoolJoinDao;
 import com.cloud.exception.StorageUnavailableException;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.StoragePoolStatus;
@@ -85,6 +86,9 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement
      */
     private SecureRandom secureRandom = new SecureRandom();
 
+    @Inject
+    protected StoragePoolJoinDao storagePoolJoinDao;
+
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
         super.configure(name, params);

engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java

@@ -24,6 +24,7 @@ import java.util.Map;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.storage.VolumeApiServiceImpl;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
@@ -78,11 +79,12 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat
             logDisabledStoragePools(dcId, podId, clusterId, ScopeType.CLUSTER);
         }
 
-        List<StoragePoolVO> pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags());
+        List<StoragePoolVO> pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags(), true, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value());
+        pools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(dcId, podId, clusterId, ScopeType.CLUSTER, List.of(dskCh.getTags())));
         s_logger.debug(String.format("Found pools [%s] that match with tags [%s].", pools, Arrays.toString(dskCh.getTags())));
 
         // add remaining pools in cluster, that did not match tags, to avoid set
-        List<StoragePoolVO> allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null);
+        List<StoragePoolVO> allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null, false, 0);
         allPools.removeAll(pools);
         for (StoragePoolVO pool : allPools) {
             s_logger.trace(String.format("Adding pool [%s] to the 'avoid' set since it did not match any tags.", pool));

engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java

@@ -101,7 +101,8 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator {
                 return null;
             }
             List<StoragePoolVO> availablePools =
-                storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags());
+                storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags(), true);
+            availablePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), ScopeType.HOST, List.of(dskCh.getTags())));
             for (StoragePoolVO pool : availablePools) {
                 if (suitablePools.size() == returnUpTo) {
                     break;
@@ -117,7 +118,7 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator {
             }
 
             // add remaining pools in cluster to the 'avoid' set which did not match tags
-            List<StoragePoolVO> allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null);
+            List<StoragePoolVO> allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null, false);
             allPools.removeAll(availablePools);
             for (StoragePoolVO pool : allPools) {
                 avoid.addPool(pool.getId());

engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java

@@ -63,9 +63,9 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator {
         }
 
         List<StoragePool> suitablePools = new ArrayList<>();
-
-        List<StoragePoolVO> storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags());
-        if (storagePools == null) {
+        List<StoragePoolVO> storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags(), true);
+        storagePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), null, null, ScopeType.ZONE, List.of(dskCh.getTags())));
+        if (storagePools.isEmpty()) {
             LOGGER.debug(String.format("Could not find any zone wide storage pool that matched with any of the following tags [%s].", Arrays.toString(dskCh.getTags())));
             storagePools = new ArrayList<>();
         }
@@ -82,7 +82,7 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator {
         storagePools.addAll(anyHypervisorStoragePools);
 
         // add remaining pools in zone, that did not match tags, to avoid set
-        List<StoragePoolVO> allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null);
+        List<StoragePoolVO> allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null, false);
         allPools.removeAll(storagePools);
         for (StoragePoolVO pool : allPools) {
             avoid.addPool(pool.getId());

engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java

@@ -136,7 +136,7 @@ public class PrimaryDataStoreHelper {
                 storageTags.add(tag);
             }
         }
-        dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags);
+        dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags, params.isTagARule());
         return dataStoreMgr.getDataStore(dataStoreVO.getId(), DataStoreRole.Primary);
     }
 

framework/db/src/main/java/com/cloud/utils/db/Filter.java

@@ -51,6 +51,10 @@ public class Filter {
         addOrderBy(clazz, field, ascending);
     }
 
+    public Filter(Class<?> clazz, String field, boolean ascending) {
+        this(clazz, field, ascending, null, null);
+    }
+
     public Filter(long limit) {
         _orderBy = " ORDER BY RAND() LIMIT " + limit;
     }

framework/db/src/main/java/com/cloud/utils/db/GenericDao.java

@@ -258,6 +258,8 @@ public interface GenericDao<T, ID extends Serializable> {
 
     public T findOneBy(final SearchCriteria<T> sc);
 
+    T findOneBy(SearchCriteria<T> sc, Filter filter);
+
     /**
      * @return
      */

framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java

@@ -422,7 +422,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             return result;
         } catch (final SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + pstmt, e);
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             throw new CloudRuntimeException("Caught: " + pstmt, e);
         }
     }
@@ -499,7 +499,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             return results;
         } catch (final SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + pstmt, e);
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             throw new CloudRuntimeException("Caught: " + pstmt, e);
         }
     }
@@ -907,6 +907,15 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
         return findOneIncludingRemovedBy(sc);
     }
 
+    @Override
+    @DB()
+    public T findOneBy(SearchCriteria<T> sc, final Filter filter) {
+        sc = checkAndSetRemovedIsNull(sc);
+        filter.setLimit(1L);
+        List<T> results = searchIncludingRemoved(sc, filter, null, false);
+        return results.isEmpty() ? null : results.get(0);
+    }
+
     @DB()
     protected List<T> listBy(SearchCriteria<T> sc, final Filter filter) {
         sc = checkAndSetRemovedIsNull(sc);
@@ -1145,7 +1154,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             return result;
         } catch (final SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + pstmt, e);
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             throw new CloudRuntimeException("Caught: " + pstmt, e);
         }
     }
@@ -1227,7 +1236,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             return pstmt.executeUpdate();
         } catch (final SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + pstmt, e);
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             throw new CloudRuntimeException("Caught: " + pstmt, e);
         }
     }
@@ -2050,7 +2059,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             return 0;
         } catch (final SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + pstmt, e);
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             throw new CloudRuntimeException("Caught: " + pstmt, e);
         }
     }
@@ -2101,7 +2110,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             return 0;
         } catch (final SQLException e) {
             throw new CloudRuntimeException("DB Exception in executing: " + sql, e);
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             throw new CloudRuntimeException("Caught exception in : " + sql, e);
         }
     }
@@ -2158,7 +2167,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
             return 0;
         } catch (final SQLException e) {
             throw new CloudRuntimeException("DB Exception on: " + pstmt, e);
-        } catch (final Throwable e) {
+        } catch (final Exception e) {
             throw new CloudRuntimeException("Caught: " + pstmt, e);
         }
     }

framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java

@@ -89,11 +89,11 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager {
 
     private TimeZone _usageTimezone;
     private int _aggregationDuration = 0;
-
-    static final BigDecimal s_hoursInMonth = BigDecimal.valueOf(DateUtil.HOURS_IN_A_MONTH);
     static final BigDecimal GiB_DECIMAL = BigDecimal.valueOf(ByteScaleUtils.GiB);
     List<Account.Type> lockablesAccountTypes = Arrays.asList(Account.Type.NORMAL, Account.Type.DOMAIN_ADMIN);
 
+    static BigDecimal hoursInCurrentMonth;
+
     public QuotaManagerImpl() {
         super();
     }
@@ -455,7 +455,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager {
 
         }
 
-        jsInterpreter.injectVariable("resourceType", presetVariables.getResourceType());
+        jsInterpreter.injectStringVariable("resourceType", presetVariables.getResourceType());
         jsInterpreter.injectVariable("value", presetVariables.getValue().toString());
         jsInterpreter.injectVariable("zone", presetVariables.getZone().toString());
     }
@@ -533,7 +533,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager {
 
     protected BigDecimal getUsageValueAccordingToUsageUnitType(UsageVO usageRecord, BigDecimal aggregatedQuotaTariffsValue, String quotaUnit) {
         BigDecimal rawUsage = BigDecimal.valueOf(usageRecord.getRawUsage());
-        BigDecimal costPerHour = aggregatedQuotaTariffsValue.divide(s_hoursInMonth, 8, RoundingMode.HALF_EVEN);
+        BigDecimal costPerHour = getCostPerHour(aggregatedQuotaTariffsValue, usageRecord.getStartDate());
 
         switch (UsageUnitTypes.getByDescription(quotaUnit)) {
             case COMPUTE_MONTH:
@@ -558,6 +558,12 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager {
         }
     }
 
+    protected BigDecimal getCostPerHour(BigDecimal costPerMonth, Date date) {
+        BigDecimal hoursInCurrentMonth = BigDecimal.valueOf(DateUtil.getHoursInCurrentMonth(date));
+        s_logger.trace(String.format("Dividing tariff cost per month [%s] by [%s] to get the tariffs cost per hour.", costPerMonth, hoursInCurrentMonth));
+        return costPerMonth.divide(hoursInCurrentMonth, 8, RoundingMode.HALF_EVEN);
+    }
+
     @Override
     public boolean isLockable(AccountVO account) {
         return lockablesAccountTypes.contains(account.getType());

framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java

@@ -22,6 +22,8 @@ import java.util.List;
 public class Host extends GenericPresetVariable {
     private List<String> tags;
 
+    private Boolean isTagARule;
+
     public List<String> getTags() {
         return tags;
     }
@@ -31,4 +33,12 @@ public class Host extends GenericPresetVariable {
         fieldNamesToIncludeInToString.add("tags");
     }
 
+    public Boolean getIsTagARule() {
+        return isTagARule;
+    }
+
+    public void setIsTagARule(Boolean isTagARule) {
+        this.isTagARule = isTagARule;
+        fieldNamesToIncludeInToString.add("isTagARule");
+    }
 }

framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java

@@ -25,8 +25,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import com.cloud.host.HostTagVO;
 import javax.inject.Inject;
 
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.storage.StoragePoolTagVO;
 import org.apache.cloudstack.acl.RoleVO;
 import org.apache.cloudstack.acl.dao.RoleDao;
 import org.apache.cloudstack.backup.BackupOfferingVO;
@@ -65,6 +68,7 @@ import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.GuestOSVO;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.VolumeVO;
 import com.cloud.storage.dao.DiskOfferingDao;
@@ -318,6 +322,10 @@ public class PresetVariableHelper {
 
         value.setTags(getPresetVariableValueResourceTags(vmId, ResourceObjectType.UserVm));
         value.setTemplate(getPresetVariableValueTemplate(vmVo.getTemplateId()));
+        Hypervisor.HypervisorType hypervisorType = vmVo.getHypervisorType();
+        if (hypervisorType != null) {
+            value.setHypervisorType(hypervisorType.name());
+        }
     }
 
     protected void logNotLoadingMessageInTrace(String resource, int usageType) {
@@ -345,7 +353,17 @@ public class PresetVariableHelper {
         Host host = new Host();
         host.setId(hostVo.getUuid());
         host.setName(hostVo.getName());
-        host.setTags(hostTagsDao.getHostTags(hostId));
+        List<HostTagVO> hostTagVOList = hostTagsDao.getHostTags(hostId);
+        List<String> hostTags = new ArrayList<>();
+        boolean isTagARule = false;
+        if (CollectionUtils.isNotEmpty(hostTagVOList)) {
+            isTagARule = hostTagVOList.get(0).getIsTagARule();
+            if (!isTagARule) {
+                hostTags = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList());
+            }
+        }
+        host.setTags(hostTags);
+        host.setIsTagARule(isTagARule);
 
         return host;
     }
@@ -470,6 +488,11 @@ public class PresetVariableHelper {
 
         value.setTags(getPresetVariableValueResourceTags(volumeId, ResourceObjectType.Volume));
         value.setSize(ByteScaleUtils.bytesToMebibytes(volumeVo.getSize()));
+
+        ImageFormat format = volumeVo.getFormat();
+        if (format != null) {
+            value.setVolumeFormat(format.name());
+        }
     }
 
     protected GenericPresetVariable getPresetVariableValueDiskOffering(Long diskOfferingId) {
@@ -497,7 +520,17 @@ public class PresetVariableHelper {
         storage.setId(storagePoolVo.getUuid());
         storage.setName(storagePoolVo.getName());
         storage.setScope(storagePoolVo.getScope());
-        storage.setTags(storagePoolTagsDao.getStoragePoolTags(storageId));
+        List<StoragePoolTagVO> storagePoolTagVOList = storagePoolTagsDao.findStoragePoolTags(storageId);
+        List<String> storageTags = new ArrayList<>();
+        boolean isTagARule = false;
+        if (CollectionUtils.isNotEmpty(storagePoolTagVOList)) {
+            isTagARule = storagePoolTagVOList.get(0).isTagARule();
+            if (!isTagARule) {
+                storageTags = storagePoolTagVOList.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList());
+            }
+        }
+        storage.setTags(storageTags);
+        storage.setIsTagARule(isTagARule);
 
         return storage;
     }
@@ -558,6 +591,10 @@ public class PresetVariableHelper {
         value.setSnapshotType(Snapshot.Type.values()[snapshotVo.getSnapshotType()]);
         value.setStorage(getPresetVariableValueStorage(getSnapshotDataStoreId(snapshotId, usageRecord.getZoneId()), usageType));
         value.setTags(getPresetVariableValueResourceTags(snapshotId, ResourceObjectType.Snapshot));
+        Hypervisor.HypervisorType hypervisorType = snapshotVo.getHypervisorType();
+        if (hypervisorType != null) {
+            value.setHypervisorType(hypervisorType.name());
+        }
     }
 
     protected SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) {
@@ -621,6 +658,11 @@ public class PresetVariableHelper {
         value.setName(vmSnapshotVo.getName());
         value.setTags(getPresetVariableValueResourceTags(vmSnapshotId, ResourceObjectType.VMSnapshot));
         value.setVmSnapshotType(vmSnapshotVo.getType());
+
+        VMInstanceVO vmVo = vmInstanceDao.findByIdIncludingRemoved(vmSnapshotVo.getVmId());
+        if (vmVo != null && vmVo.getHypervisorType() != null) {
+            value.setHypervisorType(vmVo.getHypervisorType().name());
+        }
     }
 
     protected void loadPresetVariableValueForBackup(UsageVO usageRecord, Value value) {

framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java

@@ -23,6 +23,8 @@ import com.cloud.storage.ScopeType;
 
 public class Storage extends GenericPresetVariable {
     private List<String> tags;
+
+    private Boolean isTagARule;
     private ScopeType scope;
 
     public List<String> getTags() {
@@ -34,6 +36,15 @@ public class Storage extends GenericPresetVariable {
         fieldNamesToIncludeInToString.add("tags");
     }
 
+    public Boolean getIsTagARule() {
+        return isTagARule;
+    }
+
+    public void setIsTagARule(Boolean isTagARule) {
+        this.isTagARule = isTagARule;
+        fieldNamesToIncludeInToString.add("isTagARule");
+    }
+
     public ScopeType getScope() {
         return scope;
     }

framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java

@@ -41,6 +41,8 @@ public class Value extends GenericPresetVariable {
     private Storage storage;
     private ComputingResources computingResources;
     private BackupOffering backupOffering;
+    private String hypervisorType;
+    private String volumeFormat;
 
     public Host getHost() {
         return host;
@@ -185,4 +187,22 @@ public class Value extends GenericPresetVariable {
         this.backupOffering = backupOffering;
         fieldNamesToIncludeInToString.add("backupOffering");
     }
+
+    public void setHypervisorType(String hypervisorType) {
+        this.hypervisorType = hypervisorType;
+        fieldNamesToIncludeInToString.add("hypervisorType");
+    }
+
+    public String getHypervisorType() {
+        return hypervisorType;
+    }
+
+    public void setVolumeFormat(String volumeFormat) {
+        this.volumeFormat = volumeFormat;
+        fieldNamesToIncludeInToString.add("volumeFormat");
+    }
+
+    public String getVolumeFormat() {
+        return volumeFormat;
+    }
 }

framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java

@@ -142,8 +142,10 @@ public class QuotaManagerImplTest {
     public void getUsageValueAccordingToUsageUnitTypeTestAllTypes() {
         Mockito.doReturn(10.0).when(usageVoMock).getRawUsage();
         Mockito.doReturn(ByteScaleUtils.GiB).when(usageVoMock).getSize();
+        Mockito.doReturn(new Date(0, 8, 10)).when(usageVoMock).getStartDate();
         BigDecimal aggregatedQuotaTariffsValue = new BigDecimal(400);
 
+
         Arrays.asList(UsageUnitTypes.values()).forEach(type -> {
            BigDecimal result = quotaManagerImplSpy.getUsageValueAccordingToUsageUnitType(usageVoMock, aggregatedQuotaTariffsValue, type.toString());
            Double expected = null;
@@ -267,7 +269,7 @@ public class QuotaManagerImplTest {
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("project"), Mockito.anyString());
-        Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString());
+        Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString());
     }
@@ -288,7 +290,7 @@ public class QuotaManagerImplTest {
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("project"), Mockito.anyString());
-        Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString());
+        Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString());
         Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString());
     }

framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java

@@ -27,6 +27,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import com.cloud.host.HostTagVO;
+import com.cloud.hypervisor.Hypervisor;
+import com.cloud.storage.StoragePoolTagVO;
 import org.apache.cloudstack.acl.RoleType;
 import org.apache.cloudstack.acl.RoleVO;
 import org.apache.cloudstack.acl.dao.RoleDao;
@@ -70,6 +73,7 @@ import com.cloud.storage.GuestOSVO;
 import com.cloud.storage.ScopeType;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.ProvisioningType;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.VolumeVO;
@@ -239,6 +243,14 @@ public class PresetVariableHelperTest {
         return host;
     }
 
+    private List<HostTagVO> getHostTagsForTests() {
+        return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false));
+    }
+
+    private List<HostTagVO> getHostRuleTagsForTests() {
+        return List.of(new HostTagVO(1, "tagrule", true));
+    }
+
     private Storage getStorageForTests() {
         Storage storage = new Storage();
         storage.setId("storage_id");
@@ -248,6 +260,14 @@ public class PresetVariableHelperTest {
         return storage;
     }
 
+    private List<StoragePoolTagVO> getStorageTagsForTests() {
+        return Arrays.asList(new StoragePoolTagVO(1, "tag1", false), new StoragePoolTagVO(1, "tag2", false));
+    }
+
+    private List<StoragePoolTagVO> getStorageRuleTagsForTests() {
+        return List.of(new StoragePoolTagVO(1, "tagrule", true));
+    }
+
     private Set<Map.Entry<Integer, QuotaTypes>> getQuotaTypesForTests(Integer... typesToRemove) {
         Map<Integer, QuotaTypes> quotaTypesMap = new LinkedHashMap<>(QuotaTypes.listQuotaTypes());
 
@@ -485,38 +505,42 @@ public class PresetVariableHelperTest {
 
     @Test
     public void loadPresetVariableValueForRunningAndAllocatedVmTestRecordIsRunningOrAllocatedVmSetFields() {
-        Value expected = getValueForTests();
+        for (Hypervisor.HypervisorType hypervisorType : Hypervisor.HypervisorType.values()) {
+            Value expected = getValueForTests();
 
-        Mockito.doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
+            Mockito.doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
 
-        mockMethodValidateIfObjectIsNull();
+            mockMethodValidateIfObjectIsNull();
 
-        Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableHostInValueIfUsageTypeIsRunningVm(Mockito.any(Value.class), Mockito.anyInt(),
-                Mockito.any(VMInstanceVO.class));
+            Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableHostInValueIfUsageTypeIsRunningVm(Mockito.any(Value.class), Mockito.anyInt(),
+                    Mockito.any(VMInstanceVO.class));
 
-        Mockito.doReturn(expected.getId()).when(vmInstanceVoMock).getUuid();
-        Mockito.doReturn(expected.getName()).when(vmInstanceVoMock).getHostName();
-        Mockito.doReturn(expected.getOsName()).when(presetVariableHelperSpy).getPresetVariableValueOsName(Mockito.anyLong());
-        Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableValueServiceOfferingAndComputingResources(Mockito.any(), Mockito.anyInt(), Mockito.any());
-        Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
-        Mockito.doReturn(expected.getTemplate()).when(presetVariableHelperSpy).getPresetVariableValueTemplate(Mockito.anyLong());
+            Mockito.doReturn(expected.getId()).when(vmInstanceVoMock).getUuid();
+            Mockito.doReturn(expected.getName()).when(vmInstanceVoMock).getHostName();
+            Mockito.doReturn(expected.getOsName()).when(presetVariableHelperSpy).getPresetVariableValueOsName(Mockito.anyLong());
+            Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableValueServiceOfferingAndComputingResources(Mockito.any(), Mockito.anyInt(), Mockito.any());
+            Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
+            Mockito.doReturn(expected.getTemplate()).when(presetVariableHelperSpy).getPresetVariableValueTemplate(Mockito.anyLong());
+            Mockito.doReturn(hypervisorType).when(vmInstanceVoMock).getHypervisorType();
 
-        runningAndAllocatedVmUsageTypes.forEach(type -> {
-            Mockito.doReturn(type).when(usageVoMock).getUsageType();
+            runningAndAllocatedVmUsageTypes.forEach(type -> {
+                Mockito.doReturn(type).when(usageVoMock).getUsageType();
 
-            Value result = new Value();
-            presetVariableHelperSpy.loadPresetVariableValueForRunningAndAllocatedVm(usageVoMock, result);
+                Value result = new Value();
+                presetVariableHelperSpy.loadPresetVariableValueForRunningAndAllocatedVm(usageVoMock, result);
 
-            assertPresetVariableIdAndName(expected, result);
-            Assert.assertEquals(expected.getOsName(), result.getOsName());
-            Assert.assertEquals(expected.getTags(), result.getTags());
-            Assert.assertEquals(expected.getTemplate(), result.getTemplate());
+                assertPresetVariableIdAndName(expected, result);
+                Assert.assertEquals(expected.getOsName(), result.getOsName());
+                Assert.assertEquals(expected.getTags(), result.getTags());
+                Assert.assertEquals(expected.getTemplate(), result.getTemplate());
+                Assert.assertEquals(hypervisorType.name(), result.getHypervisorType());
 
-            validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "osName", "tags", "template"), result);
-        });
+                validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "osName", "tags", "template", "hypervisorType"), result);
+            });
+        }
 
-        Mockito.verify(presetVariableHelperSpy, Mockito.times(runningAndAllocatedVmUsageTypes.size())).getPresetVariableValueResourceTags(Mockito.anyLong(),
-                Mockito.eq(ResourceObjectType.UserVm));
+        Mockito.verify(presetVariableHelperSpy, Mockito.times(runningAndAllocatedVmUsageTypes.size() * Hypervisor.HypervisorType.values().length))
+                .getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.UserVm));
     }
 
     @Test
@@ -533,15 +557,16 @@ public class PresetVariableHelperTest {
     @Test
     public void setPresetVariableHostInValueIfUsageTypeIsRunningVmTestQuotaTypeIsRunningVmSetHost() {
         Value result = new Value();
-        Host expected = getHostForTests();
+        Host expectedHost = getHostForTests();
+        List<HostTagVO> expectedHostTags = getHostTagsForTests();
 
-        Mockito.doReturn(expected).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong());
+        Mockito.doReturn(expectedHost).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong());
         presetVariableHelperSpy.setPresetVariableHostInValueIfUsageTypeIsRunningVm(result, UsageTypes.RUNNING_VM, vmInstanceVoMock);
 
         Assert.assertNotNull(result.getHost());
 
-        assertPresetVariableIdAndName(expected, result.getHost());
-        Assert.assertEquals(expected.getTags(), result.getHost().getTags());
+        assertPresetVariableIdAndName(expectedHost, result.getHost());
+        Assert.assertEquals(expectedHost.getTags(), result.getHost().getTags());
         validateFieldNamesToIncludeInToString(Arrays.asList("host"), result);
     }
 
@@ -549,18 +574,39 @@ public class PresetVariableHelperTest {
     public void getPresetVariableValueHostTestSetFieldsAndReturnObject() {
         Host expected = getHostForTests();
         HostVO hostVoMock = Mockito.mock(HostVO.class);
+        List<HostTagVO> hostTagVOListMock = getHostTagsForTests();
 
         Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
         mockMethodValidateIfObjectIsNull();
         Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid();
         Mockito.doReturn(expected.getName()).when(hostVoMock).getName();
-        Mockito.doReturn(expected.getTags()).when(hostTagsDaoMock).getHostTags(Mockito.anyLong());
+        Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong());
 
         Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l);
 
         assertPresetVariableIdAndName(expected, result);
         Assert.assertEquals(expected.getTags(), result.getTags());
-        validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "tags"), result);
+        validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result);
+    }
+
+    @Test
+    public void getPresetVariableValueHostTestSetFieldsWithRuleTagAndReturnObject() {
+        Host expected = getHostForTests();
+        HostVO hostVoMock = Mockito.mock(HostVO.class);
+        List<HostTagVO> hostTagVOListMock = getHostRuleTagsForTests();
+
+        Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
+        mockMethodValidateIfObjectIsNull();
+        Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid();
+        Mockito.doReturn(expected.getName()).when(hostVoMock).getName();
+        Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong());
+
+        Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l);
+
+        assertPresetVariableIdAndName(expected, result);
+        Assert.assertEquals(new ArrayList<>(), result.getTags());
+        Assert.assertTrue(result.getIsTagARule());
+        validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result);
     }
 
     @Test
@@ -636,75 +682,85 @@ public class PresetVariableHelperTest {
 
     @Test
     public void loadPresetVariableValueForVolumeTestRecordIsVolumeAndHasStorageSetFields() {
-        Value expected = getValueForTests();
+        for (ImageFormat imageFormat : ImageFormat.values()) {
+            Value expected = getValueForTests();
 
-        VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class);
-        Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
-        Mockito.doReturn(1l).when(volumeVoMock).getPoolId();
+            VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class);
+            Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
+            Mockito.doReturn(1l).when(volumeVoMock).getPoolId();
 
-        mockMethodValidateIfObjectIsNull();
+            mockMethodValidateIfObjectIsNull();
 
-        Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid();
-        Mockito.doReturn(expected.getName()).when(volumeVoMock).getName();
-        Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong());
-        Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType();
-        Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt());
-        Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
-        Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize();
+            Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid();
+            Mockito.doReturn(expected.getName()).when(volumeVoMock).getName();
+            Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong());
+            Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType();
+            Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt());
+            Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
+            Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize();
+            Mockito.doReturn(imageFormat).when(volumeVoMock).getFormat();
 
-        Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType();
+            Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType();
 
-        Value result = new Value();
-        presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result);
+            Value result = new Value();
+            presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result);
 
-        Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize());
+            Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize());
 
-        assertPresetVariableIdAndName(expected, result);
-        Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering());
-        Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType());
-        Assert.assertEquals(expected.getStorage(), result.getStorage());
-        Assert.assertEquals(expected.getTags(), result.getTags());
-        Assert.assertEquals(expectedSize, result.getSize());
+            assertPresetVariableIdAndName(expected, result);
+            Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering());
+            Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType());
+            Assert.assertEquals(expected.getStorage(), result.getStorage());
+            Assert.assertEquals(expected.getTags(), result.getTags());
+            Assert.assertEquals(expectedSize, result.getSize());
+            Assert.assertEquals(imageFormat.name(), result.getVolumeFormat());
 
-        validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "storage", "tags", "size"), result);
+            validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "storage", "tags", "size", "volumeFormat"), result);
+        }
 
-        Mockito.verify(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.Volume));
+        Mockito.verify(presetVariableHelperSpy, Mockito.times(ImageFormat.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(),
+                Mockito.eq(ResourceObjectType.Volume));
     }
 
     @Test
     public void loadPresetVariableValueForVolumeTestRecordIsVolumeAndDoesNotHaveStorageSetFields() {
-        Value expected = getValueForTests();
+        for (ImageFormat imageFormat : ImageFormat.values()) {
+            Value expected = getValueForTests();
 
-        VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class);
-        Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
-        Mockito.doReturn(null).when(volumeVoMock).getPoolId();
+            VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class);
+            Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
+            Mockito.doReturn(null).when(volumeVoMock).getPoolId();
 
-        mockMethodValidateIfObjectIsNull();
+            mockMethodValidateIfObjectIsNull();
 
-        Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid();
-        Mockito.doReturn(expected.getName()).when(volumeVoMock).getName();
-        Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong());
-        Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType();
-        Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
-        Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize();
+            Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid();
+            Mockito.doReturn(expected.getName()).when(volumeVoMock).getName();
+            Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong());
+            Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType();
+            Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
+            Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize();
+            Mockito.doReturn(imageFormat).when(volumeVoMock).getFormat();
 
-        Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType();
+            Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType();
 
-        Value result = new Value();
-        presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result);
+            Value result = new Value();
+            presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result);
 
-        Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize());
+            Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize());
 
-        assertPresetVariableIdAndName(expected, result);
-        Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering());
-        Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType());
-        Assert.assertEquals(null, result.getStorage());
-        Assert.assertEquals(expected.getTags(), result.getTags());
-        Assert.assertEquals(expectedSize, result.getSize());
+            assertPresetVariableIdAndName(expected, result);
+            Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering());
+            Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType());
+            Assert.assertNull(result.getStorage());
+            Assert.assertEquals(expected.getTags(), result.getTags());
+            Assert.assertEquals(expectedSize, result.getSize());
+            Assert.assertEquals(imageFormat.name(), result.getVolumeFormat());
 
-        validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "tags", "size"), result);
+            validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "tags", "size", "volumeFormat"), result);
+        }
 
-        Mockito.verify(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.Volume));
+        Mockito.verify(presetVariableHelperSpy, Mockito.times(ImageFormat.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(),
+                Mockito.eq(ResourceObjectType.Volume));
     }
 
     @Test
@@ -739,13 +795,15 @@ public class PresetVariableHelperTest {
         Storage expected = getStorageForTests();
         Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt());
 
+        List<StoragePoolTagVO> storageTagVOListMock = getStorageTagsForTests();
+
         StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class);
         Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
 
         Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid();
         Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName();
         Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope();
-        Mockito.doReturn(expected.getTags()).when(storagePoolTagsDaoMock).getStoragePoolTags(Mockito.anyLong());
+        Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong());
 
         Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2);
 
@@ -753,7 +811,32 @@ public class PresetVariableHelperTest {
         Assert.assertEquals(expected.getScope(), result.getScope());
         Assert.assertEquals(expected.getTags(), result.getTags());
 
-        validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "scope", "tags"), result);
+        validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule",  "name", "scope", "tags"), result);
+    }
+
+    @Test
+    public void getPresetVariableValueStorageTestGetSecondaryStorageForSnapshotReturnsNullWithRuleTag() {
+        Storage expected = getStorageForTests();
+        Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt());
+
+        List<StoragePoolTagVO> storageTagVOListMock = getStorageRuleTagsForTests();
+
+        StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class);
+        Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
+
+        Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid();
+        Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName();
+        Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope();
+        Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong());
+
+        Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2);
+
+        assertPresetVariableIdAndName(expected, result);
+        Assert.assertEquals(expected.getScope(), result.getScope());
+        Assert.assertEquals(new ArrayList<>(), result.getTags());
+        Assert.assertTrue(result.getIsTagARule());
+
+        validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule",  "name", "scope", "tags"), result);
     }
 
     @Test
@@ -852,37 +935,42 @@ public class PresetVariableHelperTest {
 
     @Test
     public void loadPresetVariableValueForSnapshotTestRecordIsSnapshotSetFields() {
-        Value expected = getValueForTests();
+        for (Hypervisor.HypervisorType hypervisorType : Hypervisor.HypervisorType.values()) {
+            Value expected = getValueForTests();
 
-        SnapshotVO snapshotVoMock = Mockito.mock(SnapshotVO.class);
-        Mockito.doReturn(snapshotVoMock).when(snapshotDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
+            SnapshotVO snapshotVoMock = Mockito.mock(SnapshotVO.class);
+            Mockito.doReturn(snapshotVoMock).when(snapshotDaoMock).findByIdIncludingRemoved(Mockito.anyLong());
 
-        mockMethodValidateIfObjectIsNull();
+            mockMethodValidateIfObjectIsNull();
 
-        Mockito.doReturn(expected.getId()).when(snapshotVoMock).getUuid();
-        Mockito.doReturn(expected.getName()).when(snapshotVoMock).getName();
-        Mockito.doReturn(expected.getSize()).when(snapshotVoMock).getSize();
-        Mockito.doReturn((short) 3).when(snapshotVoMock).getSnapshotType();
-        Mockito.doReturn(1l).when(presetVariableHelperSpy).getSnapshotDataStoreId(Mockito.anyLong(), Mockito.anyLong());
-        Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt());
-        Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
+            Mockito.doReturn(expected.getId()).when(snapshotVoMock).getUuid();
+            Mockito.doReturn(expected.getName()).when(snapshotVoMock).getName();
+            Mockito.doReturn(expected.getSize()).when(snapshotVoMock).getSize();
+            Mockito.doReturn((short) 3).when(snapshotVoMock).getSnapshotType();
+            Mockito.doReturn(1l).when(presetVariableHelperSpy).getSnapshotDataStoreId(Mockito.anyLong(), Mockito.anyLong());
+            Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt());
+            Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class));
+            Mockito.doReturn(hypervisorType).when(snapshotVoMock).getHypervisorType();
 
-        Mockito.doReturn(UsageTypes.SNAPSHOT).when(usageVoMock).getUsageType();
+            Mockito.doReturn(UsageTypes.SNAPSHOT).when(usageVoMock).getUsageType();
 
-        Value result = new Value();
-        presetVariableHelperSpy.loadPresetVariableValueForSnapshot(usageVoMock, result);
+            Value result = new Value();
+            presetVariableHelperSpy.loadPresetVariableValueForSnapshot(usageVoMock, result);
 
-        Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize());
+            Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize());
 
-        assertPresetVariableIdAndName(expected, result);
-        Assert.assertEquals(expected.getSnapshotType(), result.getSnapshotType());
-        Assert.assertEquals(expected.getStorage(), result.getStorage());
-        Assert.assertEquals(expected.getTags(), result.getTags());
-        Assert.assertEquals(expectedSize, result.getSize());
+            assertPresetVariableIdAndName(expected, result);
+            Assert.assertEquals(expected.getSnapshotType(), result.getSnapshotType());
+            Assert.assertEquals(expected.getStorage(), result.getStorage());
+            Assert.assertEquals(expected.getTags(), result.getTags());
+            Assert.assertEquals(expectedSize, result.getSize());
+            Assert.assertEquals(hypervisorType.name(), result.getHypervisorType());
 
-        validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "snapshotType", "storage", "tags", "size"), result);
+            validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "snapshotType", "storage", "tags", "size", "hypervisorType"), result);
+        }
 
-        Mockito.verify(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.Snapshot));
+        Mockito.verify(presetVariableHelperSpy, Mockito.times(Hypervisor.HypervisorType.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(),
+                Mockito.eq(ResourceObjectType.Snapshot));
     }
 
 

framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java

@@ -136,4 +136,18 @@ public class ValueTest {
         variable.setBackupOffering(null);
         Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("backupOffering"));
     }
+
+    @Test
+    public void setHypervisorTypeTestAddFieldHypervisorTypeToCollection() {
+        Value variable = new Value();
+        variable.setHypervisorType(null);
+        Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("hypervisorType"));
+    }
+
+    @Test
+    public void setVolumeFormatTestAddFieldVolumeFormatToCollection() {
+        Value variable = new Value();
+        variable.setVolumeFormat(null);
+        Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("volumeFormat"));
+    }
 }

plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java

@@ -22,7 +22,9 @@ import java.util.List;
 
 import javax.inject.Inject;
 
+import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.ListUtils;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -72,7 +74,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator {
         }
         String hostTag = offering.getHostTag();
         if (hostTag != null) {
-            s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId + " having host tag:" + hostTag);
+            s_logger.debug(String.format("Looking for hosts in dc [%s], pod [%s], cluster [%s] and complying with host tag [%s].", dcId, podId, clusterId, hostTag));
         } else {
             s_logger.debug("Looking for hosts in dc: " + dcId + "  pod:" + podId + "  cluster:" + clusterId);
         }
@@ -82,7 +84,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator {
             if (hostTag != null) {
                 hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag));
             } else {
-                hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId));
+                hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId));
             }
         } else {
             // list all computing hosts, regardless of whether they support routing...it's random after all
@@ -90,9 +92,16 @@ public class RandomAllocator extends AdapterBase implements HostAllocator {
             if (hostTag != null) {
                 hostsCopy = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag);
             } else {
-                hostsCopy = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId);
+                hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId);
             }
         }
+        hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTag));
+
+        if (hostsCopy.isEmpty()) {
+            s_logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTag));
+            throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile));
+        }
+
         s_logger.debug("Random Allocator found " + hostsCopy.size() + "  hosts");
         if (hostsCopy.size() == 0) {
             return suitableHosts;

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java

@@ -28,15 +28,17 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePool;
 import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.OutputInterpreter.AllLinesParser;
 import com.cloud.utils.script.Script;
+import com.cloud.agent.properties.AgentProperties;
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
 
 public class KVMHABase {
     private static final Logger s_logger = Logger.getLogger(KVMHABase.class);
     private long _timeout = 60000; /* 1 minutes */
     protected static String s_heartBeatPath;
-    protected long _heartBeatUpdateTimeout = 60000;
-    protected long _heartBeatUpdateFreq = 60000;
-    protected long _heartBeatUpdateMaxTries = 5;
-    protected long _heartBeatUpdateRetrySleep = 10000;
+    protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT);
+    protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY);
+    protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES);
+    protected long _heartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
 
     public static enum PoolType {
         PrimaryStorage, SecondaryStorage

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java

@@ -48,7 +48,6 @@ public class KVMHAMonitor extends KVMHABase implements Runnable {
         hostPrivateIp = host;
         configureHeartBeatPath(scriptPath);
 
-        _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT);
         rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT);
     }
 

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java

@@ -19,6 +19,8 @@ package com.cloud.hypervisor.kvm.storage;
 import java.util.List;
 import java.util.Map;
 
+import com.cloud.agent.properties.AgentProperties;
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
 import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat;
 import org.joda.time.Duration;
 
@@ -29,11 +31,11 @@ import com.cloud.storage.Storage.StoragePoolType;
 
 public interface KVMStoragePool {
 
-    public static final long HeartBeatUpdateTimeout = 60000;
-    public static final long HeartBeatUpdateFreq = 60000;
-    public static final long HeartBeatUpdateMaxTries = 5;
-    public static final long HeartBeatUpdateRetrySleep = 10000;
-    public static final long HeartBeatCheckerTimeout = 360000; // 6 minutes
+    public static final long HeartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT);
+    public static final long HeartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY);
+    public static final long HeartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES);
+    public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP);
+    public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT);
 
 
     public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase);

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java

@@ -294,12 +294,14 @@ public class KVMStoragePoolManager {
         String uuid = null;
         String sourceHost = "";
         StoragePoolType protocol = null;
-        if (storageUri.getScheme().equalsIgnoreCase("nfs") || storageUri.getScheme().equalsIgnoreCase("NetworkFilesystem")) {
+        final String scheme = storageUri.getScheme().toLowerCase();
+        List<String> acceptedSchemes = List.of("nfs", "networkfilesystem", "filesystem");
+        if (acceptedSchemes.contains(scheme)) {
             sourcePath = storageUri.getPath();
             sourcePath = sourcePath.replace("//", "/");
             sourceHost = storageUri.getHost();
             uuid = UUID.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString();
-            protocol = StoragePoolType.NetworkFilesystem;
+            protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem;
         }
 
         // secondary storage registers itself through here

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java

@@ -25,23 +25,26 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 import javax.naming.ConfigurationException;
 
-import org.apache.cloudstack.direct.download.DirectDownloadHelper;
-import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
-import com.cloud.storage.ScopeType;
-import com.cloud.storage.Volume;
 import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer;
 import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.direct.download.DirectDownloadHelper;
+import org.apache.cloudstack.direct.download.DirectTemplateDownloader;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
 import org.apache.cloudstack.storage.command.AttachAnswer;
 import org.apache.cloudstack.storage.command.AttachCommand;
@@ -71,7 +74,10 @@ import org.apache.cloudstack.utils.qemu.QemuImgFile;
 import org.apache.cloudstack.utils.qemu.QemuObject;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.FileUtils;
-
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 import org.apache.log4j.Logger;
 import org.libvirt.Connect;
 import org.libvirt.Domain;
@@ -110,9 +116,11 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef.DiskProtocol;
 import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper;
 import com.cloud.storage.JavaStorageLayer;
 import com.cloud.storage.MigrationOptions;
+import com.cloud.storage.ScopeType;
 import com.cloud.storage.Storage.ImageFormat;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageLayer;
+import com.cloud.storage.Volume;
 import com.cloud.storage.resource.StorageProcessor;
 import com.cloud.storage.template.Processor;
 import com.cloud.storage.template.Processor.FormatInfo;
@@ -126,16 +134,6 @@ import com.cloud.utils.script.Script;
 import com.cloud.utils.storage.S3.S3Utils;
 import com.cloud.vm.VmDetailConstants;
 
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.builder.ToStringBuilder;
-import org.apache.commons.lang3.builder.ToStringStyle;
-
 public class KVMStorageProcessor implements StorageProcessor {
     private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class);
     private final KVMStoragePoolManager storagePoolMgr;
@@ -1074,7 +1072,8 @@ public class KVMStorageProcessor implements StorageProcessor {
             s_logger.debug(String.format("This backup is temporary, not deleting snapshot [%s] on primary storage [%s]", snapshotPath, primaryPool.getUuid()));
         }
     }
-    protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params) throws
+
+    protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params, DataStoreTO store) throws
             LibvirtException, InternalErrorException {
         DiskDef iso = new DiskDef();
         boolean isUefiEnabled = MapUtils.isNotEmpty(params) && params.containsKey("UEFI");
@@ -1082,8 +1081,14 @@ public class KVMStorageProcessor implements StorageProcessor {
             final int index = isoPath.lastIndexOf("/");
             final String path = isoPath.substring(0, index);
             final String name = isoPath.substring(index + 1);
-            final KVMStoragePool secondaryPool = storagePoolMgr.getStoragePoolByURI(path);
-            final KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name);
+            KVMStoragePool storagePool;
+            if (store instanceof PrimaryDataStoreTO) {
+                PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO)store;
+                storagePool = storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), store.getUuid());
+            } else {
+                storagePool = storagePoolMgr.getStoragePoolByURI(path);
+            }
+            final KVMPhysicalDisk isoVol = storagePool.getPhysicalDisk(name);
             isoPath = isoVol.getPath();
 
             iso.defISODisk(isoPath, isUefiEnabled);
@@ -1112,7 +1117,7 @@ public class KVMStorageProcessor implements StorageProcessor {
         try {
             String dataStoreUrl = getDataStoreUrlFromStore(store);
             final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
-            attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo());
+            attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo(), store);
         } catch (final LibvirtException e) {
             return new Answer(cmd, false, e.toString());
         } catch (final InternalErrorException e) {
@@ -1133,7 +1138,7 @@ public class KVMStorageProcessor implements StorageProcessor {
         try {
             String dataStoreUrl = getDataStoreUrlFromStore(store);
             final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName());
-            attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams());
+            attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams(), store);
         } catch (final LibvirtException e) {
             return new Answer(cmd, false, e.toString());
         } catch (final InternalErrorException e) {
@@ -1149,19 +1154,25 @@ public class KVMStorageProcessor implements StorageProcessor {
      * Return data store URL from store
      */
     private String getDataStoreUrlFromStore(DataStoreTO store) {
-        if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) ||
-                store instanceof PrimaryDataStoreTO && !((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem))) {
+        List<StoragePoolType> supportedPoolType = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem);
+        if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) || !supportedPoolType.contains(((PrimaryDataStoreTO) store).getPoolType()))) {
+            s_logger.error(String.format("Unsupported protocol, store: %s", store.getUuid()));
             throw new InvalidParameterValueException("unsupported protocol");
         }
 
         if (store instanceof NfsTO) {
             NfsTO nfsStore = (NfsTO)store;
             return nfsStore.getUrl();
-        } else if (store instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem)) {
+        } else if (store instanceof PrimaryDataStoreTO) {
             //In order to support directly downloaded ISOs
+            StoragePoolType poolType = ((PrimaryDataStoreTO)store).getPoolType();
             String psHost = ((PrimaryDataStoreTO) store).getHost();
             String psPath = ((PrimaryDataStoreTO) store).getPath();
-            return "nfs://" + psHost + File.separator + psPath;
+            if (StoragePoolType.NetworkFilesystem.equals(poolType)) {
+                return "nfs://" + psHost + File.separator + psPath;
+            } else if (StoragePoolType.Filesystem.equals(poolType)) {
+                return StoragePoolType.Filesystem.toString().toLowerCase() + "://" + psHost + File.separator + psPath;
+            }
         }
         return store.getUrl();
     }

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java

@@ -129,15 +129,15 @@ class VmwareVmImplementer {
                 }
             }
         } else {
-            // for user-VM, use E1000 as default
             if (nicDeviceType == null) {
-                details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString());
+                details.put(VmDetailConstants.NIC_ADAPTER, vmwareMgr.VmwareUserVmNicDeviceType.value());
             } else {
                 try {
                     VirtualEthernetCardType.valueOf(nicDeviceType);
                 } catch (Exception e) {
-                    LOGGER.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000");
-                    details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString());
+                    LOGGER.warn(String.format("Invalid NIC device type [%s] specified in VM details, switching to value [%s] of configuration [%s].",
+                            nicDeviceType, vmwareMgr.VmwareUserVmNicDeviceType.value(), vmwareMgr.VmwareUserVmNicDeviceType.toString()));
+                    details.put(VmDetailConstants.NIC_ADAPTER, vmwareMgr.VmwareUserVmNicDeviceType.value());
                 }
             }
         }

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java

@@ -53,6 +53,14 @@ public interface VmwareManager {
             "VMware interval window (in seconds) to collect metrics. If this is set to less than 20, then default (300 seconds) will be used. The interval used must be enabled in vCenter for this change to work, "
             + "otherwise the collection of metrics will result in an error. Check VMWare docs to know how to enable metrics interval.", true);
 
+    static final ConfigKey<String> VmwareUserVmNicDeviceType = new ConfigKey<String>(
+            String.class,
+            "vmware.uservm.nic.device.type",
+            "Advanced",
+            "E1000",
+            "Specify the default network device type for user VMs, valid values are E1000, PCNet32, Vmxnet2, Vmxnet3",
+            true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "E1000,PCNet32,Vmxnet2,Vmxnet3");
+
     String composeWorkerName();
 
     String getSystemVMIsoFileNameOnDatastore();

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -296,7 +296,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout, s_vmwareCleanupPortGroups, VMWARE_STATS_TIME_WINDOW};
+        return new ConfigKey<?>[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout, s_vmwareCleanupPortGroups, VMWARE_STATS_TIME_WINDOW, VmwareUserVmNicDeviceType};
     }
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {