Merge branch '4.22'

api/src/main/java/com/cloud/event/EventTypes.java

@@ -583,6 +583,7 @@ public class EventTypes {
 
     // Network ACL
     public static final String EVENT_NETWORK_ACL_CREATE = "NETWORK.ACL.CREATE";
+    public static final String EVENT_NETWORK_ACL_IMPORT = "NETWORK.ACL.IMPORT";
     public static final String EVENT_NETWORK_ACL_DELETE = "NETWORK.ACL.DELETE";
     public static final String EVENT_NETWORK_ACL_REPLACE = "NETWORK.ACL.REPLACE";
     public static final String EVENT_NETWORK_ACL_UPDATE = "NETWORK.ACL.UPDATE";

api/src/main/java/com/cloud/network/vpc/NetworkACLService.java

@@ -19,6 +19,7 @@ package com.cloud.network.vpc;
 import java.util.List;
 
 import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.ImportNetworkACLCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkACLListsCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
 import org.apache.cloudstack.api.command.user.network.MoveNetworkAclItemCmd;
@@ -98,4 +99,6 @@ public interface NetworkACLService {
     NetworkACLItem moveNetworkAclRuleToNewPosition(MoveNetworkAclItemCmd moveNetworkAclItemCmd);
 
     NetworkACLItem moveRuleToTheTopInACLList(NetworkACLItem ruleBeingMoved);
+
+    List<NetworkACLItem> importNetworkACLRules(ImportNetworkACLCmd cmd) throws ResourceUnavailableException;
 }

api/src/main/java/org/apache/cloudstack/alert/AlertService.java

@@ -71,8 +71,8 @@ public interface AlertService {
         public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true);
         public static final AlertType ALERT_TYPE_CA_CERT = new AlertType((short)31, "ALERT.CA.CERT", true);
         public static final AlertType ALERT_TYPE_VM_SNAPSHOT = new AlertType((short)32, "ALERT.VM.SNAPSHOT", true);
-        public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true);
-        public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true);
+        public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)33, "ALERT.VR.PUBLIC.IFACE.MTU", true);
+        public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)34, "ALERT.VR.PRIVATE.IFACE.MTU", true);
         public static final AlertType ALERT_TYPE_EXTENSION_PATH_NOT_READY = new AlertType((short)33, "ALERT.TYPE.EXTENSION.PATH.NOT.READY", true);
         public static final AlertType ALERT_TYPE_VPN_GATEWAY_OBSOLETE_PARAMETERS = new AlertType((short)34, "ALERT.S2S.VPN.GATEWAY.OBSOLETE.PARAMETERS", true);
         public static final AlertType ALERT_TYPE_BACKUP_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, "ALERT.STORAGE.BACKUP", true);

api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java

@@ -29,6 +29,11 @@ import org.apache.cloudstack.api.response.ZoneResponse;
 import com.cloud.exception.DiscoveryException;
 import com.cloud.storage.ImageStore;
 import com.cloud.user.Account;
+import org.apache.commons.collections.MapUtils;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
 
 @APICommand(name = "addSecondaryStorage", description = "Adds secondary storage.", responseObject = ImageStoreResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
@@ -44,6 +49,9 @@ public class AddSecondaryStorageCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "The Zone ID for the secondary storage")
     protected Long zoneId;
 
+    @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].copytemplatesfromothersecondarystorages=true")
+    protected Map details;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -56,6 +64,20 @@ public class AddSecondaryStorageCmd extends BaseCmd {
         return zoneId;
     }
 
+    public Map<String, String> getDetails() {
+        Map<String, String> detailsMap = new HashMap<>();
+        if (MapUtils.isNotEmpty(details)) {
+            Collection<?> props = details.values();
+            for (Object prop : props) {
+                HashMap<String, String> detail = (HashMap<String, String>) prop;
+                for (Map.Entry<String, String> entry: detail.entrySet()) {
+                    detailsMap.put(entry.getKey(),entry.getValue());
+                }
+            }
+        }
+        return detailsMap;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
@@ -68,7 +90,7 @@ public class AddSecondaryStorageCmd extends BaseCmd {
     @Override
     public void execute(){
         try{
-            ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null);
+            ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), getDetails());
             ImageStoreResponse storeResponse = null;
             if (result != null ) {
                     storeResponse = _responseGenerator.createImageStoreResponse(result);

api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java

@@ -63,6 +63,7 @@ public class ListCapabilitiesCmd extends BaseCmd {
         response.setDiskOffMaxSize((Long)capabilities.get("customDiskOffMaxSize"));
         response.setRegionSecondaryEnabled((Boolean)capabilities.get("regionSecondaryEnabled"));
         response.setKVMSnapshotEnabled((Boolean)capabilities.get("KVMSnapshotEnabled"));
+        response.setSnapshotShowChainSize((Boolean)capabilities.get("SnapshotShowChainSize"));
         response.setAllowUserViewDestroyedVM((Boolean)capabilities.get("allowUserViewDestroyedVM"));
         response.setAllowUserExpungeRecoverVM((Boolean)capabilities.get("allowUserExpungeRecoverVM"));
         response.setAllowUserExpungeRecoverVolume((Boolean)capabilities.get("allowUserExpungeRecoverVolume"));

api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLCmd.java

@@ -58,7 +58,7 @@ public class CreateNetworkACLCmd extends BaseAsyncCreateCmd {
     private Integer publicEndPort;
 
     @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, description = "The CIDR list to allow traffic from/to. Multiple entries must be separated by a single comma character (,).")
-    private List<String> cidrlist;
+    private List<String> cidrList;
 
     @Parameter(name = ApiConstants.ICMP_TYPE, type = CommandType.INTEGER, description = "Type of the ICMP message being sent")
     private Integer icmpType;
@@ -118,8 +118,8 @@ public class CreateNetworkACLCmd extends BaseAsyncCreateCmd {
     }
 
     public List<String> getSourceCidrList() {
-        if (cidrlist != null) {
-            return cidrlist;
+        if (cidrList != null) {
+            return cidrList;
         } else {
             List<String> oneCidrList = new ArrayList<String>();
             oneCidrList.add(NetUtils.ALL_IP4_CIDRS);
@@ -238,6 +238,30 @@ public class CreateNetworkACLCmd extends BaseAsyncCreateCmd {
         return reason;
     }
 
+    public void setCidrList(List<String> cidrList) {
+        this.cidrList = cidrList;
+    }
+
+    public void setIcmpType(Integer icmpType) {
+        this.icmpType = icmpType;
+    }
+
+    public void setIcmpCode(Integer icmpCode) {
+        this.icmpCode = icmpCode;
+    }
+
+    public void setNumber(Integer number) {
+        this.number = number;
+    }
+
+    public void setDisplay(Boolean display) {
+        this.display = display;
+    }
+
+    public void setReason(String reason) {
+        this.reason = reason;
+    }
+
     @Override
     public void create() {
         NetworkACLItem result = _networkACLService.createNetworkACLItem(this);

api/src/main/java/org/apache/cloudstack/api/command/user/network/ImportNetworkACLCmd.java

@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.network;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseAsyncCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.NetworkACLItemResponse;
+import org.apache.cloudstack.api.response.NetworkACLResponse;
+import org.apache.cloudstack.context.CallContext;
+import org.apache.commons.collections.MapUtils;
+
+import com.cloud.event.EventTypes;
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.vpc.NetworkACLItem;
+import com.cloud.user.Account;
+
+@APICommand(name = "importNetworkACL", description = "Imports Network ACL rules.",
+        responseObject = NetworkACLItemResponse.class,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        since = "4.22.1")
+public class ImportNetworkACLCmd extends BaseAsyncCmd {
+
+    // ///////////////////////////////////////////////////
+    // ////////////// API parameters /////////////////////
+    // ///////////////////////////////////////////////////
+
+    @Parameter(
+            name = ApiConstants.ACL_ID,
+            type = CommandType.UUID,
+            entityType = NetworkACLResponse.class,
+            required = true,
+            description = "The ID of the Network ACL to which the rules will be imported"
+    )
+    private Long aclId;
+
+    @Parameter(name = ApiConstants.RULES, type = CommandType.MAP, required = true,
+            description = "Rules param list, id and protocol are must. Invalid rules will be discarded. Example: " +
+                    "rules[0].id=101&rules[0].protocol=tcp&rules[0].traffictype=ingress&rules[0].state=active&rules[0].cidrlist=192.168.1.0/24" +
+                    "&rules[0].tags=web&rules[0].aclid=acl-001&rules[0].aclname=web-acl&rules[0].number=1&rules[0].action=allow&rules[0].fordisplay=true" +
+                    "&rules[0].description=allow%20web%20traffic&rules[1].id=102&rules[1].protocol=udp&rules[1].traffictype=egress&rules[1].state=enabled" +
+                    "&rules[1].cidrlist=10.0.0.0/8&rules[1].tags=db&rules[1].aclid=acl-002&rules[1].aclname=db-acl&rules[1].number=2&rules[1].action=deny" +
+                    "&rules[1].fordisplay=false&rules[1].description=deny%20database%20traffic")
+    private Map rules;
+
+
+    // ///////////////////////////////////////////////////
+    // ///////////////// Accessors ///////////////////////
+    // ///////////////////////////////////////////////////
+
+    // Returns map, corresponds to a rule with the details in the keys:
+    // id, protocol, startport, endport, traffictype, state, cidrlist, tags, aclid, aclname, number, action, fordisplay, description
+    public Map getRules() {
+        return rules;
+    }
+
+    public Long getAclId() {
+        return aclId;
+    }
+
+    // ///////////////////////////////////////////////////
+    // ///////////// API Implementation///////////////////
+    // ///////////////////////////////////////////////////
+
+
+    @Override
+    public void execute() throws ResourceUnavailableException {
+        validateParams();
+        List<NetworkACLItem> importedRules = _networkACLService.importNetworkACLRules(this);
+        ListResponse<NetworkACLItemResponse> response = new ListResponse<>();
+        List<NetworkACLItemResponse> aclResponse = new ArrayList<>();
+        for (NetworkACLItem acl : importedRules) {
+            NetworkACLItemResponse ruleData = _responseGenerator.createNetworkACLItemResponse(acl);
+            aclResponse.add(ruleData);
+        }
+        response.setResponses(aclResponse, importedRules.size());
+        response.setResponseName(getCommandName());
+        setResponseObject(response);
+    }
+
+    @Override
+    public long getEntityOwnerId() {
+        Account account = CallContext.current().getCallingAccount();
+        if (account != null) {
+            return account.getId();
+        }
+        return Account.ACCOUNT_ID_SYSTEM;
+    }
+
+    @Override
+    public String getEventType() {
+        return EventTypes.EVENT_NETWORK_ACL_IMPORT;
+    }
+
+    @Override
+    public String getEventDescription() {
+        return "Importing ACL rules for ACL ID: " + getAclId();
+    }
+
+
+    private void validateParams() {
+        if(MapUtils.isEmpty(rules)) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Rules parameter is empty or null");
+        }
+
+        if (getAclId() == null || _networkACLService.getNetworkACL(getAclId()) == null) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find Network ACL with provided ACL ID");
+        }
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java

@@ -274,8 +274,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd {
     }
 
     public Snapshot.LocationType getLocationType() {
-
-        if (Snapshot.LocationType.values() == null || Snapshot.LocationType.values().length == 0 || locationType == null) {
+        if (locationType == null) {
             return null;
         }
 

api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java

@@ -75,6 +75,10 @@ public class CapabilitiesResponse extends BaseResponse {
     @Param(description = "True if Snapshot is supported for KVM host, false otherwise")
     private boolean kvmSnapshotEnabled;
 
+    @SerializedName("snapshotshowchainsize")
+    @Param(description = "True to show the parent and chain size (sum of physical size of snapshot and all its parents) for incremental snapshots", since = "4.22.1")
+    private boolean snapshotShowChainSize;
+
     @SerializedName("apilimitmax")
     @Param(description = "Max allowed number of api requests within the specified interval")
     private Integer apiLimitMax;
@@ -203,6 +207,10 @@ public class CapabilitiesResponse extends BaseResponse {
         this.kvmSnapshotEnabled = kvmSnapshotEnabled;
     }
 
+    public void setSnapshotShowChainSize(boolean snapshotShowChainSize) {
+        this.snapshotShowChainSize = snapshotShowChainSize;
+    }
+
     public void setApiLimitInterval(Integer apiLimitInterval) {
         this.apiLimitInterval = apiLimitInterval;
     }

api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java

@@ -155,6 +155,14 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements
     @Param(description = "download progress of a snapshot", since = "4.19.0")
     private Map<String, String> downloadDetails;
 
+    @SerializedName("parent")
+    @Param(description = "The parent ID of the Snapshot", since = "4.22.1")
+    private String parent;
+
+    @SerializedName("parentname")
+    @Param(description = "The parent name of the Snapshot", since = "4.22.1")
+    private String parentName;
+
     public SnapshotResponse() {
         tags = new LinkedHashSet<ResourceTagResponse>();
     }
@@ -313,4 +321,12 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements
     public void setDownloadDetails(Map<String, String> downloadDetails) {
         this.downloadDetails = downloadDetails;
     }
+
+    public void setParent(String parent) {
+        this.parent = parent;
+    }
+
+    public void setParentName(String parentName) {
+        this.parentName = parentName;
+    }
 }

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -71,11 +71,6 @@ public class ServerDaemon implements Daemon {
     private static final String BIND_INTERFACE = "bind.interface";
     private static final String CONTEXT_PATH = "context.path";
     private static final String SESSION_TIMEOUT = "session.timeout";
-    private static final String HTTP_ENABLE = "http.enable";
-    private static final String HTTP_PORT = "http.port";
-    private static final String HTTPS_ENABLE = "https.enable";
-    private static final String HTTPS_PORT = "https.port";
-    private static final String KEYSTORE_FILE = "https.keystore";
     private static final String KEYSTORE_PASSWORD = "https.keystore.password";
     private static final String WEBAPP_DIR = "webapp.dir";
     private static final String ACCESS_LOG = "access.log";
@@ -137,11 +132,11 @@ public class ServerDaemon implements Daemon {
             }
             setBindInterface(properties.getProperty(BIND_INTERFACE, null));
             setContextPath(properties.getProperty(CONTEXT_PATH, "/client"));
-            setHttpEnable(Boolean.valueOf(properties.getProperty(HTTP_ENABLE, "true")));
-            setHttpPort(Integer.valueOf(properties.getProperty(HTTP_PORT, "8080")));
-            setHttpsEnable(Boolean.valueOf(properties.getProperty(HTTPS_ENABLE, "false")));
-            setHttpsPort(Integer.valueOf(properties.getProperty(HTTPS_PORT, "8443")));
-            setKeystoreFile(properties.getProperty(KEYSTORE_FILE));
+            setHttpEnable(Boolean.valueOf(properties.getProperty(ServerProperties.HTTP_ENABLE, "true")));
+            setHttpPort(Integer.valueOf(properties.getProperty(ServerProperties.HTTP_PORT, "8080")));
+            setHttpsEnable(Boolean.valueOf(properties.getProperty(ServerProperties.HTTPS_ENABLE, "false")));
+            setHttpsPort(Integer.valueOf(properties.getProperty(ServerProperties.HTTPS_PORT, "8443")));
+            setKeystoreFile(properties.getProperty(ServerProperties.KEYSTORE_FILE));
             setKeystorePassword(properties.getProperty(KEYSTORE_PASSWORD));
             setWebAppLocation(properties.getProperty(WEBAPP_DIR));
             setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log"));

engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java

@@ -22,7 +22,6 @@ import java.util.concurrent.Future;
 
 import org.apache.cloudstack.api.response.MigrationResponse;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
-import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
 import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy;
 
@@ -31,5 +30,5 @@ public interface StorageOrchestrationService {
 
     MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List<Long> templateIdList, List<Long> snapshotIdList);
 
-    Future<TemplateApiResult> orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore);
+    Future<TemplateApiResult> orchestrateTemplateCopyFromSecondaryStores(long templateId, DataStore destStore);
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java

@@ -80,4 +80,6 @@ public interface TemplateService {
     List<DatadiskTO> getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId);
 
     AsyncCallFuture<TemplateApiResult> copyTemplateToImageStore(DataObject source, DataStore destStore);
-}
+
+    void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore);
+ }

engine/components-api/src/main/java/com/cloud/alert/AlertManager.java

@@ -54,5 +54,4 @@ public interface AlertManager extends Manager, AlertService {
     void recalculateCapacity();
 
     void sendAlert(AlertType alertType, long dataCenterId, Long podId, String subject, String body);
-
 }

engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java

@@ -57,8 +57,8 @@ public interface ResourceManager extends ResourceService, Configurable {
 
     ConfigKey<Boolean> KvmSshToAgentEnabled = new ConfigKey<>("Advanced", Boolean.class,
             "kvm.ssh.to.agent","true",
-            "Number of retries when preparing a host into Maintenance Mode is faulty before failing",
-            false);
+            "True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations",
+            true);
 
     ConfigKey<String> HOST_MAINTENANCE_LOCAL_STRATEGY = new ConfigKey<>(String.class,
     "host.maintenance.local.storage.strategy", "Advanced","Error",

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -228,8 +228,9 @@ public interface StorageManager extends StorageService {
             ConfigKey.Scope.Global,
             null);
 
-    ConfigKey<Boolean> COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES = new ConfigKey<>(Boolean.class, "copy.public.templates.from.other.storages",
-            "Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.",
+    ConfigKey<Boolean> COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES = new ConfigKey<>(Boolean.class, "copy.templates.from.other.secondary.storages",
+            "Storage", "true", "When enabled, this feature allows templates to be copied from existing Secondary Storage servers (within the same zone or across zones) " +
+            "while adding a new Secondary Storage. If the copy operation fails, the system falls back to downloading the template from the source URL.",
             true, ConfigKey.Scope.Zone, null);
 
     /**

engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java

@@ -36,6 +36,9 @@ import java.util.stream.Collectors;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.template.TemplateManager;
 import org.apache.cloudstack.api.response.MigrationResponse;
 import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@@ -45,6 +48,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageServic
 import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
@@ -103,6 +107,15 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
     VolumeDataStoreDao volumeDataStoreDao;
     @Inject
     DataMigrationUtility migrationHelper;
+    @Inject
+    TemplateManager templateManager;
+    @Inject
+    VMTemplateDao templateDao;
+    @Inject
+    TemplateDataFactory templateDataFactory;
+    @Inject
+    DataCenterDao dcDao;
+
 
     ConfigKey<Double> ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class,
             "image.store.imbalance.threshold",
@@ -304,8 +317,9 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
     }
 
     @Override
-    public Future<TemplateApiResult> orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore) {
-        return submit(destStore.getScope().getScopeId(), new CopyTemplateTask(source, destStore));
+    public Future<TemplateApiResult> orchestrateTemplateCopyFromSecondaryStores(long srcTemplateId, DataStore destStore) {
+        Long dstZoneId = destStore.getScope().getScopeId();
+        return submit(dstZoneId, new CopyTemplateFromSecondaryStorageTask(srcTemplateId, destStore));
     }
 
     protected Pair<String, Boolean> migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List<DataObject> files, MigrationPolicy migrationPolicy, int skipped) {
@@ -624,13 +638,13 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
         }
     }
 
-    private class CopyTemplateTask implements Callable<TemplateApiResult> {
-        private TemplateInfo sourceTmpl;
-        private DataStore destStore;
-        private String logid;
+    private class CopyTemplateFromSecondaryStorageTask implements Callable<TemplateApiResult> {
+        private final long srcTemplateId;
+        private final DataStore destStore;
+        private final String logid;
 
-        public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) {
-            this.sourceTmpl = sourceTmpl;
+        CopyTemplateFromSecondaryStorageTask(long srcTemplateId, DataStore destStore) {
+            this.srcTemplateId = srcTemplateId;
             this.destStore = destStore;
             this.logid = ThreadContext.get(LOGCONTEXTID);
         }
@@ -639,17 +653,16 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra
         public TemplateApiResult call() {
             ThreadContext.put(LOGCONTEXTID, logid);
             TemplateApiResult result;
-            AsyncCallFuture<TemplateApiResult> future = templateService.copyTemplateToImageStore(sourceTmpl, destStore);
+            long destZoneId = destStore.getScope().getScopeId();
+            TemplateInfo sourceTmpl = templateDataFactory.getTemplate(srcTemplateId, DataStoreRole.Image);
             try {
-                result = future.get();
-            } catch (ExecutionException | InterruptedException e) {
-                logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}",
-                        sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString());
+                templateService.handleTemplateCopyFromSecondaryStores(srcTemplateId, destStore);
                 result = new TemplateApiResult(sourceTmpl);
-                result.setResult(e.getMessage());
+            } finally {
+                tryCleaningUpExecutor(destZoneId);
+                ThreadContext.clearAll();
             }
-            tryCleaningUpExecutor(destStore.getScope().getScopeId());
-            ThreadContext.clearAll();
+
             return result;
         }
     }

engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java

@@ -62,7 +62,7 @@ public interface UsageDao extends GenericDao<UsageVO, Long> {
 
     void saveUsageRecords(List<UsageVO> usageRecords);
 
-    void removeOldUsageRecords(int days);
+    void expungeAllOlderThan(int days, long limitPerQuery);
 
     UsageVO persistUsage(final UsageVO usage);
 

engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java

@@ -26,15 +26,16 @@ import com.cloud.utils.Pair;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.QueryBuilder;
+import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallback;
-import com.cloud.utils.db.TransactionCallbackNoReturn;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
 
 import org.apache.cloudstack.acl.RoleType;
+import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.stereotype.Component;
 
 import java.sql.PreparedStatement;
@@ -51,7 +52,6 @@ import java.util.TimeZone;
 public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements UsageDao {
     private static final String DELETE_ALL = "DELETE FROM cloud_usage";
     private static final String DELETE_ALL_BY_ACCOUNTID = "DELETE FROM cloud_usage WHERE account_id = ?";
-    private static final String DELETE_ALL_BY_INTERVAL = "DELETE FROM cloud_usage WHERE end_date < DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY)";
     private static final String INSERT_ACCOUNT = "INSERT INTO cloud_usage.account (id, account_name, uuid, type, role_id, domain_id, removed, cleanup_needed) VALUES (?,?,?,?,?,?,?,?)";
     private static final String INSERT_USER_STATS = "INSERT INTO cloud_usage.user_statistics (id, data_center_id, account_id, public_ip_address, device_id, device_type, network_id, net_bytes_received,"
             + " net_bytes_sent, current_bytes_received, current_bytes_sent, agg_bytes_received, agg_bytes_sent) VALUES (?,?,?,?,?,?,?,?,?,?, ?, ?, ?)";
@@ -88,8 +88,12 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage
 
     private static final String UPDATE_BUCKET_STATS = "UPDATE cloud_usage.bucket_statistics SET size=? WHERE id=?";
 
+    protected SearchBuilder<UsageVO> endDateLessThanSearch;
 
     public UsageDaoImpl() {
+        endDateLessThanSearch = createSearchBuilder();
+        endDateLessThanSearch.and("endDate", endDateLessThanSearch.entity().getEndDate(), SearchCriteria.Op.LT);
+        endDateLessThanSearch.done();
     }
 
     @Override
@@ -539,21 +543,20 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage
     }
 
     @Override
-    public void removeOldUsageRecords(int days) {
-        Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallbackNoReturn() {
-            @Override
-            public void doInTransactionWithoutResult(TransactionStatus status) {
-                TransactionLegacy txn = TransactionLegacy.currentTxn();
-                PreparedStatement pstmt = null;
-                try {
-                    pstmt = txn.prepareAutoCloseStatement(DELETE_ALL_BY_INTERVAL);
-                    pstmt.setLong(1, days);
-                    pstmt.executeUpdate();
-                } catch (Exception ex) {
-                    logger.error("error removing old cloud_usage records for interval: " + days);
-                }
-            }
-        });
+    public void expungeAllOlderThan(int days, long limitPerQuery) {
+        SearchCriteria<UsageVO> sc = endDateLessThanSearch.create();
+
+        Date limit = DateUtils.addDays(new Date(), -days);
+        sc.setParameters("endDate", limit);
+
+        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
+        try {
+            logger.debug("Removing all cloud_usage records older than [{}].", limit);
+            int totalExpunged = batchExpunge(sc, limitPerQuery);
+            logger.info("Removed a total of [{}] cloud_usage records older than [{}].", totalExpunged, limit);
+        } finally {
+            txn.close();
+        }
     }
 
     public UsageVO persistUsage(final UsageVO usage) {

engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java

@@ -28,6 +28,8 @@ public interface UsageJobDao extends GenericDao<UsageJobVO, Long> {
 
     UsageJobVO getLastJob();
 
+    UsageJobVO getNextRecurringJob();
+
     UsageJobVO getNextImmediateJob();
 
     long getLastJobSuccessDateMillis();

engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java

@@ -156,7 +156,8 @@ public class UsageJobDaoImpl extends GenericDaoBase<UsageJobVO, Long> implements
         return jobs.get(0);
     }
 
-    private UsageJobVO getNextRecurringJob() {
+    @Override
+    public UsageJobVO getNextRecurringJob() {
         Filter filter = new Filter(UsageJobVO.class, "id", false, Long.valueOf(0), Long.valueOf(1));
         SearchCriteria<UsageJobVO> sc = createSearchCriteria();
         sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0));

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java

@@ -130,4 +130,18 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
     void updateDisplayForSnapshotStoreRole(long snapshotId, long storeId, DataStoreRole role, boolean display);
 
     int expungeBySnapshotList(List<Long> snapshotIds, Long batchSize);
+
+    /**
+     * Returns the total physical size, in bytes, of all snapshots stored on primary
+     * storage for the specified account that have not yet been backed up to
+     * secondary storage.
+     *
+     * <p>If no such snapshots are found, this method returns {@code 0}.</p>
+     *
+     * @param accountId the ID of the account whose snapshots on primary storage
+     *                  should be considered
+     * @return the total physical size in bytes of matching snapshots on primary
+     *         storage, or {@code 0} if none are found
+     */
+    long getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(long accountId);
 }

engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java

@@ -67,7 +67,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
     private SearchBuilder<SnapshotDataStoreVO> searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq;
     protected SearchBuilder<SnapshotDataStoreVO> searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq;
     private SearchBuilder<SnapshotDataStoreVO> stateSearch;
-    private SearchBuilder<SnapshotDataStoreVO> idStateNinSearch;
+    private SearchBuilder<SnapshotDataStoreVO> idStateNeqSearch;
     protected SearchBuilder<SnapshotVO> snapshotVOSearch;
     private SearchBuilder<SnapshotDataStoreVO> snapshotCreatedSearch;
     private SearchBuilder<SnapshotDataStoreVO> dataStoreAndInstallPathSearch;
@@ -96,6 +96,15 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
             "snapshot_store_ref ssr, snapshots s " +
             "WHERE ssr.snapshot_id=? AND ssr.snapshot_id = s.id AND s.data_center_id=?;";
 
+    private static final String GET_PHYSICAL_SIZE_OF_SNAPSHOTS_ON_PRIMARY_BY_ACCOUNT = "SELECT SUM(s.physical_size) " +
+            "FROM cloud.snapshot_store_ref s " +
+            "LEFT JOIN cloud.snapshots ON s.snapshot_id = snapshots.id " +
+            "WHERE snapshots.account_id = ? " +
+            "AND snapshots.removed IS NULL " +
+            "AND s.state = 'Ready' " +
+            "AND s.store_role = 'Primary' " +
+            "AND NOT EXISTS (SELECT 1 FROM cloud.snapshot_store_ref i WHERE i.snapshot_id = s.snapshot_id AND i.store_role = 'Image')";
+
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
         super.configure(name, params);
@@ -137,10 +146,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
         stateSearch.done();
 
 
-        idStateNinSearch = createSearchBuilder();
-        idStateNinSearch.and(SNAPSHOT_ID, idStateNinSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ);
-        idStateNinSearch.and(STATE, idStateNinSearch.entity().getState(), SearchCriteria.Op.NOTIN);
-        idStateNinSearch.done();
+        idStateNeqSearch = createSearchBuilder();
+        idStateNeqSearch.and(SNAPSHOT_ID, idStateNeqSearch.entity().getSnapshotId(), SearchCriteria.Op.EQ);
+        idStateNeqSearch.and(STATE, idStateNeqSearch.entity().getState(), SearchCriteria.Op.NEQ);
+        idStateNeqSearch.done();
 
         snapshotVOSearch = snapshotDao.createSearchBuilder();
         snapshotVOSearch.and(VOLUME_ID, snapshotVOSearch.entity().getVolumeId(), SearchCriteria.Op.EQ);
@@ -471,7 +480,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
 
     @Override
     public List<SnapshotDataStoreVO> findBySnapshotIdWithNonDestroyedState(long snapshotId) {
-        SearchCriteria<SnapshotDataStoreVO> sc = idStateNinSearch.create();
+        SearchCriteria<SnapshotDataStoreVO> sc = idStateNeqSearch.create();
         sc.setParameters(SNAPSHOT_ID, snapshotId);
         sc.setParameters(STATE, State.Destroyed.name());
         return listBy(sc);
@@ -479,7 +488,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
 
     @Override
     public List<SnapshotDataStoreVO> findBySnapshotIdAndNotInDestroyedHiddenState(long snapshotId) {
-        SearchCriteria<SnapshotDataStoreVO> sc = idStateNinSearch.create();
+        SearchCriteria<SnapshotDataStoreVO> sc = idStateNeqSearch.create();
         sc.setParameters(SNAPSHOT_ID, snapshotId);
         sc.setParameters(STATE, State.Destroyed.name(), State.Hidden.name());
         return listBy(sc);
@@ -732,4 +741,23 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
         sc.setParameters("snapshotIds", snapshotIds.toArray());
         return batchExpunge(sc, batchSize);
     }
+
+    @Override
+    public long getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(long accountId) {
+        long snapshotsPhysicalSize = 0;
+        try (TransactionLegacy transactionLegacy = TransactionLegacy.currentTxn()) {
+            try (PreparedStatement preparedStatement = transactionLegacy.prepareStatement(GET_PHYSICAL_SIZE_OF_SNAPSHOTS_ON_PRIMARY_BY_ACCOUNT)) {
+                preparedStatement.setLong(1, accountId);
+
+                try (ResultSet resultSet = preparedStatement.executeQuery()) {
+                    if (resultSet.next()) {
+                        snapshotsPhysicalSize = resultSet.getLong(1);
+                    }
+                }
+            }
+        } catch (SQLException e) {
+            logger.warn("Failed to get the snapshots physical size for the account [{}] due to [{}].", accountId, e.getMessage(), e);
+        }
+        return snapshotsPhysicalSize;
+    }
 }

engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql

@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Add new OS categories if not present
+DROP PROCEDURE IF EXISTS `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`;
+CREATE PROCEDURE `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`(IN os_name VARCHAR(255))
+BEGIN
+    IF NOT EXISTS ((SELECT 1 FROM `cloud`.`guest_os_category` WHERE name = os_name))
+    THEN
+        INSERT INTO `cloud`.`guest_os_category` (name, uuid)
+            VALUES (os_name, UUID())
+;   END IF
+; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql

@@ -0,0 +1,46 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_DETAILS_IF_NOT_EXISTS`;
+CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_DETAILS_IF_NOT_EXISTS` (
+    IN ext_name VARCHAR(255),
+    IN action_name VARCHAR(255),
+    IN param_json TEXT
+)
+BEGIN
+    DECLARE action_id BIGINT UNSIGNED
+;   SELECT `eca`.`id` INTO action_id FROM `cloud`.`extension_custom_action` `eca`
+    JOIN `cloud`.`extension` `e` ON `e`.`id` = `eca`.`extension_id`
+    WHERE `eca`.`name` = action_name AND `e`.`name` = ext_name LIMIT 1
+;   IF NOT EXISTS (
+        SELECT 1 FROM `cloud`.`extension_custom_action_details`
+        WHERE `extension_custom_action_id` = action_id
+          AND `name` = 'parameters'
+    ) THEN
+        INSERT INTO `cloud`.`extension_custom_action_details` (
+            `extension_custom_action_id`,
+            `name`,
+            `value`,
+            `display`
+        ) VALUES (
+            action_id,
+            'parameters',
+            param_json,
+            0
+        )
+;   END IF
+;END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql

@@ -0,0 +1,46 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_IF_NOT_EXISTS`;
+CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_IF_NOT_EXISTS`(
+    IN ext_name VARCHAR(255),
+    IN action_name VARCHAR(255),
+    IN action_desc VARCHAR(4096),
+    IN resource_type VARCHAR(255),
+    IN allowed_roles INT UNSIGNED,
+    IN success_msg VARCHAR(4096),
+    IN error_msg VARCHAR(4096),
+    IN timeout_seconds INT UNSIGNED
+)
+BEGIN
+    DECLARE ext_id BIGINT
+;   SELECT `id` INTO ext_id FROM `cloud`.`extension` WHERE `name` = ext_name LIMIT 1
+;   IF NOT EXISTS (
+        SELECT 1 FROM `cloud`.`extension_custom_action` WHERE `name` = action_name AND `extension_id` = ext_id
+    ) THEN
+        INSERT INTO `cloud`.`extension_custom_action` (
+            `uuid`, `name`, `description`, `extension_id`, `resource_type`,
+            `allowed_role_types`, `success_message`, `error_message`,
+            `enabled`, `timeout`, `created`, `removed`
+        )
+        VALUES (
+            UUID(), action_name, action_desc, ext_id, resource_type,
+            allowed_roles, success_msg, error_msg,
+            1, timeout_seconds, NOW(), NULL
+        )
+;   END IF
+;END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql

@@ -0,0 +1,39 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`;
+CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`(
+    IN ext_name VARCHAR(255),
+    IN detail_key VARCHAR(255),
+    IN detail_value TEXT,
+    IN display TINYINT(1)
+)
+BEGIN
+    DECLARE ext_id BIGINT
+;   SELECT `id` INTO ext_id FROM `cloud`.`extension` WHERE `name` = ext_name LIMIT 1
+;   IF NOT EXISTS (
+        SELECT 1 FROM `cloud`.`extension_details`
+        WHERE `extension_id` = ext_id AND `name` = detail_key
+    ) THEN
+        INSERT INTO `cloud`.`extension_details` (
+            `extension_id`, `name`, `value`, `display`
+        )
+        VALUES (
+            ext_id, detail_key, detail_value, display
+        )
+;   END IF
+;END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql

@@ -0,0 +1,38 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_IF_NOT_EXISTS`;
+CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_IF_NOT_EXISTS`(
+    IN ext_name VARCHAR(255),
+    IN ext_desc VARCHAR(255),
+    IN ext_path VARCHAR(255)
+)
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1 FROM `cloud`.`extension` WHERE `name` = ext_name
+    ) THEN
+        INSERT INTO `cloud`.`extension` (
+            `uuid`, `name`, `description`, `type`,
+            `relative_path`, `path_ready`,
+            `is_user_defined`, `state`, `created`, `removed`
+        )
+        VALUES (
+            UUID(), ext_name, ext_desc, 'Orchestrator',
+            ext_path, 1, 0, 'Enabled', NOW(), NULL
+        )
+;   END IF
+;END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql

@@ -0,0 +1,33 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Move existing guest OS to new categories
+DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`;
+CREATE PROCEDURE `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`(IN category_name VARCHAR(255), IN os_name VARCHAR(255))
+BEGIN
+    DECLARE category_id BIGINT
+;   SELECT `id` INTO category_id
+    FROM `cloud`.`guest_os_category`
+    WHERE `name` = category_name
+    LIMIT 1
+;   IF category_id IS NULL THEN
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Category not found'
+;   END IF
+;   UPDATE `cloud`.`guest_os`
+    SET `category_id` = category_id
+    WHERE `display_name` LIKE CONCAT('%', os_name, '%')
+; END;

engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql

@@ -0,0 +1,35 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Move existing guest OS whose category will be deleted to Other category
+DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`;
+CREATE PROCEDURE `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`(IN to_category_name VARCHAR(255), IN from_category_name VARCHAR(255))
+BEGIN
+    DECLARE done INT DEFAULT 0
+;   DECLARE to_category_id BIGINT
+;   SELECT id INTO to_category_id
+    FROM `cloud`.`guest_os_category`
+    WHERE `name` = to_category_name
+    LIMIT 1
+;   IF to_category_id IS NULL THEN
+        SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'ToCategory not found'
+;   END IF
+;   UPDATE `cloud`.`guest_os`
+    SET `category_id` = to_category_id
+    WHERE `category_id` = (SELECT `id` FROM `cloud`.`guest_os_category` WHERE `name` = from_category_name)
+;   UPDATE `cloud`.`guest_os_category` SET `removed`=now() WHERE `name` = from_category_name
+; END;

engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql

@@ -20,3 +20,9 @@
 --;
 
 ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048);
+
+UPDATE `cloud`.`alert` SET type = 33 WHERE name = 'ALERT.VR.PUBLIC.IFACE.MTU';
+UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU';
+
+-- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields
+UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent';

engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql

@@ -27,3 +27,9 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'b
 CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"');
 
 ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048);
+
+UPDATE `cloud`.`alert` SET type = 33 WHERE name = 'ALERT.VR.PUBLIC.IFACE.MTU';
+UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU';
+
+-- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields
+UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent';

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -31,6 +31,8 @@ import java.util.concurrent.ExecutionException;
 
 import javax.inject.Inject;
 
+import com.cloud.exception.StorageUnavailableException;
+import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
@@ -67,9 +69,11 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.ThreadContext;
 import org.springframework.stereotype.Component;
 
 import com.cloud.agent.api.Answer;
@@ -569,10 +573,7 @@ public class TemplateServiceImpl implements TemplateService {
                             }
 
                             if (availHypers.contains(tmplt.getHypervisorType())) {
-                                boolean copied = isCopyFromOtherStoragesEnabled(zoneId) && tryCopyingTemplateToImageStore(tmplt, store);
-                                if (!copied) {
-                                    tryDownloadingTemplateToImageStore(tmplt, store);
-                                }
+                                storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(tmplt.getId(), store);
                             } else {
                                 logger.info("Skip downloading template {} since current data center does not have hypervisor {}", tmplt, tmplt.getHypervisorType());
                             }
@@ -619,6 +620,16 @@ public class TemplateServiceImpl implements TemplateService {
 
     }
 
+    @Override
+    public void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore) {
+        VMTemplateVO template = _templateDao.findById(templateId);
+        long zoneId = destStore.getScope().getScopeId();
+        boolean copied = imageStoreDetailsUtil.isCopyTemplatesFromOtherStoragesEnabled(destStore.getId(), zoneId) && tryCopyingTemplateToImageStore(template, destStore);
+        if (!copied) {
+            tryDownloadingTemplateToImageStore(template, destStore);
+        }
+    }
+
     protected void tryDownloadingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) {
         if (tmplt.getUrl() == null) {
             logger.info("Not downloading template [{}] to image store [{}], as it has no URL.", tmplt.getUniqueName(),
@@ -636,28 +647,134 @@ public class TemplateServiceImpl implements TemplateService {
     }
 
     protected boolean tryCopyingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) {
-        Long zoneId = destStore.getScope().getScopeId();
-        List<DataStore> storesInZone = _storeMgr.getImageStoresByZoneIds(zoneId);
-        for (DataStore sourceStore : storesInZone) {
-            Map<String, TemplateProp> existingTemplatesInSourceStore = listTemplate(sourceStore);
-            if (existingTemplatesInSourceStore == null || !existingTemplatesInSourceStore.containsKey(tmplt.getUniqueName())) {
-                logger.debug("Template [{}] does not exist on image store [{}]; searching on another one.",
-                        tmplt.getUniqueName(), sourceStore.getName());
-                continue;
-            }
-            TemplateObject sourceTmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), sourceStore);
-            if (sourceTmpl.getInstallPath() == null) {
-                logger.warn("Can not copy template [{}] from image store [{}], as it returned a null install path.", tmplt.getUniqueName(),
-                        sourceStore.getName());
-                continue;
-            }
-            storageOrchestrator.orchestrateTemplateCopyToImageStore(sourceTmpl, destStore);
+        if (searchAndCopyWithinZone(tmplt, destStore)) {
             return true;
         }
-        logger.debug("Can't copy template [{}] from another image store.", tmplt.getUniqueName());
+
+        Long destZoneId = destStore.getScope().getScopeId();
+        logger.debug("Template [{}] not found in any image store of zone [{}]. Checking other zones.",
+                tmplt.getUniqueName(), destZoneId);
+
+        return searchAndCopyAcrossZones(tmplt, destStore, destZoneId);
+    }
+
+    private boolean searchAndCopyAcrossZones(VMTemplateVO tmplt, DataStore destStore, Long destZoneId) {
+        List<Long> allZoneIds = _dcDao.listAllIds();
+        for (Long otherZoneId : allZoneIds) {
+            if (otherZoneId.equals(destZoneId)) {
+                continue;
+            }
+
+            List<DataStore> storesInOtherZone = _storeMgr.getImageStoresByZoneIds(otherZoneId);
+            logger.debug("Checking zone [{}] for template [{}]...", otherZoneId, tmplt.getUniqueName());
+
+            if (CollectionUtils.isEmpty(storesInOtherZone)) {
+                logger.debug("Zone [{}] has no image stores. Skipping.", otherZoneId);
+                continue;
+            }
+
+            TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInOtherZone);
+            if (sourceTmpl == null) {
+                logger.debug("Template [{}] not found with a valid install path in any image store of zone [{}].",
+                        tmplt.getUniqueName(), otherZoneId);
+                continue;
+            }
+
+            logger.info("Template [{}] found in zone [{}]. Initiating cross-zone copy to zone [{}].",
+                    tmplt.getUniqueName(), otherZoneId, destZoneId);
+
+            return copyTemplateAcrossZones(destStore, sourceTmpl);
+        }
+
+        logger.debug("Template [{}] was not found in any zone. Cannot perform zone-to-zone copy.", tmplt.getUniqueName());
         return false;
     }
 
+    protected TemplateObject findUsableTemplate(VMTemplateVO tmplt, List<DataStore> imageStores) {
+        for (DataStore store : imageStores) {
+
+            Map<String, TemplateProp> templates = listTemplate(store);
+            if (templates == null || !templates.containsKey(tmplt.getUniqueName())) {
+                continue;
+            }
+
+            TemplateObject tmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), store);
+            if (tmpl.getInstallPath() == null) {
+                logger.debug("Template [{}] found in image store [{}] but install path is null. Skipping.",
+                        tmplt.getUniqueName(), store.getName());
+                continue;
+            }
+            return tmpl;
+        }
+        return null;
+    }
+
+    private boolean searchAndCopyWithinZone(VMTemplateVO tmplt, DataStore destStore) {
+        Long destZoneId = destStore.getScope().getScopeId();
+        List<DataStore> storesInSameZone = _storeMgr.getImageStoresByZoneIds(destZoneId);
+
+        TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInSameZone);
+        if (sourceTmpl == null) {
+            return false;
+        }
+
+        TemplateApiResult result;
+        AsyncCallFuture<TemplateApiResult> future = copyTemplateToImageStore(sourceTmpl, destStore);
+        try {
+            result = future.get();
+        } catch (ExecutionException | InterruptedException e) {
+            logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}",
+                    sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString());
+            result = new TemplateApiResult(sourceTmpl);
+            result.setResult(e.getMessage());
+        }
+        return result.isSuccess();
+    }
+
+    private boolean copyTemplateAcrossZones(DataStore destStore, TemplateObject sourceTmpl) {
+        Long dstZoneId = destStore.getScope().getScopeId();
+        DataCenterVO dstZone = _dcDao.findById(dstZoneId);
+
+        if (dstZone == null) {
+            logger.warn("Destination zone [{}] not found for template [{}].", dstZoneId, sourceTmpl.getUniqueName());
+            return false;
+        }
+
+        TemplateApiResult result;
+        try {
+            VMTemplateVO template = _templateDao.findById(sourceTmpl.getId());
+            try {
+                DataStore sourceStore = sourceTmpl.getDataStore();
+                long userId = CallContext.current().getCallingUserId();
+                boolean success = _tmpltMgr.copy(userId, template, sourceStore, dstZone);
+
+                result = new TemplateApiResult(sourceTmpl);
+                if (!success) {
+                    result.setResult("Cross-zone template copy failed");
+                }
+            } catch (StorageUnavailableException | ResourceAllocationException e) {
+                logger.error("Exception while copying template [{}] from zone [{}] to zone [{}]",
+                        template,
+                        sourceTmpl.getDataStore().getScope().getScopeId(),
+                        dstZone.getId(),
+                        e);
+                result = new TemplateApiResult(sourceTmpl);
+                result.setResult(e.getMessage());
+            } finally {
+                ThreadContext.clearAll();
+            }
+        } catch (Exception e) {
+            logger.error("Failed to copy template [{}] from zone [{}] to zone [{}].",
+                    sourceTmpl.getUniqueName(),
+                    sourceTmpl.getDataStore().getScope().getScopeId(),
+                    dstZoneId,
+                    e);
+            return false;
+        }
+
+        return result.isSuccess();
+    }
+
     @Override
     public AsyncCallFuture<TemplateApiResult> copyTemplateToImageStore(DataObject source, DataStore destStore) {
         TemplateObject sourceTmpl = (TemplateObject) source;
@@ -701,10 +818,6 @@ public class TemplateServiceImpl implements TemplateService {
         return null;
     }
 
-    protected boolean isCopyFromOtherStoragesEnabled(Long zoneId) {
-        return StorageManager.COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES.valueIn(zoneId);
-    }
-
     protected void publishTemplateCreation(TemplateInfo tmplt) {
         VMTemplateVO tmpltVo = _templateDao.findById(tmplt.getId());
 

engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java

@@ -18,13 +18,20 @@
  */
 package org.apache.cloudstack.storage.image;
 
+import com.cloud.dc.DataCenterVO;
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.StorageUnavailableException;
+import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.template.TemplateProp;
 import com.cloud.template.TemplateManager;
+import com.cloud.user.Account;
+import com.cloud.user.User;
+import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.Scope;
-import org.apache.cloudstack.framework.async.AsyncCallFuture;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.image.store.TemplateObject;
@@ -46,6 +53,8 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import static org.mockito.Mockito.mock;
+
 @RunWith(MockitoJUnitRunner.class)
 public class TemplateServiceImplTest {
 
@@ -89,6 +98,12 @@ public class TemplateServiceImplTest {
     @Mock
     TemplateManager templateManagerMock;
 
+    @Mock
+    VMTemplateDao templateDao;
+
+    @Mock
+    DataCenterDao _dcDao;
+
     Map<String, TemplateProp> templatesInSourceStore = new HashMap<>();
 
     @Before
@@ -101,7 +116,6 @@ public class TemplateServiceImplTest {
         Mockito.doReturn(List.of(sourceStoreMock, destStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(zoneId);
         Mockito.doReturn(templatesInSourceStore).when(templateService).listTemplate(sourceStoreMock);
         Mockito.doReturn(null).when(templateService).listTemplate(destStoreMock);
-        Mockito.doReturn("install-path").when(templateInfoMock).getInstallPath();
         Mockito.doReturn(templateInfoMock).when(templateDataFactoryMock).getTemplate(2L, sourceStoreMock);
         Mockito.doReturn(3L).when(dataStoreMock).getId();
         Mockito.doReturn(zoneScopeMock).when(dataStoreMock).getScope();
@@ -166,7 +180,7 @@ public class TemplateServiceImplTest {
         boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
 
         Assert.assertFalse(result);
-        Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
+        Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any());
     }
 
     @Test
@@ -174,20 +188,161 @@ public class TemplateServiceImplTest {
         templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock);
         Mockito.doReturn(null).when(templateInfoMock).getInstallPath();
 
+        Scope scopeMock = Mockito.mock(Scope.class);
+        Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
+        Mockito.doReturn(1L).when(scopeMock).getScopeId();
+        Mockito.doReturn(List.of(1L)).when(_dcDao).listAllIds();
+
         boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
 
         Assert.assertFalse(result);
-        Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
+        Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any());
     }
 
     @Test
-    public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherStorageAndTaskWasScheduled() {
-        templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock);
-        Mockito.doReturn(new AsyncCallFuture<>()).when(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
+    public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherZone() throws StorageUnavailableException, ResourceAllocationException {
+        Scope scopeMock = Mockito.mock(Scope.class);
+        Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
+        Mockito.doReturn(1L).when(scopeMock).getScopeId();
+        Mockito.doReturn(100L).when(tmpltMock).getId();
+        Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName();
+        Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(1L);
+        Mockito.doReturn(null).when(templateService).listTemplate(sourceStoreMock);
+        Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
+
+        DataStore otherZoneStoreMock = Mockito.mock(DataStore.class);
+        Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L);
+
+        Map<String, TemplateProp> templatesInOtherZone = new HashMap<>();
+        templatesInOtherZone.put("unique-name", tmpltPropMock);
+        Mockito.doReturn(templatesInOtherZone).when(templateService).listTemplate(otherZoneStoreMock);
+
+        TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class);
+        Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock);
+        Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath();
+
+        DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class);
+        Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L);
+        Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any());
 
         boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
 
         Assert.assertTrue(result);
-        Mockito.verify(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any());
+    }
+
+    @Test
+    public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenDestinationZoneIsMissing() {
+        Scope scopeMock = Mockito.mock(Scope.class);
+        Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
+        Mockito.doReturn(1L).when(scopeMock).getScopeId();
+        Mockito.doReturn(100L).when(tmpltMock).getId();
+        Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName();
+        Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
+        Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L);
+
+        DataStore otherZoneStoreMock = Mockito.mock(DataStore.class);
+        Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L);
+
+        Map<String, TemplateProp> templates = new HashMap<>();
+        templates.put("unique-name", tmpltPropMock);
+        Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock);
+
+        TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class);
+        Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock);
+        Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath();
+        Mockito.doReturn(null).when(_dcDao).findById(1L);
+
+        boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
+
+        Assert.assertFalse(result);
+    }
+
+    @Test
+    public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenCrossZoneCopyTaskIsScheduled() throws StorageUnavailableException, ResourceAllocationException {
+        Scope scopeMock = Mockito.mock(Scope.class);
+        Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
+        Mockito.doReturn(1L).when(scopeMock).getScopeId();
+        Mockito.doReturn(100L).when(tmpltMock).getId();
+        Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName();
+        Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
+        Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L);
+
+        DataStore otherZoneStoreMock = Mockito.mock(DataStore.class);
+        Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L);
+
+        Map<String, TemplateProp> templates = new HashMap<>();
+        templates.put("unique-name", tmpltPropMock);
+        Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock);
+
+        TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class);
+        Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock);
+        Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath();
+        Mockito.doReturn(100L).when(sourceTmplMock).getId();
+
+        DataStore sourceStoreMock = Mockito.mock(DataStore.class);
+        Scope sourceScopeMock = Mockito.mock(Scope.class);
+        Mockito.doReturn(sourceStoreMock).when(sourceTmplMock).getDataStore();
+
+        DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class);
+        Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L);
+        VMTemplateVO templateVoMock = Mockito.mock(VMTemplateVO.class);
+        Mockito.doReturn(templateVoMock).when(templateDao).findById(100L);
+
+        Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any());
+
+        Account account = mock(Account.class);
+        User user = mock(User.class);
+        CallContext callContext = mock(CallContext.class);
+
+        boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
+
+        Assert.assertTrue(result);
+    }
+
+    @Test
+    public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenTemplateNotFoundInAnyZone() {
+        Scope scopeMock = Mockito.mock(Scope.class);
+        Mockito.doReturn(scopeMock).when(destStoreMock).getScope();
+        Mockito.doReturn(1L).when(scopeMock).getScopeId();
+        Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds();
+        Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(Mockito.anyLong());
+        Mockito.doReturn(null).when(templateService).listTemplate(Mockito.any());
+
+        boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock);
+
+        Assert.assertFalse(result);
+    }
+
+    @Test
+    public void testFindUsableTemplateReturnsTemplateWithNonNullInstallPath() {
+        VMTemplateVO template = Mockito.mock(VMTemplateVO.class);
+        Mockito.when(template.getId()).thenReturn(10L);
+        Mockito.when(template.getUniqueName()).thenReturn("test-template");
+
+        DataStore storeWithNullPath = Mockito.mock(DataStore.class);
+        Mockito.when(storeWithNullPath.getName()).thenReturn("store-null");
+
+        DataStore storeWithValidPath = Mockito.mock(DataStore.class);
+        TemplateObject tmplWithNullPath = Mockito.mock(TemplateObject.class);
+        Mockito.when(tmplWithNullPath.getInstallPath()).thenReturn(null);
+
+        TemplateObject tmplWithValidPath = Mockito.mock(TemplateObject.class);
+        Mockito.when(tmplWithValidPath.getInstallPath()).thenReturn("/mnt/secondary/template.qcow2");
+
+        Mockito.doReturn(tmplWithNullPath).when(templateDataFactoryMock).getTemplate(10L, storeWithNullPath);
+        Mockito.doReturn(tmplWithValidPath).when(templateDataFactoryMock).getTemplate(10L, storeWithValidPath);
+
+        Map<String, TemplateProp> templates = new HashMap<>();
+        templates.put("test-template", Mockito.mock(TemplateProp.class));
+
+        Mockito.doReturn(templates).when(templateService).listTemplate(storeWithNullPath);
+        Mockito.doReturn(templates).when(templateService).listTemplate(storeWithValidPath);
+
+        List<DataStore> imageStores = List.of(storeWithNullPath, storeWithValidPath);
+
+        TemplateObject result = templateService.findUsableTemplate(template, imageStores);
+
+        Assert.assertNotNull(result);
+        Assert.assertEquals(tmplWithValidPath, result);
     }
 }

engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java

@@ -387,6 +387,7 @@ public class VolumeServiceImpl implements VolumeService {
             logger.info("Expunge volume with no data store specified");
             if (canVolumeBeRemoved(volume.getId())) {
                 logger.info("Volume {} is not referred anywhere, remove it from volumes table", volume);
+                snapshotMgr.deletePoliciesForVolume(volume.getId());
                 volDao.remove(volume.getId());
             }
             future.complete(result);
@@ -422,6 +423,7 @@ public class VolumeServiceImpl implements VolumeService {
                 }
                 VMTemplateVO template = templateDao.findById(vol.getTemplateId());
                 if (template != null && !template.isDeployAsIs()) {
+                    snapshotMgr.deletePoliciesForVolume(vol.getId());
                     volDao.remove(vol.getId());
                     future.complete(result);
                     return future;
@@ -493,6 +495,7 @@ public class VolumeServiceImpl implements VolumeService {
 
                 if (canVolumeBeRemoved(vo.getId())) {
                     logger.info("Volume {} is not referred anywhere, remove it from volumes table", vo);
+                    snapshotMgr.deletePoliciesForVolume(vo.getId());
                     volDao.remove(vo.getId());
                 }
 
@@ -1656,7 +1659,6 @@ public class VolumeServiceImpl implements VolumeService {
         // mark volume entry in volumes table as destroy state
         VolumeInfo vol = volFactory.getVolume(volumeId);
         vol.stateTransit(Volume.Event.DestroyRequested);
-        snapshotMgr.deletePoliciesForVolume(volumeId);
         annotationDao.removeByEntityType(AnnotationService.EntityType.VOLUME.name(), vol.getUuid());
 
         vol.stateTransit(Volume.Event.OperationSucceeded);

framework/db/src/main/java/com/cloud/utils/db/Filter.java

@@ -57,7 +57,8 @@ public class Filter {
     }
 
     public Filter(long limit) {
-        _orderBy = " ORDER BY RAND() LIMIT " + limit;
+        _orderBy = " ORDER BY RAND()";
+        _limit = limit;
     }
 
     public Filter(Long offset, Long limit) {

framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java

@@ -1172,6 +1172,8 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
                 if (filter.getLimit() != null) {
                     sql.append(", ").append(filter.getLimit());
                 }
+            } else if (filter.getLimit() != null) {
+                sql.append(" LIMIT ").append(filter.getLimit());
             }
         }
     }
@@ -1333,7 +1335,7 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
         Filter filter = null;
         final long batchSizeFinal = ObjectUtils.defaultIfNull(batchSize, 0L);
         if (batchSizeFinal > 0) {
-            filter = new Filter(batchSizeFinal);
+            filter = new Filter(null, batchSizeFinal);
         }
         int expunged = 0;
         int currentExpunged = 0;

plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java

@@ -364,7 +364,9 @@ public class VeeamClient {
      * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}.
      */
     protected void checkIfRestoreSessionFinished(String type, String path) throws IOException {
-        for (int j = 0; j < restoreTimeout; j++) {
+        long startTime = System.currentTimeMillis();
+        long timeoutMs = restoreTimeout * 1000L;
+        while (System.currentTimeMillis() - startTime < timeoutMs) {
             HttpResponse relatedResponse = get(path);
             RestoreSession session = parseRestoreSessionResponse(relatedResponse);
             if (session.getResult().equals("Success")) {
@@ -378,7 +380,8 @@ public class VeeamClient {
                         getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":"))));
                 throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid));
             }
-            logger.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout));
+            logger.debug("Waiting {} seconds, out of a total of {} seconds, for the restore backup process to finish.",
+                    (System.currentTimeMillis() - startTime) / 1000, restoreTimeout);
 
             try {
                 Thread.sleep(1000);

plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java

@@ -25,7 +25,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
 import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching;
 import static com.github.tomakehurst.wiremock.client.WireMock.verify;
 import static org.junit.Assert.fail;
-import static org.mockito.Mockito.times;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -157,7 +156,7 @@ public class VeeamClientTest {
     @Test
     public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOException {
         try {
-            ReflectionTestUtils.setField(mockClient, "restoreTimeout", 10);
+            ReflectionTestUtils.setField(mockClient, "restoreTimeout", 2);
             RestoreSession restoreSession = Mockito.mock(RestoreSession.class);
             HttpResponse httpResponse = Mockito.mock(HttpResponse.class);
             Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse);
@@ -169,7 +168,7 @@ public class VeeamClientTest {
         } catch (Exception e) {
             Assert.assertEquals("Related job type: RestoreTest was not successful", e.getMessage());
         }
-        Mockito.verify(mockClient, times(10)).get(Mockito.anyString());
+        Mockito.verify(mockClient, Mockito.atLeastOnce()).get(Mockito.anyString());
     }
 
     @Test

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateHostPasswordCommandWrapper.java

@@ -37,7 +37,8 @@ public final class LibvirtUpdateHostPasswordCommandWrapper extends CommandWrappe
         final String newPassword = command.getNewPassword();
 
         final Script script = libvirtUtilitiesHelper.buildScript(libvirtComputingResource.getUpdateHostPasswdPath());
-        script.add(username, newPassword);
+        script.add(username);
+        script.addSensitive(newPassword);
         final String result = script.execute();
 
         if (result != null) {

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java

@@ -47,7 +47,7 @@ public final class CitrixUpdateHostPasswordCommandWrapper extends CommandWrapper
         try {
             logger.debug("Executing password update command on host: {} for user: {}", hostIp, username);
             final String hostPassword = citrixResourceBase.getPwdFromQueue();
-            result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine.toString());
+            result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine);
         } catch (final Exception e) {
             return new Answer(command, false, e.getMessage());
         }

server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java

@@ -109,6 +109,8 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation<Snapsh
             if (showChainSize && snapshotInfo.getParent() != null) {
                 long chainSize = calculateChainSize(snapshotInfo);
                 snapshotResponse.setChainSize(chainSize);
+                snapshotResponse.setParent(snapshotInfo.getParent().getUuid());
+                snapshotResponse.setParentName(snapshotInfo.getParent().getName());
             }
         }
     }

server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -541,7 +541,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
     public static final ConfigKey<Long> DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0",
             "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," +
                     " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " +
-                    "be applied. Default value is 0. For now, this is used for deletion of vm & volume stats only.", true);
+                    "be applied. Default value is 0. For now, this is used for deletion of VM stats, volume stats, and usage records.", true);
 
     private static final String IOPS_READ_RATE = "IOPS Read";
     private static final String IOPS_WRITE_RATE = "IOPS Write";

server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java

@@ -37,6 +37,7 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
 import com.cloud.gpu.dao.VgpuProfileDao;
+import com.cloud.resource.ResourceState;
 import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO;
 import org.apache.cloudstack.affinity.AffinityGroupProcessor;
 import org.apache.cloudstack.affinity.AffinityGroupService;
@@ -383,22 +384,12 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
             planner = getDeploymentPlannerByName(plannerName);
         }
 
-        Host lastHost = null;
-
-        String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost);
-        boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null &&
-                (considerLastHostStr == null || Boolean.TRUE.toString().equalsIgnoreCase(considerLastHostStr));
-        if (considerLastHost) {
-            HostVO host = _hostDao.findById(vm.getLastHostId());
-            logger.debug("This VM has last host_id specified, trying to choose the same host: " + host);
-            lastHost = host;
-
-            DeployDestination deployDestination = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption);
-            if (deployDestination != null) {
-                return deployDestination;
-            }
+        DeployDestination deployDestinationForVmLasthost = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption);
+        if (deployDestinationForVmLasthost != null) {
+            return deployDestinationForVmLasthost;
         }
 
+        HostVO lastHost = _hostDao.findById(vm.getLastHostId());
         avoidOtherClustersForDeploymentIfMigrationDisabled(vm, lastHost, avoids);
 
         DeployDestination dest = null;
@@ -480,47 +471,56 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
     private DeployDestination deployInVmLastHost(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoids,
             DeploymentPlanner planner, VirtualMachine vm, DataCenter dc, ServiceOffering offering, int cpuRequested, long ramRequested,
             boolean volumesRequireEncryption) throws InsufficientServerCapacityException {
-        HostVO host = _hostDao.findById(vm.getLastHostId());
-        if (canUseLastHost(host, avoids, plan, vm, offering, volumesRequireEncryption)) {
-            _hostDao.loadHostTags(host);
-            _hostDao.loadDetails(host);
-            if (host.getStatus() != Status.Up) {
+        String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost);
+        String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag);
+        boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null &&
+                !(Boolean.FALSE.toString().equalsIgnoreCase(considerLastHostStr));
+        if (!considerLastHost) {
+            return null;
+        }
+
+        logger.debug("This VM has last host_id: {}", vm.getLastHostId());
+        HostVO lastHost = _hostDao.findById(vm.getLastHostId());
+        if (canUseLastHost(lastHost, avoids, plan, vm, offering, volumesRequireEncryption)) {
+            _hostDao.loadHostTags(lastHost);
+            _hostDao.loadDetails(lastHost);
+            if (lastHost.getStatus() != Status.Up) {
                 logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host is not in UP state or is not enabled. Host current status [{}] and resource status [{}].",
-                        vm, host, host.getState().name(), host.getResourceState());
+                        vm, lastHost, lastHost.getState().name(), lastHost.getResourceState());
                 return null;
             }
-            if (checkVmProfileAndHost(vmProfile, host)) {
-                long cluster_id = host.getClusterId();
+            if (checkVmProfileAndHost(vmProfile, lastHost)) {
+                long cluster_id = lastHost.getClusterId();
                 ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio");
                 ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio");
                 float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue());
                 float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue());
 
                 boolean hostHasCpuCapability, hostHasCapacity = false;
-                hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed());
+                hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(lastHost, offering.getCpu(), offering.getSpeed());
 
                 if (hostHasCpuCapability) {
                     // first check from reserved capacity
-                    hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true);
+                    hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true);
 
                     // if not reserved, check the free capacity
                     if (!hostHasCapacity)
-                        hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true);
+                        hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true);
                     }
 
                 boolean displayStorage = getDisplayStorageFromVmProfile(vmProfile);
                 if (!hostHasCapacity || !hostHasCpuCapability) {
-                    logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, host);
+                    logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, lastHost);
                     return null;
                 }
-                Pod pod = _podDao.findById(host.getPodId());
-                Cluster cluster = _clusterDao.findById(host.getClusterId());
+                Pod pod = _podDao.findById(lastHost.getPodId());
+                Cluster cluster = _clusterDao.findById(lastHost.getClusterId());
 
                 logger.debug("Last host [{}] of VM [{}] is UP and has enough capacity. Checking for suitable pools for this host under zone [{}], pod [{}] and cluster [{}].",
-                        host, vm, dc, pod, cluster);
+                        lastHost, vm, dc, pod, cluster);
 
-                if (DEPLOYMENT_PLANNING_SKIP_HYPERVISORS.contains(vm.getHypervisorType())) {
-                    DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap<>(), displayStorage);
+                if (vm.getHypervisorType() == HypervisorType.BareMetal) {
+                    DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, new HashMap<>(), displayStorage);
                     logger.debug("Returning Deployment Destination: {}.", dest);
                     return dest;
                 }
@@ -528,8 +528,8 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
                 // search for storage under the zone, pod, cluster
                 // of
                 // the last host.
-                DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(),
-                        host.getPodId(), host.getClusterId(), host.getId(), plan.getPoolId(), null);
+                DataCenterDeployment lastPlan = new DataCenterDeployment(lastHost.getDataCenterId(),
+                        lastHost.getPodId(), lastHost.getClusterId(), lastHost.getId(), plan.getPoolId(), null);
                 Pair<Map<Volume, List<StoragePool>>, List<Volume>> result = findSuitablePoolsForVolumes(
                         vmProfile, lastPlan, avoids, HostAllocator.RETURN_UPTO_ALL);
                 Map<Volume, List<StoragePool>> suitableVolumeStoragePools = result.first();
@@ -538,11 +538,11 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
                 // choose the potential pool for this VM for this
                 // host
                 if (suitableVolumeStoragePools.isEmpty()) {
-                    logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", host, vm);
+                    logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", lastHost, vm);
                     return null;
                 }
                 List<Host> suitableHosts = new ArrayList<>();
-                suitableHosts.add(host);
+                suitableHosts.add(lastHost);
                 Pair<Host, Map<Volume, StoragePool>> potentialResources = findPotentialDeploymentResources(
                         suitableHosts, suitableVolumeStoragePools, avoids,
                         getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts(), vm);
@@ -555,7 +555,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
                     for (Volume vol : readyAndReusedVolumes) {
                         storageVolMap.remove(vol);
                     }
-                    DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap, displayStorage);
+                    DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, storageVolMap, displayStorage);
                     logger.debug("Returning Deployment Destination: {}", dest);
                     return dest;
                 }
@@ -567,7 +567,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
 
     private boolean canUseLastHost(HostVO host, ExcludeList avoids, DeploymentPlan plan, VirtualMachine vm, ServiceOffering offering, boolean volumesRequireEncryption) {
         if (host == null) {
-            logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping this and trying other available hosts.", vm, vm.getLastHostId());
+            logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping it", vm, vm.getLastHostId());
             return false;
         }
 
@@ -581,6 +581,12 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
             return false;
         }
 
+        logger.debug("VM's last host is {}, trying to choose the same host if it is not in maintenance, error or degraded state", host);
+        if (host.isInMaintenanceStates() || Arrays.asList(ResourceState.Error, ResourceState.Degraded).contains(host.getResourceState())) {
+            logger.debug("Unable to deploy VM {} in the last host, last host {} is in {} state", vm.getName(), host.getName(), host.getResourceState());
+            return false;
+        }
+
         if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) {
             logger.debug("Cannot deploy VM [{}] in the last host [{}] because this host already has the max number of running VMs (users and system VMs). Skipping this and trying other available hosts.",
                     vm, host);
@@ -1477,7 +1483,7 @@ StateListener<State, VirtualMachine.Event, VirtualMachine>, Configurable {
 
     protected Pair<Host, Map<Volume, StoragePool>> findPotentialDeploymentResources(List<Host> suitableHosts, Map<Volume, List<StoragePool>> suitableVolumeStoragePools,
                                                                                     ExcludeList avoid, PlannerResourceUsage resourceUsageRequired, List<Volume> readyAndReusedVolumes, List<Long> preferredHosts, VirtualMachine vm) {
-        logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM");
+        logger.debug("Trying to find a potential host and associated storage pools from the suitable host/pool lists for this VM");
 
         boolean hostCanAccessPool = false;
         boolean haveEnoughSpace = false;

server/src/main/java/com/cloud/event/AlertGenerator.java

@@ -67,12 +67,13 @@ public class AlertGenerator {
     }
 
     public static void publishAlertOnEventBus(String alertType, long dataCenterId, Long podId, String subject, String body) {
-
         String configKey = Config.PublishAlertEvent.key();
         String value = s_configDao.getValue(configKey);
         boolean configValue = Boolean.parseBoolean(value);
-        if(!configValue)
+        if (!configValue) {
             return;
+        }
+
         try {
             eventDistributor = ComponentContext.getComponent(EventDistributor.class);
         } catch (NoSuchBeanDefinitionException nbe) {

server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java

@@ -837,7 +837,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
         if (checkAndCancelWorkIfNeeded(work)) {
             return null;
         }
-        logger.info("Migration attempt: for VM {}from host {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries);
+        logger.info("Migration attempt: for {} from {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries);
 
         if (VirtualMachine.State.Stopped.equals(vm.getState())) {
             logger.info(String.format("vm %s is Stopped, skipping migrate.", vm));
@@ -847,8 +847,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur
             logger.info(String.format("VM %s is running on a different host %s, skipping migration", vm, vm.getHostId()));
             return null;
         }
-        logger.info("Migration attempt: for VM " + vm.getUuid() + "from host id " + srcHostId +
-                ". Starting attempt: " + (1 + work.getTimesTried()) + "/" + _maxRetries + " times.");
 
         try {
             work.setStep(Step.Migrating);

server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java

@@ -26,15 +26,11 @@ import java.util.Objects;
 
 import javax.inject.Inject;
 
-import com.cloud.dc.DataCenter;
-import com.cloud.exception.PermissionDeniedException;
-import com.cloud.network.dao.NetrisProviderDao;
-import com.cloud.network.dao.NsxProviderDao;
-import com.cloud.network.element.NetrisProviderVO;
-import com.cloud.network.element.NsxProviderVO;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiErrorCode;
 import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.ImportNetworkACLCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkACLListsCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
 import org.apache.cloudstack.api.command.user.network.MoveNetworkAclItemCmd;
@@ -43,19 +39,26 @@ import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLListCmd;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
+import com.cloud.dc.DataCenter;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.EventTypes;
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.Network;
 import com.cloud.network.NetworkModel;
 import com.cloud.network.Networks;
+import com.cloud.network.dao.NetrisProviderDao;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.dao.NetworkVO;
+import com.cloud.network.dao.NsxProviderDao;
+import com.cloud.network.element.NetrisProviderVO;
+import com.cloud.network.element.NsxProviderVO;
 import com.cloud.network.vpc.NetworkACLItem.Action;
 import com.cloud.network.vpc.NetworkACLItem.TrafficType;
 import com.cloud.network.vpc.dao.NetworkACLDao;
@@ -1070,6 +1073,111 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ
         return moveRuleToTheTop(ruleBeingMoved, allRules);
     }
 
+    @Override
+    public List<NetworkACLItem> importNetworkACLRules(ImportNetworkACLCmd cmd) throws ResourceUnavailableException {
+        long aclId = cmd.getAclId();
+        Map<Object, Object> rules = cmd.getRules();
+        List<NetworkACLItem> createdRules = new ArrayList<>();
+        List<String> errors = new ArrayList<>();
+        for (Map.Entry<Object, Object> entry : rules.entrySet()) {
+            try {
+                Map<String, Object> ruleMap = (Map<String, Object>) entry.getValue();
+                NetworkACLItem item = createACLRuleFromMap(ruleMap, aclId);
+                createdRules.add(item);
+            } catch (Exception ex) {
+                String error = "Failed to import rule at index " + entry.getKey() + ": " + ex.getMessage();
+                errors.add(error);
+                logger.error(error, ex);
+            }
+        }
+        // no rules got imported
+        if (createdRules.isEmpty() && !errors.isEmpty()) {
+            logger.error("Failed to import any ACL rules. Errors: {}", String.join("; ", errors));
+            throw new CloudRuntimeException("Failed to import any ACL rules.");
+        }
+
+        // apply ACL to network
+        if (!createdRules.isEmpty()) {
+            applyNetworkACL(aclId);
+        }
+        return createdRules;
+    }
+
+    private NetworkACLItem createACLRuleFromMap(Map<String, Object> ruleMap, long aclId) {
+        String protocol = (String) ruleMap.get(ApiConstants.PROTOCOL);
+        if (protocol == null || protocol.trim().isEmpty()) {
+            throw new InvalidParameterValueException("Protocol is required");
+        }
+        String action = (String) ruleMap.getOrDefault(ApiConstants.ACTION, "deny");
+        String trafficType = (String) ruleMap.getOrDefault(ApiConstants.TRAFFIC_TYPE, NetworkACLItem.TrafficType.Ingress);
+        String forDisplay = (String) ruleMap.getOrDefault(ApiConstants.FOR_DISPLAY, "true");
+
+        // Create ACL rule using the service
+        CreateNetworkACLCmd cmd = new CreateNetworkACLCmd();
+        cmd.setAclId(aclId);
+        cmd.setProtocol(protocol.toLowerCase());
+        cmd.setAction(action.toLowerCase());
+        cmd.setTrafficType(trafficType.toLowerCase());
+        cmd.setDisplay(BooleanUtils.toBoolean(forDisplay));
+
+        // Optional parameters
+        if (ruleMap.containsKey(ApiConstants.CIDR_LIST)) {
+            Object cidrObj = ruleMap.get(ApiConstants.CIDR_LIST);
+            List<String> cidrList = new ArrayList<>();
+            if (cidrObj instanceof String) {
+                for (String cidr : ((String) cidrObj).split(",")) {
+                    cidrList.add(cidr.trim());
+                }
+            } else if (cidrObj instanceof List) {
+                cidrList.addAll((List<String>) cidrObj);
+            }
+            cmd.setCidrList(cidrList);
+        }
+
+        if (ruleMap.containsKey(ApiConstants.START_PORT)) {
+            cmd.setPublicStartPort(parseInt(ruleMap.get(ApiConstants.START_PORT)));
+        }
+
+        if (ruleMap.containsKey(ApiConstants.END_PORT)) {
+            cmd.setPublicEndPort(parseInt(ruleMap.get(ApiConstants.END_PORT)));
+        }
+
+        if (ruleMap.containsKey(ApiConstants.NUMBER)) {
+            cmd.setNumber(parseInt(ruleMap.get(ApiConstants.NUMBER)));
+        }
+
+        if (ruleMap.containsKey(ApiConstants.ICMP_TYPE)) {
+            cmd.setIcmpType(parseInt(ruleMap.get(ApiConstants.ICMP_TYPE)));
+        }
+
+        if (ruleMap.containsKey(ApiConstants.ICMP_CODE)) {
+            cmd.setIcmpCode(parseInt(ruleMap.get(ApiConstants.ICMP_CODE)));
+        }
+
+        if (ruleMap.containsKey(ApiConstants.ACL_REASON)) {
+            cmd.setReason((String) ruleMap.get(ApiConstants.ACL_REASON));
+        }
+
+        return createNetworkACLItem(cmd);
+    }
+
+    private Integer parseInt(Object value) {
+        if (value == null) {
+            return null;
+        }
+        if (value instanceof Integer) {
+            return (Integer) value;
+        }
+        if (value instanceof String) {
+            try {
+                return Integer.parseInt((String) value);
+            } catch (NumberFormatException e) {
+                throw new InvalidParameterValueException("Invalid integer value: " + value);
+            }
+        }
+        throw new InvalidParameterValueException("Cannot convert to integer: " + value);
+    }
+
     /**
      * Validates the consistency of the ACL; the validation process is the following.
      * <ul>

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -1563,7 +1563,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
                         throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage.");
                     }
                 } else {
-                    logger.info("Maintenance: scheduling migration of VM {} from host {}", vm, host);
+                    logger.info("Maintenance: scheduling migration of {} from {}", vm, host);
                     _haMgr.scheduleMigration(vm, HighAvailabilityManager.ReasonType.HostMaintenance);
                 }
             }
@@ -3833,8 +3833,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         if (!isAgentOnHost || vmsMigrating || host.getStatus() == Status.Up) {
             return;
         }
-        final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(KvmSshToAgentEnabled.key()));
-        if (sshToAgent) {
+        if (KvmSshToAgentEnabled.value()) {
             Ternary<String, String, String> credentials = getHostCredentials(host);
             connectAndRestartAgentOnHost(host, credentials.first(), credentials.second(), credentials.third());
         } else {

server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java

@@ -1222,7 +1222,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
         }
 
         return Transaction.execute((TransactionCallback<Long>) status -> {
-            long newResourceCount = 0L;
             List<Long> domainIdList = childDomains.stream().map(DomainVO::getId).collect(Collectors.toList());
             domainIdList.add(domainId);
             List<Long> accountIdList = accounts.stream().map(AccountVO::getId).collect(Collectors.toList());
@@ -1240,6 +1239,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
             List<ResourceCountVO> resourceCounts = _resourceCountDao.lockRows(rowIdsToLock);
 
             long oldResourceCount = 0L;
+            long newResourceCount = 0L;
             ResourceCountVO domainRC = null;
 
             // calculate project count here
@@ -1261,7 +1261,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
             if (oldResourceCount != newResourceCount) {
                 domainRC.setCount(newResourceCount);
                 _resourceCountDao.update(domainRC.getId(), domainRC);
-                logger.warn("Discrepency in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.",
+                logger.warn("Discrepancy in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.",
                         oldResourceCount, newResourceCount, type, domainId);
             }
             return newResourceCount;
@@ -1524,16 +1524,17 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
     }
 
     protected long calculatePrimaryStorageForAccount(long accountId, String tag) {
+        long snapshotsPhysicalSizeOnPrimaryStorage = _snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId);
         if (StringUtils.isEmpty(tag)) {
             List<Long> virtualRouters = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId);
-            return _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters);
+            return snapshotsPhysicalSizeOnPrimaryStorage + _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters);
         }
         long storage = 0;
         List<VolumeVO> volumes = getVolumesWithAccountAndTag(accountId, tag);
         for (VolumeVO volume : volumes) {
             storage += volume.getSize() == null ? 0L : volume.getSize();
         }
-        return storage;
+        return snapshotsPhysicalSizeOnPrimaryStorage + storage;
     }
 
     @Override
@@ -2293,7 +2294,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim
 
     protected class ResourceCountCheckTask extends ManagedContextRunnable {
         public ResourceCountCheckTask() {
-
         }
 
         @Override

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -454,6 +454,7 @@ import org.apache.cloudstack.api.command.user.network.CreateNetworkPermissionsCm
 import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd;
 import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLListCmd;
 import org.apache.cloudstack.api.command.user.network.DeleteNetworkCmd;
+import org.apache.cloudstack.api.command.user.network.ImportNetworkACLCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkACLListsCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
 import org.apache.cloudstack.api.command.user.network.ListNetworkOfferingsCmd;
@@ -4041,6 +4042,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         cmdList.add(EnableStaticNatCmd.class);
         cmdList.add(ListIpForwardingRulesCmd.class);
         cmdList.add(CreateNetworkACLCmd.class);
+        cmdList.add(ImportNetworkACLCmd.class);
         cmdList.add(CreateNetworkCmd.class);
         cmdList.add(DeleteNetworkACLCmd.class);
         cmdList.add(DeleteNetworkCmd.class);
@@ -4809,6 +4811,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         final long diskOffMinSize = VolumeOrchestrationService.CustomDiskOfferingMinSize.value();
         final long diskOffMaxSize = VolumeOrchestrationService.CustomDiskOfferingMaxSize.value();
         final boolean KVMSnapshotEnabled = SnapshotManager.KVMSnapshotEnabled.value();
+        final boolean SnapshotShowChainSize = SnapshotManager.snapshotShowChainSize.value();
 
         final boolean userPublicTemplateEnabled = TemplateManager.AllowPublicUserTemplates.valueIn(caller.getId());
 
@@ -4849,6 +4852,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
         capabilities.put("customDiskOffMaxSize", diskOffMaxSize);
         capabilities.put("regionSecondaryEnabled", regionSecondaryEnabled);
         capabilities.put("KVMSnapshotEnabled", KVMSnapshotEnabled);
+        capabilities.put("SnapshotShowChainSize", SnapshotShowChainSize);
         capabilities.put("allowUserViewDestroyedVM", allowUserViewDestroyedVM);
         capabilities.put("allowUserExpungeRecoverVM", allowUserExpungeRecoverVM);
         capabilities.put("allowUserExpungeRecoverVolume", allowUserExpungeRecoverVolume);

server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java

@@ -78,4 +78,15 @@ public class ImageStoreDetailsUtil {
         return getGlobalDefaultNfsVersion();
     }
 
+    public boolean isCopyTemplatesFromOtherStoragesEnabled(Long storeId, Long zoneId) {
+        final Map<String, String> storeDetails = imageStoreDetailsDao.getDetails(storeId);
+        final String keyWithoutDots = StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.key()
+                .replace(".", "");
+
+        if (storeDetails != null && storeDetails.containsKey(keyWithoutDots)) {
+            return Boolean.parseBoolean(storeDetails.get(keyWithoutDots));
+        }
+
+        return StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.valueIn(zoneId);
+    }
 }

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -4616,7 +4616,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                 AllowVolumeReSizeBeyondAllocation,
                 StoragePoolHostConnectWorkers,
                 ObjectStorageCapacityThreshold,
-                COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES
+                COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES
         };
     }
 

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -290,6 +290,15 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         return !DataCenter.Type.Edge.equals(zone.getType());
     }
 
+    private ResourceType getStoreResourceType(long dataCenterId, Snapshot.LocationType locationType) {
+        ResourceType storeResourceType = ResourceType.secondary_storage;
+        if (!isBackupSnapshotToSecondaryForZone(dataCenterId) ||
+                Snapshot.LocationType.PRIMARY.equals(locationType)) {
+            storeResourceType = ResourceType.primary_storage;
+        }
+        return storeResourceType;
+    }
+
     @Override
     public String getConfigComponentName() {
         return SnapshotManager.class.getSimpleName();
@@ -676,7 +685,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         _snapshotDao.update(snapshot.getId(), snapshot);
         snapshotInfo = this.snapshotFactory.getSnapshot(snapshotId, store);
 
-        Long snapshotOwnerId = vm.getAccountId();
+        long snapshotOwnerId = vm.getAccountId();
 
         try {
             SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.BACKUP);
@@ -684,7 +693,6 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
                 throw new CloudRuntimeException(String.format("Unable to find Snapshot strategy to handle Snapshot [%s]", snapshot));
             }
             snapshotInfo = snapshotStrategy.backupSnapshot(snapshotInfo);
-
         } catch (Exception e) {
             logger.debug("Failed to backup Snapshot from Instance Snapshot", e);
             _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.snapshot);
@@ -894,12 +902,11 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         _accountMgr.checkAccess(caller, null, true, snapshotCheck);
 
         SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshotCheck, zoneId, SnapshotOperation.DELETE);
-
         if (snapshotStrategy == null) {
             logger.error("Unable to find snapshot strategy to handle snapshot [{}]", snapshotCheck);
-
             return false;
         }
+
         Pair<List<SnapshotDataStoreVO>, List<Long>> storeRefAndZones = getStoreRefsAndZonesForSnapshotDelete(snapshotId, zoneId);
         List<SnapshotDataStoreVO> snapshotStoreRefs = storeRefAndZones.first();
         List<Long> zoneIds = storeRefAndZones.second();
@@ -1689,8 +1696,9 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
                 UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null,
                         snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid());
 
+                ResourceType storeResourceType = dataStoreRole == DataStoreRole.Image ? ResourceType.secondary_storage : ResourceType.primary_storage;
                 // Correct the resource count of snapshot in case of delta snapshots.
-                _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize()));
+                _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize()));
 
                 if (!payload.getAsyncBackup()) {
                     if (backupSnapToSecondary) {
@@ -1707,15 +1715,17 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
             if (logger.isDebugEnabled()) {
                 logger.debug("Failed to create snapshot" + cre.getLocalizedMessage());
             }
+            ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType());
             _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot);
-            _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize()));
+            _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize()));
             throw cre;
         } catch (Exception e) {
             if (logger.isDebugEnabled()) {
                 logger.debug("Failed to create snapshot", e);
             }
+            ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType());
             _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot);
-            _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize()));
+            _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize()));
             throw new CloudRuntimeException("Failed to create snapshot", e);
         }
         return snapshot;
@@ -1890,9 +1900,25 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
                 logger.debug("Failed to delete snapshot in destroying state: {}", snapshotVO);
             }
         }
+        cleanupOrphanSnapshotPolicies();
+
         return true;
     }
 
+    private void cleanupOrphanSnapshotPolicies() {
+        List<SnapshotPolicyVO> policies = _snapshotPolicyDao.listActivePolicies();
+        if (CollectionUtils.isEmpty(policies)) {
+            return;
+        }
+        for (SnapshotPolicyVO policy : policies) {
+            VolumeVO volume = _volsDao.findByIdIncludingRemoved(policy.getVolumeId());
+            if (volume == null || volume.getState() == Volume.State.Expunged) {
+                logger.info("Removing orphan snapshot policy {} for non-existent volume {}", policy.getId(), policy.getVolumeId());
+                deletePolicy(policy.getId());
+            }
+        }
+    }
+
     @Override
     public boolean stop() {
         backupSnapshotExecutor.shutdown();
@@ -1925,7 +1951,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
             if (snapshotPolicyVO == null) {
                 throw new InvalidParameterValueException("Policy id given: " + policy + " does not exist");
             }
-            VolumeVO volume = _volsDao.findById(snapshotPolicyVO.getVolumeId());
+            VolumeVO volume = _volsDao.findByIdIncludingRemoved(snapshotPolicyVO.getVolumeId());
             if (volume == null) {
                 throw new InvalidParameterValueException("Policy id given: " + policy + " does not belong to a valid volume");
             }
@@ -1992,11 +2018,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement
         Type snapshotType = getSnapshotType(policyId);
         Account owner = _accountMgr.getAccount(volume.getAccountId());
 
-        ResourceType storeResourceType = ResourceType.secondary_storage;
-        if (!isBackupSnapshotToSecondaryForZone(volume.getDataCenterId()) ||
-                Snapshot.LocationType.PRIMARY.equals(locationType)) {
-            storeResourceType = ResourceType.primary_storage;
-        }
+        ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), locationType);
         try {
             _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot);
             _resourceLimitMgr.checkResourceLimit(owner, storeResourceType, volume.getSize());

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -848,6 +848,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
         // Copy will just find one eligible image store for the destination zone
         // and copy template there, not propagate to all image stores
         // for that zone
+
+        boolean copied = false;
+
         for (DataStore dstSecStore : dstSecStores) {
             TemplateDataStoreVO dstTmpltStore = _tmplStoreDao.findByStoreTemplate(dstSecStore.getId(), tmpltId);
             if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOADED) {
@@ -862,9 +865,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
                 TemplateApiResult result = future.get();
                 if (result.isFailed()) {
                     logger.debug("Copy Template failed for image store {}: {}", dstSecStore, result.getResult());
+                    _tmplStoreDao.removeByTemplateStore(tmpltId, dstSecStore.getId());
                     continue; // try next image store
                 }
 
+                copied = true;
+
                 _tmpltDao.addTemplateToZone(template, dstZoneId);
 
                 if (account.getId() != Account.ACCOUNT_ID_SYSTEM) {
@@ -892,12 +898,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
                         }
                     }
                 }
+
+                return true;
+
             } catch (Exception ex) {
-                logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore);
+                logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore, ex);
             }
         }
-        return true;
-
+        return copied;
     }
 
     @Override

server/src/main/java/com/cloud/usage/UsageServiceImpl.java

@@ -26,6 +26,7 @@ import java.util.TimeZone;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.configuration.ConfigurationManagerImpl;
 import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd;
 import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd;
 import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd;
@@ -489,7 +490,7 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag
                     }
                 }
             }
-            _usageDao.removeOldUsageRecords(interval);
+            _usageDao.expungeAllOlderThan(interval, ConfigurationManagerImpl.DELETE_QUERY_BATCH_SIZE.value());
         } else {
             throw new InvalidParameterValueException("Invalid interval value. Interval to remove cloud_usage records should be greater than 0");
         }

server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java

@@ -352,7 +352,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager {
         final Filter searchFilter = new Filter(BackupOfferingVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal());
         SearchBuilder<BackupOfferingVO> sb = backupOfferingDao.createSearchBuilder();
         sb.and("zone_id", sb.entity().getZoneId(), SearchCriteria.Op.EQ);
-        sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ);
+        sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE);
 
         CallContext ctx = CallContext.current();
         final Account caller = ctx.getCallingAccount();

server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java

@@ -78,7 +78,9 @@ public interface UserPasswordResetManager {
 
     ConfigKey<String> UserPasswordResetDomainURL = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED,
             String.class, "user.password.reset.mail.domain.url", null,
-            "Domain URL for reset password links sent to the user via email", true,
+            "Domain URL (along with scheme - http:// or https:// and port as applicable) for reset password links sent to the user via email. " +
+                    "If this is not set, CloudStack would determine the domain url based on the first management server from 'host' setting " +
+                    "and http scheme based on the https.enabled flag from server.properties file in the management server.", true,
             ConfigKey.Scope.Global);
 
     void setResetTokenAndSend(UserAccount userAccount);

server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java

@@ -23,6 +23,7 @@ import com.cloud.user.UserVO;
 import com.cloud.user.dao.UserDao;
 import com.cloud.utils.StringUtils;
 import com.cloud.utils.component.ManagerBase;
+import com.cloud.utils.server.ServerProperties;
 import com.github.mustachejava.DefaultMustacheFactory;
 import com.github.mustachejava.Mustache;
 import com.github.mustachejava.MustacheFactory;
@@ -48,6 +49,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
+import static org.apache.cloudstack.config.ApiServiceConfiguration.ManagementServerAddresses;
 import static org.apache.cloudstack.resourcedetail.UserDetailVO.PasswordResetToken;
 import static org.apache.cloudstack.resourcedetail.UserDetailVO.PasswordResetTokenExpiryDate;
 
@@ -68,7 +70,7 @@ public class UserPasswordResetManagerImpl extends ManagerBase implements UserPas
             new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, String.class,
             "user.password.reset.mail.template", "Hello {{username}}!\n" +
             "You have requested to reset your password. Please click the following link to reset your password:\n" +
-            "{{{domainUrl}}}{{{resetLink}}}\n" +
+            "{{{resetLink}}}\n" +
             "If you did not request a password reset, please ignore this email.\n" +
             "\n" +
             "Regards,\n" +
@@ -179,10 +181,26 @@ public class UserPasswordResetManagerImpl extends ManagerBase implements UserPas
         final String email = userAccount.getEmail();
         final String username = userAccount.getUsername();
         final String subject = "Password Reset Request";
-        final String domainUrl = UserPasswordResetDomainURL.value();
+        String domainUrl = UserPasswordResetDomainURL.value();
+        if (StringUtils.isBlank(domainUrl)) {
+            String mgmtServerAddr = ManagementServerAddresses.value().split(",")[0];
+            if (ServerProperties.isHttpsEnabled()) {
+                domainUrl = "https://" + mgmtServerAddr + ":" + ServerProperties.getHttpsPort();
+            } else {
+                domainUrl = "http://" + mgmtServerAddr + ":" + ServerProperties.getHttpPort();
+            }
+        } else if (!domainUrl.startsWith("http://") && !domainUrl.startsWith("https://")) {
+            if (ServerProperties.isHttpsEnabled()) {
+                domainUrl = "https://" + domainUrl;
+            } else {
+                domainUrl = "http://" + domainUrl;
+            }
+        }
 
-        String resetLink = String.format("/client/#/user/resetPassword?username=%s&token=%s",
-                username, resetToken);
+        domainUrl = domainUrl.replaceAll("/+$", "");
+
+        String resetLink = String.format("%s/client/#/user/resetPassword?username=%s&token=%s",
+                domainUrl, username, resetToken);
         String content = getMessageBody(userAccount, resetToken, resetLink);
 
         SMTPMailProperties mailProperties = new SMTPMailProperties();

server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java

@@ -57,6 +57,7 @@ import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd;
 import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
+import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
@@ -75,6 +76,7 @@ import org.mockito.MockitoAnnotations;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -179,6 +181,12 @@ public class ResourceManagerImplTest {
     private MockedConstruction<GetVncPortCommand> getVncPortCommandMockedConstruction;
     private AutoCloseable closeable;
 
+    private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException {
+        Field f = ConfigKey.class.getDeclaredField(name);
+        f.setAccessible(true);
+        f.set(configKey, o);
+    }
+
     @Before
     public void setup() throws Exception {
         closeable = MockitoAnnotations.openMocks(this);
@@ -221,7 +229,7 @@ public class ResourceManagerImplTest {
                 eq("service cloudstack-agent restart"))).
                 willReturn(new SSHCmdHelper.SSHCmdResult(0,"",""));
 
-        when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("true");
+        overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "true");
 
         rootDisks = Arrays.asList(rootDisk1, rootDisk2);
         dataDisks = Collections.singletonList(dataDisk);
@@ -399,9 +407,9 @@ public class ResourceManagerImplTest {
     }
 
     @Test(expected = CloudRuntimeException.class)
-    public void testHandleAgentSSHDisabledNotConnectedAgent() {
+    public void testHandleAgentSSHDisabledNotConnectedAgent() throws NoSuchFieldException, IllegalAccessException {
         when(host.getStatus()).thenReturn(Status.Disconnected);
-        when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("false");
+        overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "false");
         resourceManager.handleAgentIfNotConnected(host, false);
     }
 

server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java

@@ -34,6 +34,7 @@ import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.reservation.dao.ReservationDao;
+import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
@@ -129,6 +130,8 @@ public class ResourceLimitManagerImplTest {
     UserVmDao userVmDao;
     @Mock
     EntityManager entityManager;
+    @Mock
+    SnapshotDataStoreDao snapshotDataStoreDao;
 
     private CallContext callContext;
     private List<String> hostTags = List.of("htag1", "htag2", "htag3");
@@ -900,12 +903,13 @@ public class ResourceLimitManagerImplTest {
         String tag = null;
         Mockito.when(vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId))
                 .thenReturn(List.of(1L));
+        Mockito.when(snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId)).thenReturn(100L);
         Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(100L);
-        Assert.assertEquals(100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag));
+        Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag));
 
         tag = "";
         Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(200L);
-        Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag));
+        Assert.assertEquals(300L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag));
 
         tag = "tag";
         VolumeVO vol = Mockito.mock(VolumeVO.class);
@@ -913,7 +917,7 @@ public class ResourceLimitManagerImplTest {
         Mockito.when(vol.getSize()).thenReturn(size);
         List<VolumeVO> vols = List.of(vol, vol);
         Mockito.doReturn(vols).when(resourceLimitManager).getVolumesWithAccountAndTag(accountId, tag);
-        Assert.assertEquals(vols.size() * size, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag));
+        Assert.assertEquals((vols.size() * size) + 100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag));
     }
 
     @Test

server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java

@@ -30,6 +30,7 @@ import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotPolicyVO;
 import com.cloud.storage.SnapshotVO;
 import com.cloud.storage.VolumeVO;
+import com.cloud.server.TaggedResourceService;
 import com.cloud.storage.dao.SnapshotDao;
 import com.cloud.storage.dao.SnapshotPolicyDao;
 import com.cloud.storage.dao.SnapshotZoneDao;
@@ -44,6 +45,7 @@ import com.cloud.utils.Pair;
 
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
+import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd;
 import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
@@ -100,6 +102,10 @@ public class SnapshotManagerImplTest {
     VolumeDao volumeDao;
     @Mock
     SnapshotPolicyDao snapshotPolicyDao;
+    @Mock
+    SnapshotScheduler snapshotScheduler;
+    @Mock
+    TaggedResourceService taggedResourceService;
     @InjectMocks
     SnapshotManagerImpl snapshotManager = new SnapshotManagerImpl();
 
@@ -108,6 +114,8 @@ public class SnapshotManagerImplTest {
         snapshotManager._snapshotPolicyDao = snapshotPolicyDao;
         snapshotManager._volsDao = volumeDao;
         snapshotManager._accountMgr = accountManager;
+        snapshotManager._snapSchedMgr = snapshotScheduler;
+        snapshotManager.taggedResourceService = taggedResourceService;
     }
 
     @After
@@ -520,4 +528,88 @@ public class SnapshotManagerImplTest {
         Assert.assertEquals(1, result.first().size());
         Assert.assertEquals(Integer.valueOf(1), result.second());
     }
+
+    @Test
+    public void testDeleteSnapshotPoliciesForRemovedVolume() {
+        Long policyId = 1L;
+        Long volumeId = 10L;
+        Long accountId = 2L;
+
+        DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class);
+        Mockito.when(cmd.getId()).thenReturn(policyId);
+        Mockito.when(cmd.getIds()).thenReturn(null);
+
+        Account caller = Mockito.mock(Account.class);
+        Mockito.when(caller.getId()).thenReturn(accountId);
+        CallContext.register(Mockito.mock(User.class), caller);
+
+        SnapshotPolicyVO policyVO = Mockito.mock(SnapshotPolicyVO.class);
+        Mockito.when(policyVO.getId()).thenReturn(policyId);
+        Mockito.when(policyVO.getVolumeId()).thenReturn(volumeId);
+        Mockito.when(policyVO.getUuid()).thenReturn("policy-uuid");
+        Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(policyVO);
+
+        // Volume is removed (expunged) but findByIdIncludingRemoved should still return it
+        VolumeVO volumeVO = Mockito.mock(VolumeVO.class);
+        Mockito.when(volumeDao.findByIdIncludingRemoved(volumeId)).thenReturn(volumeVO);
+
+        Mockito.when(snapshotPolicyDao.remove(policyId)).thenReturn(true);
+
+        boolean result = snapshotManager.deleteSnapshotPolicies(cmd);
+
+        Assert.assertTrue(result);
+        Mockito.verify(volumeDao).findByIdIncludingRemoved(volumeId);
+        Mockito.verify(snapshotScheduler).removeSchedule(volumeId, policyId);
+        Mockito.verify(snapshotPolicyDao).remove(policyId);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testDeleteSnapshotPoliciesNoPolicyId() {
+        DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class);
+        Mockito.when(cmd.getId()).thenReturn(null);
+        Mockito.when(cmd.getIds()).thenReturn(null);
+
+        snapshotManager.deleteSnapshotPolicies(cmd);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testDeleteSnapshotPoliciesPolicyNotFound() {
+        Long policyId = 1L;
+
+        DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class);
+        Mockito.when(cmd.getId()).thenReturn(policyId);
+        Mockito.when(cmd.getIds()).thenReturn(null);
+
+        Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(null);
+
+        snapshotManager.deleteSnapshotPolicies(cmd);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testDeleteSnapshotPoliciesVolumeNotFound() {
+        Long policyId = 1L;
+        Long volumeId = 10L;
+
+        DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class);
+        Mockito.when(cmd.getId()).thenReturn(policyId);
+        Mockito.when(cmd.getIds()).thenReturn(null);
+
+        SnapshotPolicyVO policyVO = Mockito.mock(SnapshotPolicyVO.class);
+        Mockito.when(policyVO.getVolumeId()).thenReturn(volumeId);
+        Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(policyVO);
+
+        // Volume doesn't exist at all (even when including removed)
+        Mockito.when(volumeDao.findByIdIncludingRemoved(volumeId)).thenReturn(null);
+
+        snapshotManager.deleteSnapshotPolicies(cmd);
+    }
+
+    @Test(expected = InvalidParameterValueException.class)
+    public void testDeleteSnapshotPoliciesManualPolicyId() {
+        DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class);
+        Mockito.when(cmd.getId()).thenReturn(Snapshot.MANUAL_POLICY_ID);
+        Mockito.when(cmd.getIds()).thenReturn(null);
+
+        snapshotManager.deleteSnapshotPolicies(cmd);
+    }
 }

systemvm/debian/opt/cloud/bin/cs/CsDhcp.py

@@ -110,7 +110,7 @@ class CsDhcp(CsDataBag):
             if gn.get_dns() and device:
                 sline = "dhcp-option=tag:interface-%s-%s,6" % (device, idx)
                 dns_list = [x for x in gn.get_dns() if x]
-                if (self.config.is_vpc() or self.config.is_router()) and ('is_vr_guest_gateway' in gn.data and gn.data['is_vr_guest_gateway']):
+                if self.config.is_vpc() and not gn.is_vr_guest_gateway():
                   if gateway in dns_list:
                     dns_list.remove(gateway)
                   if gn.data['router_guest_ip'] != ip:

ui/README.md

@@ -27,18 +27,18 @@ A modern role-based progressive CloudStack UI based on Vue.js and Ant Design.
 
 Install node: (Debian/Ubuntu)
 
-    curl -sL https://deb.nodesource.com/setup_20.x | sudo -E bash -
+    curl -sL https://deb.nodesource.com/setup_24.x | sudo -E bash -
     sudo apt-get install -y nodejs
     # Or use distro provided: sudo apt-get install npm nodejs
 
 Install node: (CentOS/Fedora/RHEL)
 
-    curl -sL https://rpm.nodesource.com/setup_20.x | sudo bash -
+    curl -sL https://rpm.nodesource.com/setup_24.x | sudo bash -
     sudo yum install nodejs
 
 Install node: (Mac OS)
 
-    brew install node@20
+    brew install node@24
 
 Optionally, you may also install system-wide dev tools:
 

ui/package.json

@@ -101,15 +101,18 @@
     "eslint-plugin-vue": "^7.0.0",
     "less": "^3.0.4",
     "less-loader": "^5.0.0",
+    "nan": "2.18.0",
+    "node-gyp": "10.0.1",
     "sass": "^1.49.9",
     "sass-loader": "^8.0.2",
     "uglifyjs-webpack-plugin": "^2.2.0",
     "vue-jest": "^5.0.0-0",
     "vue-svg-loader": "^0.17.0-beta.2",
-    "webpack": "^4.46.0",
-    "node-gyp": "10.0.1",  "nan": "2.18.0"
+    "webpack": "^4.46.0"
+  },
+  "resolutions": {
+    "nan": "2.18.0"
   },
-  "resolutions": { "nan": "2.18.0" },
   "eslintConfig": {
     "root": true,
     "env": {

ui/public/locales/en.json

@@ -42,7 +42,9 @@
 "label.accounts": "Accounts",
 "label.accountstate": "Account state",
 "label.accounttype": "Account type",
-"label.acl.export": "Export ACL rules",
+"label.import": "Import",
+"label.acl.import": "Import rules",
+"label.acl.export": "Export rules",
 "label.acl.id": "ACL ID",
 "label.acl.rules": "ACL rules",
 "label.acl.reason.description": "Enter the reason behind an ACL rule.",
@@ -254,7 +256,7 @@
 "label.activeviewersessions": "Active sessions",
 "label.add": "Add",
 "label.add.account": "Add Account",
-"label.add.acl.rule": "Add ACL rule",
+"label.add.acl.rule": "Add rule",
 "label.add.acl": "Add ACL",
 "label.add.affinity.group": "Add new Affinity Group",
 "label.add.backup.schedule": "Add Backup Schedule",
@@ -653,6 +655,8 @@
 "label.copy.consoleurl": "Copy console URL to clipboard",
 "label.copyid": "Copy ID",
 "label.copy.password": "Copy password",
+"label.copy.templates.from.other.secondary.storages": "Copy Templates from other storages instead of fetching from URLs",
+"label.copy.templates.from.other.secondary.storages.add.zone": "Copy Templates from other storages",
 "label.core": "Core",
 "label.core.zone.type": "Core Zone type",
 "label.count": "Count",
@@ -710,6 +714,7 @@
 "label.cron.mode": "Cron mode",
 "label.crosszones": "Cross Zones",
 "label.csienabled": "CSI Enabled",
+"label.csv.preview": "Data preview",
 "label.currency": "Currency",
 "label.current": "Current",
 "label.currentstep": "Current step",
@@ -2923,6 +2928,8 @@
 "message.action.create.snapshot.from.vmsnapshot": "Please confirm that you want to create Snapshot from Instance Snapshot",
 "message.action.create.instance.from.backup": "Please confirm that you want to create a new Instance from the given Backup.<br>Click on configure to edit the parameters for the new Instance before creation.",
 "message.create.instance.from.backup.different.zone": "Creating Instance from Backup on a different Zone. Please ensure that the backup repository is accessible in the selected Zone.",
+"message.csv.empty": "Empty CSV File",
+"message.csv.missing.headers": "Columns are missing from headers in CSV",
 "message.template.ostype.different.from.backup": "Selected Template has a different OS type than the Backup. Please proceed with caution.",
 "message.iso.ostype.different.from.backup": "Selected ISO has a different OS type than the Backup. Please proceed with caution.",
 "message.action.delete.asnrange": "Please confirm the AS range that you want to delete",
@@ -3346,7 +3353,7 @@
 "message.desc.register.user.data": "Please fill in the following to register new User Data.",
 "message.desc.registered.user.data": "Registered a User Data.",
 "message.desc.reset.ssh.key.pair": "Please specify a ssh key pair that you would like to add to this Instance.",
-"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.<br/><br/>Provide the IP address and exported path.",
+"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.<br/><br/>Provide the IP address and exported path.<br/><br/> \"Copy templates from other secondary storages\" switch can be used to automatically copy existing templates from secondary storages in other zones instead of fetching from their URLs.",
 "message.desc.validationformat": "Specifies the format used to validate the parameter value, such as EMAIL, URL, UUID, DECIMAL, etc.",
 "message.desc.valueoptions": "Provide a comma-separated list of values that will appear as selectable options for this parameter",
 "message.desc.zone": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.",
@@ -3676,6 +3683,7 @@
 "message.move.acl.order.processing": "Moving ACL rule...",
 "message.network.acl.default.allow": "Warning: With this policy all traffic will be allowed through the firewall to this VPC Network Tier. You should consider securing your Network.",
 "message.network.acl.default.deny": "Warning: With this policy all traffic will be denied through the firewall to this VPC Network Tier. In order to allow traffic through you will need to change policies.",
+"message.network.acl.import.note": "Note: Only valid rules from the CSV will be imported. Invalid entries will be discarded.",
 "message.network.addvm.desc": "Please specify the Network that you would like to add this Instance to. A new NIC will be added for this Network.",
 "message.network.description": "Setup Network and traffic.",
 "message.network.error": "Network Error",

ui/src/components/view/DedicateDomain.vue

@@ -18,52 +18,44 @@
 <template>
   <div class="form">
     <div class="form__item" :class="{'error': domainError}">
-      <a-spin :spinning="domainsLoading">
-        <p class="form__label">{{ $t('label.domain') }}<span class="required">*</span></p>
-        <p class="required required-label">{{ $t('label.required') }}</p>
-        <a-select
-          style="width: 100%"
-          showSearch
-          optionFilterProp="label"
-          :filterOption="(input, option) => {
-            return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-          }"
-          @change="handleChangeDomain"
-          v-focus="true"
-          v-model:value="domainId">
-          <a-select-option
-            v-for="(domain, index) in domainsList"
-            :value="domain.id"
-            :key="index"
-            :label="domain.path || domain.name || domain.description">
-            {{ domain.path || domain.name || domain.description }}
-          </a-select-option>
-        </a-select>
-      </a-spin>
-    </div>
-    <div class="form__item" v-if="accountsList">
-      <p class="form__label">{{ $t('label.account') }}</p>
-      <a-select
+      <p class="form__label">{{ $t('label.domain') }}<span class="required">*</span></p>
+      <p class="required required-label">{{ $t('label.required') }}</p>
+      <infinite-scroll-select
         style="width: 100%"
-        @change="handleChangeAccount"
-        showSearch
-        optionFilterProp="value"
-        :filterOption="(input, option) => {
-          return option.value.toLowerCase().indexOf(input.toLowerCase()) >= 0
-        }" >
-        <a-select-option v-for="(account, index) in accountsList" :value="account.name" :key="index">
-          {{ account.name }}
-        </a-select-option>
-      </a-select>
+        v-model:value="domainId"
+        api="listDomains"
+        :apiParams="domainsApiParams"
+        resourceType="domain"
+        optionValueKey="id"
+        optionLabelKey="path"
+        defaultIcon="block-outlined"
+        v-focus="true"
+        @change-option-value="handleChangeDomain" />
+    </div>
+    <div class="form__item">
+      <p class="form__label">{{ $t('label.account') }}</p>
+      <infinite-scroll-select
+        style="width: 100%"
+        v-model:value="selectedAccount"
+        api="listAccounts"
+        :apiParams="accountsApiParams"
+        resourceType="account"
+        optionValueKey="name"
+        optionLabelKey="name"
+        defaultIcon="team-outlined"
+        @change-option-value="handleChangeAccount" />
     </div>
   </div>
 </template>
 
 <script>
-import { getAPI } from '@/api'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 export default {
   name: 'DedicateDomain',
+  components: {
+    InfiniteScrollSelect
+  },
   props: {
     error: {
       type: Boolean,
@@ -72,59 +64,48 @@ export default {
   },
   data () {
     return {
-      domainsLoading: false,
       domainId: null,
-      accountsList: null,
-      domainsList: null,
+      selectedAccount: null,
       domainError: false
     }
   },
+  computed: {
+    domainsApiParams () {
+      return {
+        listall: true,
+        details: 'min'
+      }
+    },
+    accountsApiParams () {
+      if (!this.domainId) {
+        return {
+          listall: true,
+          showicon: true
+        }
+      }
+      return {
+        showicon: true,
+        domainid: this.domainId
+      }
+    }
+  },
   watch: {
     error () {
       this.domainError = this.error
     }
   },
   created () {
-    this.fetchData()
   },
   methods: {
-    fetchData () {
-      this.domainsLoading = true
-      getAPI('listDomains', {
-        listAll: true,
-        details: 'min'
-      }).then(response => {
-        this.domainsList = response.listdomainsresponse.domain
-
-        if (this.domainsList[0]) {
-          this.domainId = this.domainsList[0].id
-          this.handleChangeDomain(this.domainId)
-        }
-      }).catch(error => {
-        this.$notifyError(error)
-      }).finally(() => {
-        this.domainsLoading = false
-      })
-    },
-    fetchAccounts () {
-      getAPI('listAccounts', {
-        domainid: this.domainId
-      }).then(response => {
-        this.accountsList = response.listaccountsresponse.account || []
-        if (this.accountsList && this.accountsList.length === 0) {
-          this.handleChangeAccount(null)
-        }
-      }).catch(error => {
-        this.$notifyError(error)
-      })
-    },
-    handleChangeDomain (e) {
-      this.$emit('domainChange', e)
+    handleChangeDomain (domainId) {
+      this.domainId = domainId
+      this.selectedAccount = null
+      this.$emit('domainChange', domainId)
       this.domainError = false
-      this.fetchAccounts()
     },
-    handleChangeAccount (e) {
-      this.$emit('accountChange', e)
+    handleChangeAccount (accountName) {
+      this.selectedAccount = accountName
+      this.$emit('accountChange', accountName)
     }
   }
 }

ui/src/components/view/ListView.vue

@@ -747,6 +747,20 @@
         >{{ text }}</router-link>
         <span v-else>{{ text }}</span>
       </template>
+      <template v-if="column.key === 'parentname' && ['snapshot'].includes($route.path.split('/')[1])">
+        <router-link
+          v-if="record.parent && $router.resolve('/snapshot/' + record.parent).matched[0].redirect !== '/exception/404'"
+          :to="{ path: '/snapshot/' + record.parent }"
+        >{{ text }}</router-link>
+        <span v-else>{{ text }}</span>
+      </template>
+      <template v-if="column.key === 'parentName' && ['vmsnapshot'].includes($route.path.split('/')[1])">
+        <router-link
+          v-if="record.parent && $router.resolve('/vmsnapshot/' + record.parent).matched[0].redirect !== '/exception/404'"
+          :to="{ path: '/vmsnapshot/' + record.parent }"
+        >{{ text }}</router-link>
+        <span v-else>{{ text }}</span>
+      </template>
       <template v-if="column.key === 'templateversion'">
         <span> {{ record.version }} </span>
       </template>
@@ -862,6 +876,14 @@
       <template v-if="['isfeatured'].includes(column.key) && ['guestoscategory'].includes($route.path.split('/')[1])">
         {{ record.isfeatured ? $t('label.yes') : $t('label.no') }}
       </template>
+      <template v-if="['agentscount'].includes(column.key)">
+        <router-link
+          v-if="['managementserver'].includes($route.path.split('/')[1]) && $router.resolve('/host').matched[0].redirect !== '/exception/404'"
+          :to="{ path: '/host', query: { managementserverid: record.id } }">
+          {{ text }}
+        </router-link>
+        <span v-else> {{ text }} </span>
+      </template>
       <template v-if="column.key === 'order'">
         <div class="shift-btns">
           <a-tooltip

ui/src/components/widgets/InfiniteScrollSelect.vue

@@ -41,9 +41,11 @@
   - optionValueKey (String, optional): Property to use as the value for options (e.g., 'name'). Default is 'id'
   - optionLabelKey (String, optional): Property to use as the label for options (e.g., 'name'). Default is 'name'
   - defaultOption (Object, optional): Preselected object to include initially
+  - allowClear (Boolean, optional): Whether to allow clearing the selection. Default is false
   - showIcon (Boolean, optional): Whether to show icon for the options. Default is true
   - defaultIcon (String, optional): Icon to be shown when there is no resource icon for the option. Default is 'cloud-outlined'
   - autoSelectFirstOption (Boolean, optional): Whether to automatically select the first option when options are loaded. Default is false
+  - selectFirstOption (Boolean, optional): Whether to automatically select the first option when options are loaded. Default is false
 
   Events:
   - @change-option-value (Function): Emits the selected option value(s) when value(s) changes. Do not use @change as it will give warnings and may not work
@@ -59,6 +61,7 @@
     :filter-option="false"
     :loading="loading"
     show-search
+    :allowClear="allowClear"
     placeholder="Select"
     @search="onSearchTimed"
     @popupScroll="onScroll"
@@ -76,9 +79,9 @@
         </div>
       </div>
     </template>
-    <a-select-option v-for="option in options" :key="option.id" :value="option[optionValueKey]">
+    <a-select-option v-for="option in selectableOptions" :key="option.id" :value="option[optionValueKey]">
       <span>
-        <span v-if="showIcon && option.showicon !== false">
+        <span v-if="showIcon && option.showicon !== false && option.id !== null && option.id !== undefined">
           <resource-icon v-if="option.icon && option.icon.base64image" :image="option.icon.base64image" size="1x" style="margin-right: 5px"/>
           <render-icon v-else :icon="defaultIcon" style="margin-right: 5px" />
         </span>
@@ -129,6 +132,10 @@ export default {
       type: Object,
       default: null
     },
+    allowClear: {
+      type: Boolean,
+      default: false
+    },
     showIcon: {
       type: Boolean,
       default: true
@@ -144,6 +151,10 @@ export default {
     autoSelectFirstOption: {
       type: Boolean,
       default: false
+    },
+    selectFirstOption: {
+      type: Boolean,
+      default: false
     }
   },
   data () {
@@ -157,7 +168,8 @@ export default {
       scrollHandlerAttached: false,
       preselectedOptionValue: null,
       successiveFetches: 0,
-      canSelectFirstOption: false
+      canSelectFirstOption: false,
+      hasAutoSelectedFirst: false
     }
   },
   created () {
@@ -176,6 +188,36 @@ export default {
     },
     formattedSearchFooterMessage () {
       return `${this.$t('label.showing.results.for').replace('%x', this.searchQuery)}`
+    },
+    selectableOptions () {
+      const currentValue = this.$attrs.value
+      // Only filter out null/empty options when the current value is also null/undefined/empty
+      // This prevents such options from being selected and allows the placeholder to show instead
+      if (currentValue === null || currentValue === undefined || currentValue === '') {
+        return this.options.filter(option => {
+          const optionValue = option[this.optionValueKey]
+          return optionValue !== null && optionValue !== undefined && optionValue !== ''
+        })
+      }
+      // When a valid value is selected, show all options
+      return this.options
+    },
+    apiOptionsCount () {
+      if (this.defaultOption) {
+        const defaultOptionValue = this.defaultOption[this.optionValueKey]
+        return this.options.filter(option => option[this.optionValueKey] !== defaultOptionValue).length
+      }
+      return this.options.length
+    },
+    preselectedMatchValue () {
+      // Extract the first value from preselectedOptionValue if it's an array, otherwise return the value itself
+      if (!this.preselectedOptionValue) return null
+      return Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue
+    },
+    preselectedMatch () {
+      // Find the matching option for the preselected value
+      if (!this.preselectedMatchValue) return null
+      return this.options.find(entry => entry[this.optionValueKey] === this.preselectedMatchValue) || null
     }
   },
   watch: {
@@ -221,6 +263,7 @@ export default {
         this.canSelectFirstOption = true
         if (this.successiveFetches === 0) {
           this.loading = false
+          this.autoSelectFirstOptionIfNeeded()
         }
       })
     },
@@ -237,11 +280,10 @@ export default {
         }
         return
       }
-      const matchValue = Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue
-      const match = this.options.find(entry => entry[this.optionValueKey] === matchValue)
-      if (!match) {
+      if (!this.preselectedMatch) {
         this.successiveFetches++
-        if (this.options.length < this.totalCount) {
+        // Exclude defaultOption from count when comparing with totalCount
+        if (this.apiOptionsCount < this.totalCount) {
           this.fetchItems()
         } else {
           this.resetPreselectedOptionValue()
@@ -249,7 +291,7 @@ export default {
         return
       }
       if (Array.isArray(this.preselectedOptionValue) && this.preselectedOptionValue.length > 1) {
-        this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== match)
+        this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== this.preselectedMatchValue)
       } else {
         this.resetPreselectedOptionValue()
       }
@@ -264,6 +306,36 @@ export default {
       this.preselectedOptionValue = null
       this.successiveFetches = 0
     },
+    autoSelectFirstOptionIfNeeded () {
+      if (!this.selectFirstOption || this.hasAutoSelectedFirst) {
+        return
+      }
+      // Don't auto-select if there's a preselected value being fetched
+      if (this.preselectedOptionValue) {
+        return
+      }
+      const currentValue = this.$attrs.value
+      if (currentValue !== undefined && currentValue !== null && currentValue !== '') {
+        return
+      }
+      if (this.options.length === 0) {
+        return
+      }
+      if (this.searchQuery && this.searchQuery.length > 0) {
+        return
+      }
+      // Only auto-select after initial load is complete (no more successive fetches)
+      if (this.successiveFetches > 0) {
+        return
+      }
+      const firstOption = this.options[0]
+      if (firstOption) {
+        const firstValue = firstOption[this.optionValueKey]
+        this.hasAutoSelectedFirst = true
+        this.$emit('change-option-value', firstValue)
+        this.$emit('change-option', firstOption)
+      }
+    },
     onSearchTimed (value) {
       clearTimeout(this.searchTimer)
       this.searchTimer = setTimeout(() => {
@@ -282,7 +354,8 @@ export default {
     },
     onScroll (e) {
       const nearBottom = e.target.scrollTop + e.target.clientHeight >= e.target.scrollHeight - 10
-      const hasMore = this.options.length < this.totalCount
+      // Exclude defaultOption from count when comparing with totalCount
+      const hasMore = this.apiOptionsCount < this.totalCount
       if (nearBottom && hasMore && !this.loading) {
         this.fetchItems()
       }

ui/src/components/widgets/OsLogo.vue

@@ -31,6 +31,9 @@
 <script>
 import { getAPI } from '@/api'
 
+const CACHE_TTL_MS = 30_000
+const osTypeCache = new Map() // osId -> { ts, value?, promise? }
+
 export default {
   name: 'OsLogo',
   props: {
@@ -45,55 +48,57 @@ export default {
     size: {
       type: String,
       default: 'lg'
+    },
+    useCache: {
+      type: Boolean,
+      default: false
     }
   },
   data () {
-    return {
-      name: '',
-      osLogo: ['fas', 'image']
-    }
-  },
-  computed: {
-    logo: function () {
-      if (!this.name) {
-        this.fetchData()
-      }
-      return this.osLogo
-    }
+    return { name: '', osLogo: ['fas', 'image'] }
   },
+  computed: { logo () { return this.osLogo } },
+  mounted () { this.fetchData() },
   watch: {
-    osId: function () {
-      this.fetchData()
-    }
+    osId () { this.fetchData() },
+    osName () { this.fetchData() }
   },
   methods: {
+    async fetchOsTypeName (osId, useCache = this.useCache) {
+      const now = Date.now()
+      if (useCache) {
+        const cached = osTypeCache.get(osId)
+        if (cached?.value && (now - cached.ts) < CACHE_TTL_MS) return cached.value
+        if (cached?.promise) return cached.promise
+        const promise = getAPI('listOsTypes', { id: osId })
+          .then(json => {
+            const t = json?.listostypesresponse?.ostype
+            const name = t?.length
+              ? (t[0].description || t[0].osdisplayname || 'Linux')
+              : 'Linux'
+            osTypeCache.set(osId, { ts: Date.now(), value: name })
+            return name
+          })
+          .catch(e => { osTypeCache.delete(osId); throw e })
+        osTypeCache.set(osId, { ts: now, promise })
+        return promise
+      }
+      const json = await getAPI('listOsTypes', { id: osId })
+      const t = json?.listostypesresponse?.ostype
+      return t?.length ? (t[0].description || t[0].osdisplayname || 'Linux') : 'Linux'
+    },
     fetchData () {
       if (this.osName) {
         this.discoverOsLogo(this.osName)
-      } else if (this.osId) {
-        this.findOsName(this.osId)
+      } else if (this.osId && ('listOsTypes' in this.$store.getters.apis)) {
+        this.fetchOsTypeName(this.osId)
+          .then(this.discoverOsLogo)
+          .catch(() => this.discoverOsLogo('Linux'))
       }
     },
-    findOsName (osId) {
-      if (!('listOsTypes' in this.$store.getters.apis)) {
-        return
-      }
-      this.name = 'linux'
-      getAPI('listOsTypes', { id: osId }).then(json => {
-        if (json && json.listostypesresponse && json.listostypesresponse.ostype && json.listostypesresponse.ostype.length > 0) {
-          this.discoverOsLogo(json.listostypesresponse.ostype[0].description)
-        } else {
-          this.discoverOsLogo('Linux')
-        }
-      })
-    },
-    getFontAwesomeIcon (name) {
-      return ['fab', name]
-    },
     discoverOsLogo (name) {
       this.name = name
-      this.$emit('update-osname', this.name)
-      const osname = name.toLowerCase()
+      const osname = (name || '').toLowerCase()
       const logos = [
         { name: 'centos' },
         { name: 'debian' },
@@ -119,6 +124,3 @@ export default {
   }
 }
 </script>
-
-<style scoped>
-</style>

ui/src/config/section/storage.js

@@ -92,7 +92,7 @@ export default {
         }
       ],
       searchFilters: () => {
-        var filters = ['name', 'zoneid', 'domainid', 'account', 'state', 'tags', 'serviceofferingid', 'diskofferingid', 'isencrypted']
+        const filters = ['name', 'zoneid', 'domainid', 'account', 'state', 'tags', 'serviceofferingid', 'diskofferingid', 'isencrypted']
         if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) {
           filters.push('storageid')
         }
@@ -311,7 +311,10 @@ export default {
       permission: ['listSnapshots'],
       resourceType: 'Snapshot',
       columns: () => {
-        var fields = ['name', 'state', 'volumename', 'intervaltype', 'physicalsize', 'created']
+        const fields = ['name', 'state', 'volumename', 'intervaltype', 'physicalsize', 'created']
+        if (store.getters.features.snapshotshowchainsize) {
+          fields.splice(fields.indexOf('created'), 0, 'chainsize', 'parentname')
+        }
         if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) {
           fields.push('account')
           if (store.getters.listAllProjects) {
@@ -324,7 +327,13 @@ export default {
         fields.push('zonename')
         return fields
       },
-      details: ['name', 'id', 'volumename', 'volumetype', 'snapshottype', 'intervaltype', 'physicalsize', 'virtualsize', 'chainsize', 'account', 'domain', 'created'],
+      details: () => {
+        const fields = ['name', 'id', 'volumename', 'volumetype', 'snapshottype', 'intervaltype', 'physicalsize', 'virtualsize', 'account', 'domain', 'created']
+        if (store.getters.features.snapshotshowchainsize) {
+          fields.splice(fields.indexOf('account'), 0, 'chainsize', 'parentname')
+        }
+        return fields
+      },
       tabs: [
         {
           name: 'details',
@@ -346,7 +355,7 @@ export default {
         }
       ],
       searchFilters: () => {
-        var filters = ['name', 'domainid', 'account', 'tags', 'zoneid']
+        const filters = ['name', 'domainid', 'account', 'tags', 'zoneid']
         if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) {
           filters.push('storageid')
           filters.push('imagestoreid')

ui/src/config/section/user.js

@@ -105,9 +105,10 @@ export default {
       message: 'message.enable.user',
       dataView: true,
       show: (record, store) => {
-        return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault &&
-          !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) &&
-          ['disabled', 'locked'].includes(record.state)
+        if (!['disabled', 'locked'].includes(record.state) || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) {
+          return false
+        }
+        return ![1, 4].includes(record.accounttype) || store.userInfo.roletype === 'Admin'
       }
     },
     {
@@ -117,9 +118,10 @@ export default {
       message: 'message.disable.user',
       dataView: true,
       show: (record, store) => {
-        return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault &&
-          !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) &&
-          record.state === 'enabled'
+        if (record.state !== 'enabled' || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) {
+          return false
+        }
+        return ![1, 4].includes(record.accounttype) || (store.userInfo.roletype === 'Admin' && record.id !== store.userInfo.id)
       }
     },
     {
@@ -131,9 +133,10 @@ export default {
       dataView: true,
       popup: true,
       show: (record, store) => {
-        return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault &&
-          !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) &&
-          record.state === 'enabled'
+        if (record.state !== 'enabled' || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) {
+          return false
+        }
+        return ![1, 4].includes(record.accounttype) || (store.userInfo.roletype === 'Admin' && record.id !== store.userInfo.id)
       }
     },
     {

ui/src/views/AutogenView.vue

@@ -1196,15 +1196,12 @@ export default {
             }
           }
         }
-        if (this.items.length > 0) {
-          if (!this.showAction || this.dataView) {
-            this.resource = this.items[0]
-            this.$emit('change-resource', this.resource)
-          }
-        } else {
-          if (this.dataView) {
-            this.$router.push({ path: '/exception/404' })
-          }
+        if (this.items.length <= 0 && this.dataView) {
+          this.$router.push({ path: '/exception/404' })
+        }
+        if (!this.showAction || this.dataView) {
+          this.resource = this.items?.[0] || {}
+          this.$emit('change-resource', this.resource)
         }
       }).catch(error => {
         if (!error || !error.message) {

ui/src/views/compute/wizard/OsBasedImageRadioGroup.vue

@@ -42,7 +42,8 @@
               class="radio-group__os-logo"
               size="2x"
               :osId="item.ostypeid"
-              :os-name="item.osName" />
+              :os-name="item.osName"
+              :use-cache="true" />
              
             {{ item.displaytext }}
             <span v-if="item?.projectid">

ui/src/views/iam/AddUser.vue

@@ -90,45 +90,31 @@
           <template #label>
             <tooltip-label :title="$t('label.domainid')" :tooltip="apiParams.domainid.description"/>
           </template>
-          <a-select
-            :loading="domainLoading"
+          <infinite-scroll-select
             v-model:value="form.domainid"
+            api="listDomains"
+            :apiParams="domainsApiParams"
+            resourceType="domain"
+            optionValueKey="id"
+            optionLabelKey="path"
+            defaultIcon="block-outlined"
+            :selectFirstOption="true"
             :placeholder="apiParams.domainid.description"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return  option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }" >
-            <a-select-option v-for="domain in domainsList" :key="domain.id" :label="domain.path || domain.name || domain.description">
-              <span>
-                <resource-icon v-if="domain && domain.icon" :image="domain.icon.base64image" size="1x" style="margin-right: 5px"/>
-                <block-outlined v-else style="margin-right: 5px" />
-                {{ domain.path || domain.name || domain.description }}
-              </span>
-            </a-select-option>
-          </a-select>
+            @change-option-value="handleDomainChange" />
         </a-form-item>
         <a-form-item name="account" ref="account" v-if="!account">
           <template #label>
             <tooltip-label :title="$t('label.account')" :tooltip="apiParams.account.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.account"
-            :loading="loadingAccount"
-            :placeholder="apiParams.account.description"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return  option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }" >
-            <a-select-option v-for="(item, idx) in accountList" :key="idx" :label="item.name">
-              <span>
-                <resource-icon v-if="item && item.icon" :image="item.icon.base64image" size="1x" style="margin-right: 5px"/>
-                <team-outlined v-else style="margin-right: 5px" />
-                {{ item.name }}
-              </span>
-            </a-select-option>
-          </a-select>
+            api="listAccounts"
+            :apiParams="accountsApiParams"
+            resourceType="account"
+            optionValueKey="name"
+            optionLabelKey="name"
+            defaultIcon="team-outlined"
+            :placeholder="apiParams.account.description" />
         </a-form-item>
         <a-form-item name="timezone" ref="timezone">
           <template #label>
@@ -185,12 +171,14 @@ import { timeZone } from '@/utils/timezone'
 import debounce from 'lodash/debounce'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 export default {
   name: 'AddUser',
   components: {
     TooltipLabel,
-    ResourceIcon
+    ResourceIcon,
+    InfiniteScrollSelect
   },
   data () {
     this.fetchTimeZone = debounce(this.fetchTimeZone, 800)
@@ -198,14 +186,9 @@ export default {
       loading: false,
       timeZoneLoading: false,
       timeZoneMap: [],
-      domainLoading: false,
-      domainsList: [],
-      selectedDomain: '',
       samlEnable: false,
       idpLoading: false,
       idps: [],
-      loadingAccount: false,
-      accountList: [],
       account: null,
       domainid: null
     }
@@ -218,6 +201,19 @@ export default {
   computed: {
     samlAllowed () {
       return 'authorizeSamlSso' in this.$store.getters.apis
+    },
+    domainsApiParams () {
+      return {
+        listall: true,
+        showicon: true,
+        details: 'min'
+      }
+    },
+    accountsApiParams () {
+      return {
+        showicon: true,
+        domainid: this.form?.domainid || null
+      }
     }
   },
   methods: {
@@ -241,53 +237,18 @@ export default {
     fetchData () {
       this.account = this.$route.query && this.$route.query.account ? this.$route.query.account : null
       this.domainid = this.$route.query && this.$route.query.domainid ? this.$route.query.domainid : null
-      if (!this.domianid) {
-        this.fetchDomains()
+      // Set initial domain if provided from route
+      if (this.domainid) {
+        this.form.domainid = this.domainid
       }
       this.fetchTimeZone()
       if (this.samlAllowed) {
         this.fetchIdps()
       }
     },
-    fetchDomains () {
-      this.domainLoading = true
-      var params = {
-        listAll: true,
-        showicon: true,
-        details: 'min'
-      }
-      getAPI('listDomains', params).then(response => {
-        this.domainsList = response.listdomainsresponse.domain || []
-      }).catch(error => {
-        this.$notification.error({
-          message: `${this.$t('label.error')} ${error.response.status}`,
-          description: error.response.data.errorresponse.errortext
-        })
-      }).finally(() => {
-        const domainid = this.domainsList[0]?.id || ''
-        this.form.domainid = domainid
-        this.fetchAccount(domainid)
-        this.domainLoading = false
-      })
-    },
-    fetchAccount (domainid) {
-      this.accountList = []
+    handleDomainChange (domainId) {
+      this.form.domainid = domainId
       this.form.account = null
-      this.loadingAccount = true
-      var params = { listAll: true, showicon: true }
-      if (domainid) {
-        params.domainid = domainid
-      }
-      getAPI('listAccounts', params).then(response => {
-        this.accountList = response.listaccountsresponse.account || []
-      }).catch(error => {
-        this.$notification.error({
-          message: `${this.$t('label.error')} ${error.response.status}`,
-          description: error.response.data.errorresponse.errortext
-        })
-      }).finally(() => {
-        this.loadingAccount = false
-      })
     },
     fetchTimeZone (value) {
       this.timeZoneMap = []
@@ -369,12 +330,14 @@ export default {
         accounttype: 0
       }
 
+      // Account: use route query account if available, otherwise use form value (which is the account name)
       if (this.account) {
         params.account = this.account
-      } else if (this.accountList[rawParams.account]) {
-        params.account = this.accountList[rawParams.account].name
+      } else if (rawParams.account) {
+        params.account = rawParams.account
       }
 
+      // Domain: use route query domainid if available, otherwise use form value
       if (this.domainid) {
         params.domainid = this.domainid
       } else if (rawParams.domainid) {

ui/src/views/infra/AddSecondaryStorage.vue

@@ -48,6 +48,7 @@
           <a-form-item name="zone" ref="zone" :label="$t('label.zone')">
             <a-select
               v-model:value="form.zone"
+              @change="onZoneChange"
               showSearch
               optionFilterProp="label"
               :filterOption="(input, option) => {
@@ -105,6 +106,7 @@
           <a-form-item name="zone" ref="zone" :label="$t('label.zone')">
             <a-select
               v-model:value="form.zone"
+              @change="onZoneChange"
               showSearch
               optionFilterProp="label"
               :filterOption="(input, option) => {
@@ -159,6 +161,17 @@
             <a-input v-model:value="form.secondaryStorageNFSPath"/>
           </a-form-item>
         </div>
+        <div v-if="showCopyTemplatesToggle">
+          <a-form-item
+            name="copyTemplatesFromOtherSecondaryStorages"
+            ref="copyTemplatesFromOtherSecondaryStorages"
+            :label="$t('label.copy.templates.from.other.secondary.storages')">
+            <a-switch
+              v-model:checked="form.copyTemplatesFromOtherSecondaryStorages"
+              @change="onCopyTemplatesToggleChanged"
+            />
+          </a-form-item>
+        </div>
         <div :span="24" class="action-button">
           <a-button @click="closeModal">{{ $t('label.cancel') }}</a-button>
           <a-button type="primary" ref="submit" @click="handleSubmit">{{ $t('label.ok') }}</a-button>
@@ -191,7 +204,9 @@ export default {
       providers: ['NFS', 'SMB/CIFS', 'S3', 'Swift'],
       zones: [],
       loading: false,
-      secondaryStorageNFSStaging: false
+      secondaryStorageNFSStaging: false,
+      showCopyTemplatesToggle: false,
+      copyTemplatesTouched: false
     }
   },
   created () {
@@ -203,7 +218,8 @@ export default {
       this.formRef = ref()
       this.form = reactive({
         provider: 'NFS',
-        secondaryStorageHttps: true
+        secondaryStorageHttps: true,
+        copyTemplatesFromOtherSecondaryStorages: true
       })
       this.rules = reactive({
         zone: [{ required: true, message: this.$t('label.required') }],
@@ -225,20 +241,56 @@ export default {
     },
     fetchData () {
       this.listZones()
+      this.checkOtherSecondaryStorages()
     },
     closeModal () {
       this.$emit('close-action')
     },
+    fetchCopyTemplatesConfig () {
+      if (!this.form.zone) {
+        return
+      }
+
+      getAPI('listConfigurations', {
+        name: 'copy.templates.from.other.secondary.storages',
+        zoneid: this.form.zone
+      }).then(json => {
+        const items =
+          json?.listconfigurationsresponse?.configuration || []
+
+        items.forEach(item => {
+          if (item.name === 'copy.templates.from.other.secondary.storages') {
+            this.form.copyTemplatesFromOtherSecondaryStorages =
+              item.value === 'true'
+          }
+        })
+      })
+    },
+    onZoneChange (val) {
+      this.form.zone = val
+      this.copyTemplatesTouched = false
+      this.fetchCopyTemplatesConfig()
+    },
     listZones () {
       getAPI('listZones', { showicon: true }).then(json => {
-        if (json && json.listzonesresponse && json.listzonesresponse.zone) {
-          this.zones = json.listzonesresponse.zone
-          if (this.zones.length > 0) {
-            this.form.zone = this.zones[0].id || ''
-          }
+        this.zones = json.listzonesresponse.zone || []
+
+        if (this.zones.length > 0) {
+          this.form.zone = this.zones[0].id
+          this.fetchCopyTemplatesConfig()
         }
       })
     },
+    checkOtherSecondaryStorages () {
+      getAPI('listImageStores', { listall: true }).then(json => {
+        const stores = json?.listimagestoresresponse?.imagestore || []
+
+        this.showCopyTemplatesToggle = stores.length > 0
+      })
+    },
+    onCopyTemplatesToggleChanged (val) {
+      this.copyTemplatesTouched = true
+    },
     nfsURL (server, path) {
       var url
       if (path.substring(0, 1) !== '/') {
@@ -362,6 +414,22 @@ export default {
           nfsParams.url = nfsUrl
         }
 
+        if (
+          this.showCopyTemplatesToggle &&
+          this.copyTemplatesTouched
+        ) {
+          const copyTemplatesKey = 'copytemplatesfromothersecondarystorages'
+
+          const detailIdx = Object.keys(data)
+            .filter(k => k.startsWith('details['))
+            .map(k => parseInt(k.match(/details\[(\d+)\]/)[1]))
+            .reduce((a, b) => Math.max(a, b), -1) + 1
+
+          data[`details[${detailIdx}].key`] = copyTemplatesKey
+          data[`details[${detailIdx}].value`] =
+            values.copyTemplatesFromOtherSecondaryStorages.toString()
+        }
+
         this.loading = true
 
         try {

ui/src/views/infra/UsageRecords.vue

@@ -121,15 +121,20 @@
                   ref="domain"
                   name="domain"
                 >
-                  <a-auto-complete
+                  <infinite-scroll-select
                     v-model:value="form.domain"
-                    :options="domains"
+                    api="listDomains"
+                    :apiParams="domainsApiParams"
+                    resourceType="domain"
+                    optionValueKey="id"
+                    optionLabelKey="path"
+                    defaultIcon="block-outlined"
                     :placeholder="$t('label.domain')"
-                    :filter-option="filterOption"
+                    :defaultOption="{ id: null, path: ''}"
+                    :allowClear="true"
                     style="width: 100%;"
-                    @select="getAccounts"
-                    :dropdownMatchSelectWidth="false"
-                  />
+                    @change-option-value="handleDomainChange"
+                    @change-option="handleDomainOptionChange" />
                 </a-form-item>
               </a-col>
             </a-row>&nbsp;
@@ -150,15 +155,20 @@
               ref="account"
               name="account"
             >
-            <a-auto-complete
+              <infinite-scroll-select
                 v-model:value="form.account"
-                :options="accounts"
+                api="listAccounts"
+                :apiParams="accountsApiParams"
+                resourceType="account"
+                optionValueKey="id"
+                optionLabelKey="name"
+                defaultIcon="team-outlined"
                 :placeholder="$t('label.account')"
-                :filter-option="filterOption"
                 :disabled="form.isRecursive"
-                :dropdownMatchSelectWidth="false"
-                @select="selectAccount"
-              />
+                :defaultOption="{ id: null, name: ''}"
+                allowClear="true"
+                @change-option-value="selectAccount"
+                @change-option="selectAccountOption" />
             </a-form-item>
           </a-col>
           <a-col :span="3" v-if="'listUsageTypes' in $store.getters.apis">
@@ -361,6 +371,7 @@ import ListView from '@/components/view/ListView'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
 import TooltipButton from '@/components/widgets/TooltipButton'
 import Status from '@/components/widgets/Status'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 dayjs.extend(relativeTime)
 dayjs.extend(utc)
@@ -374,7 +385,8 @@ export default {
     ListView,
     Status,
     TooltipLabel,
-    TooltipButton
+    TooltipButton,
+    InfiniteScrollSelect
   },
   props: {
     resource: {
@@ -402,8 +414,6 @@ export default {
       page: 1,
       pageSize: 20,
       usageTypes: [],
-      domains: [],
-      accounts: [],
       account: null,
       domain: null,
       usageType: null,
@@ -436,6 +446,23 @@ export default {
     this.fetchData()
     this.updateColumns()
   },
+  computed: {
+    domainsApiParams () {
+      return {
+        listall: true
+      }
+    },
+    accountsApiParams () {
+      if (!this.form.domain) {
+        return {
+          listall: true
+        }
+      }
+      return {
+        domainid: this.form.domain
+      }
+    }
+  },
   methods: {
     clearFilters () {
       this.formRef.value.resetFields()
@@ -445,8 +472,6 @@ export default {
       this.usageType = null
       this.page = 1
       this.pageSize = 20
-
-      this.getAccounts()
     },
     disabledDate (current) {
       return current && current > dayjs().endOf('day')
@@ -473,8 +498,6 @@ export default {
       this.listUsageServerMetrics()
       this.getUsageTypes()
       this.getAllUsageRecordColumns()
-      this.getDomains()
-      this.getAccounts()
       if (!this.$store.getters.customColumns[this.$store.getters.userInfo.id]) {
         this.$store.getters.customColumns[this.$store.getters.userInfo.id] = {}
         this.$store.getters.customColumns[this.$store.getters.userInfo.id][this.$route.path] = this.selectedColumnKeys
@@ -528,16 +551,6 @@ export default {
         this.formRef.value.scrollToField(error.errorFields[0].name)
       })
     },
-    selectAccount (value, option) {
-      if (option && option.id) {
-        this.account = option
-      } else {
-        this.account = null
-        if (this.formRef?.value) {
-          this.formRef.value.resetFields('account')
-        }
-      }
-    },
     selectUsageType (value, option) {
       if (option && option.id) {
         this.usageType = option
@@ -548,24 +561,12 @@ export default {
         }
       }
     },
-    getDomains () {
-      getAPI('listDomains', { listAll: true }).then(json => {
-        if (json && json.listdomainsresponse && json.listdomainsresponse.domain) {
-          this.domains = [{ id: null, value: '' }, ...json.listdomainsresponse.domain.map(x => {
-            return {
-              id: x.id,
-              value: x.path
-            }
-          })]
-        }
-      })
+    handleDomainChange (domainId) {
+      this.form.domain = domainId
+      this.form.account = null
     },
-    getAccounts (value, option) {
-      var params = {
-        listAll: true
-      }
+    handleDomainOptionChange (option) {
       if (option && option.id) {
-        params.domainid = option.id
         this.domain = option
       } else {
         this.domain = null
@@ -573,16 +574,19 @@ export default {
           this.formRef.value.resetFields('domain')
         }
       }
-      getAPI('listAccounts', params).then(json => {
-        if (json && json.listaccountsresponse && json.listaccountsresponse.account) {
-          this.accounts = [{ id: null, value: '' }, ...json.listaccountsresponse.account.map(x => {
-            return {
-              id: x.id,
-              value: x.name
-            }
-          })]
+    },
+    selectAccount (accountId) {
+      this.form.account = accountId
+    },
+    selectAccountOption (option) {
+      if (option && option.id) {
+        this.account = option
+      } else {
+        this.account = null
+        if (this.formRef?.value) {
+          this.formRef.value.resetFields('account')
         }
-      })
+      }
     },
     getParams (page, pageSize) {
       const formRaw = toRaw(this.form)

ui/src/views/infra/zone/ZoneWizardAddResources.vue

@@ -848,6 +848,13 @@ export default {
           display: {
             secondaryStorageProvider: ['Swift']
           }
+        },
+        {
+          title: 'label.copy.templates.from.other.secondary.storages.add.zone',
+          key: 'copyTemplatesFromOtherSecondaryStorages',
+          required: false,
+          switch: true,
+          checked: this.copytemplate
         }
       ]
     }
@@ -871,7 +878,8 @@ export default {
       }],
       storageProviders: [],
       currentStep: null,
-      options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider']
+      options: ['primaryStorageScope', 'primaryStorageProtocol', 'provider', 'primaryStorageProvider'],
+      copytemplate: true
     }
   },
   created () {
@@ -896,6 +904,7 @@ export default {
           primaryStorageScope: null
         })
       }
+      this.applyCopyTemplatesOptionFromGlobalSettingDuringSecondaryStorageAddition()
     }
   },
   watch: {
@@ -1119,6 +1128,20 @@ export default {
         this.storageProviders = storageProviders
       })
     },
+    applyCopyTemplatesOptionFromGlobalSettingDuringSecondaryStorageAddition () {
+      getAPI('listConfigurations', {
+        name: 'copy.templates.from.other.secondary.storages'
+      }).then(json => {
+        const config = json?.listconfigurationsresponse?.configuration?.[0]
+
+        if (!config || config.value === undefined) {
+          return
+        }
+
+        const value = String(config.value).toLowerCase() === 'true'
+        this.copytemplate = value
+      })
+    },
     fetchPrimaryStorageProvider () {
       this.primaryStorageProviders = []
       getAPI('listStorageProviders', { type: 'primary' }).then(json => {

ui/src/views/infra/zone/ZoneWizardLaunchZone.vue

@@ -1639,6 +1639,11 @@ export default {
         params.provider = this.prefillContent.secondaryStorageProvider
         params.zoneid = this.stepData.zoneReturned.id
         params.url = url
+        if (this.prefillContent.copyTemplatesFromOtherSecondaryStorages !== undefined) {
+          params['details[0].key'] = 'copytemplatesfromothersecondarystorages'
+          params['details[0].value'] =
+            this.prefillContent.copyTemplatesFromOtherSecondaryStorages
+        }
       } else if (this.prefillContent.secondaryStorageProvider === 'SMB') {
         const nfsServer = this.prefillContent.secondaryStorageServer
         const path = this.prefillContent.secondaryStoragePath

ui/src/views/network/AclRulesTab.vue

@@ -28,10 +28,16 @@
         {{ $t('label.add.acl.rule') }}
       </a-button>
 
+      <a-button type="dashed" @click="handleImportRules" style="width: 100%;  margin-right: 10px">
+        <template #icon><upload-outlined /></template>
+        {{ $t('label.acl.import') }}
+      </a-button>
+
       <a-button type="dashed" @click="exportAclList" style="width: 100%">
         <template #icon><download-outlined /></template>
         {{ $t('label.acl.export') }}
       </a-button>
+
       <div class="search-bar">
         <a-input-search
           style="width: 25vw;float: right;margin-left: 10px; z-index: 8"
@@ -304,6 +310,21 @@
         </div>
       </a-form>
     </a-modal>
+
+    <a-modal
+      v-if="showImportModal"
+      :visible="showImportModal"
+      :title="$t('label.acl.import')"
+      :closable="true"
+      :maskClosable="false"
+      :footer="null"
+      :width="800"
+      @cancel="closeImportModal">
+      <import-network-a-c-l
+        :resource="resource"
+        @refresh-data="fetchData"
+        @close-action="closeImportModal" />
+    </a-modal>
   </a-spin>
 </template>
 
@@ -313,13 +334,15 @@ import { getAPI, postAPI } from '@/api'
 import draggable from 'vuedraggable'
 import { mixinForm } from '@/utils/mixin'
 import TooltipButton from '@/components/widgets/TooltipButton'
+import ImportNetworkACL from './ImportNetworkACL'
 
 export default {
   name: 'AclListRulesTab',
   mixins: [mixinForm],
   components: {
     draggable,
-    TooltipButton
+    TooltipButton,
+    ImportNetworkACL
   },
   props: {
     resource: {
@@ -344,6 +367,7 @@ export default {
       tagsModalVisible: false,
       tagsLoading: false,
       ruleModalVisible: false,
+      showImportModal: false,
       ruleModalTitle: this.$t('label.edit.rule'),
       ruleFormMode: 'edit'
     }
@@ -788,6 +812,12 @@ export default {
     },
     capitalise (val) {
       return val.toUpperCase()
+    },
+    handleImportRules () {
+      this.showImportModal = true
+    },
+    closeImportModal () {
+      this.showImportModal = false
     }
   }
 }

ui/src/views/network/ImportNetworkACL.vue

@@ -0,0 +1,381 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+<template>
+  <div v-ctrl-enter="handleSubmit">
+    <a-spin :spinning="loading">
+      <a-form
+        :ref="formRef"
+        :model="form"
+        :rules="rules"
+        layout="vertical"
+        @finish="handleSubmit">
+
+        <div class="info-section" style="margin-bottom: 24px; padding: 16px; background: #fafafa; border-radius: 4px;">
+          <a-descriptions :column="1" size="small">
+            <a-descriptions-item :label="$t('label.acl.id')">
+              <span style="font-family: monospace;">{{ resource.id }}</span>
+            </a-descriptions-item>
+            <a-descriptions-item :label="$t('label.add.acl.name')">
+              <strong>{{ resource.name }}</strong>
+            </a-descriptions-item>
+          </a-descriptions>
+        </div>
+
+        <a-form-item name="file" ref="file">
+          <template #label>
+            <tooltip-label :title="$t('label.rules.file')" :tooltip="$t('label.rules.file.to.import')"/>
+          </template>
+          <a-upload-dragger
+            :multiple=false
+            :fileList="fileList"
+            @remove="handleRemove"
+            :beforeUpload="beforeUpload"
+            @change="handleChange"
+            v-model:value="form.file">
+            <p class="ant-upload-drag-icon">
+              <cloud-upload-outlined />
+            </p>
+            <p class="ant-upload-text" v-if="fileList.length === 0">
+              {{ $t('label.rules.file.import.description') }}
+            </p>
+          </a-upload-dragger>
+        </a-form-item>
+
+        <a-form-item v-if="csvData.length > 0" :label="$t('label.csv.preview')">
+          <div class="csv-preview">
+            <a-table
+                :columns="columns"
+                :dataSource="csvData"
+                :pagination="{ pageSize: 5 }"
+                :scroll="{ x: true }"
+                size="small">
+
+                <template #action="{ record }">
+                  <a-tag :color="record.action && record.action.toLowerCase() === 'allow' ? 'green' : 'red'">
+                    {{ record.action ? record.action.toUpperCase() : 'N/A' }}
+                  </a-tag>
+                </template>
+
+                <template #traffictype="{ record }">
+                  <a-tag :color="record.traffictype && record.traffictype.toLowerCase() === 'ingress' ? 'blue' : 'orange'">
+                    {{ record.traffictype ? record.traffictype.toUpperCase() : 'N/A' }}
+                  </a-tag>
+                </template>
+
+            </a-table>
+          </div>
+        </a-form-item>
+
+        <span>{{ $t('message.network.acl.import.note') }}</span><br/>
+
+        <div :span="24" class="action-button">
+          <a-button class="button-cancel" @click="closeAction">{{ $t('label.cancel') }}</a-button>
+          <a-button
+            class="button-submit"
+            ref="submit"
+            type="primary"
+            :loading="loading"
+            :disabled="csvData.length === 0"
+            @click="handleSubmit">{{ $t('label.import') }}</a-button>
+        </div>
+
+      </a-form>
+    </a-spin>
+  </div>
+</template>
+
+<script>
+import { ref, reactive, toRaw } from 'vue'
+import { postAPI } from '@/api'
+import TooltipLabel from '@/components/widgets/TooltipLabel'
+
+export default {
+  name: 'ImportNetworkACL',
+  components: {
+    TooltipLabel
+  },
+  props: {
+    resource: {
+      type: Object,
+      required: true
+    }
+  },
+  data () {
+    return {
+      loading: false,
+      fileList: [],
+      csvData: '',
+      csvFileType: ['.csv', 'text/csv', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/vnd.ms-excel'],
+      columns: [
+        {
+          title: this.$t('label.protocol'),
+          dataIndex: 'protocol',
+          key: 'protocol',
+          width: 100
+        },
+        {
+          title: this.$t('label.action'),
+          dataIndex: 'action',
+          key: 'action',
+          width: 100,
+          slots: { customRender: 'action' }
+        },
+        {
+          title: this.$t('label.cidr'),
+          dataIndex: 'cidrlist',
+          width: 150,
+          ellipsis: true
+        },
+        {
+          title: this.$t('label.startport'),
+          dataIndex: 'startport',
+          width: 100
+        },
+        {
+          title: this.$t('label.endport'),
+          dataIndex: 'endport',
+          width: 100
+        },
+        {
+          title: this.$t('label.traffictype'),
+          dataIndex: 'traffictype',
+          key: 'traffictype',
+          width: 120,
+          slots: { customRender: 'traffictype' }
+        },
+        {
+          title: this.$t('label.number'),
+          dataIndex: 'number',
+          width: 80
+        },
+        {
+          title: this.$t('label.reason'),
+          dataIndex: 'reason',
+          ellipsis: true
+        }
+      ]
+    }
+  },
+  beforeCreate () {
+    this.apiParams = this.$getApiParams('importNetworkACL')
+  },
+  created () {
+    this.initForm()
+  },
+  methods: {
+    initForm () {
+      this.formRef = ref()
+      this.form = reactive({})
+      this.rules = reactive({
+        file: [
+          { required: true, message: this.$t('message.error.required.input') },
+          {
+            validator: this.checkCsvRulesFile,
+            message: this.$t('label.error.rules.file.import')
+          }
+        ]
+      })
+    },
+    beforeUpload (file) {
+      if (!this.csvFileType.includes(file.type)) {
+        return false
+      }
+      this.fileList = [file]
+      this.form.file = file
+      return false // Stop from uploading automatically
+    },
+    handleSubmit (e) {
+      e.preventDefault()
+      if (this.loading) return
+      this.formRef.value.validate().then(() => {
+        const values = toRaw(this.form)
+        const params = {}
+        for (const key in values) {
+          const input = values[key]
+          if (input === undefined) {
+            continue
+          }
+          if (key === 'file') {
+            continue
+          }
+          params[key] = input
+        }
+
+        if (this.csvData.length === 0) {
+          this.$message.error(this.$t('message.csv.no.data'))
+          return
+        }
+        this.importNetworkACL()
+      }).catch(error => {
+        this.formRef.value.scrollToField(error.errorFields[0].name)
+      })
+    },
+    handleRemove (file) {
+      const index = this.fileList.indexOf(file)
+      const newFileList = this.fileList.slice()
+      newFileList.splice(index, 1)
+      this.fileList = newFileList
+      this.form.file = undefined
+    },
+    handleChange (info) {
+      if (info.file.status === 'error') {
+        this.$notification.error({
+          message: this.$t('label.error.file.upload'),
+          description: this.$t('label.error.file.upload')
+        })
+      }
+    },
+    async checkCsvRulesFile (rule, value) {
+      if (!value || value === '') {
+        return Promise.resolve()
+      } else {
+        if (!this.csvFileType.includes(value.type)) {
+          return Promise.reject(rule.message)
+        }
+
+        try {
+          const validFile = await this.readCsvFile(value)
+          if (!validFile) {
+            return Promise.reject(rule.message)
+          } else {
+            return Promise.resolve()
+          }
+        } catch (reason) {
+          return Promise.reject(rule.message)
+        }
+      }
+    },
+    readCsvFile (file) {
+      return new Promise((resolve, reject) => {
+        if (window.FileReader) {
+          const reader = new FileReader()
+          reader.onload = (event) => {
+            const text = event.target.result
+            const lines = text.split('\n').filter(line => line.trim() !== '')
+            if (lines.length < 2) {
+              this.$message.error(this.$t('message.csv.empty'))
+              resolve(false)
+            }
+            const headers = this.parseCSVLine(lines[0])
+            const requiredHeaders = ['protocol', 'cidrlist', 'traffictype']
+            const missingHeaders = requiredHeaders.filter(h => !headers.includes(h.toLowerCase()))
+            if (missingHeaders.length > 0) {
+              this.$message.error(this.$t('message.csv.missing.headers') + ': ' + missingHeaders.join(', '))
+              resolve(false)
+            }
+            // Parse data rows
+            const data = []
+            for (let i = 1; i < lines.length; i++) {
+              const values = this.parseCSVLine(lines[i])
+              if (values.length === headers.length) {
+                const row = {}
+                headers.forEach((header, index) => {
+                  const value = values[index].trim()
+                  if (value !== '' && value !== 'null') {
+                    row[header.toLowerCase()] = value
+                  }
+                })
+                data.push(row)
+              }
+            }
+
+            this.csvData = data
+            resolve(true)
+          }
+
+          reader.onerror = (event) => {
+            if (event.target.error.name === 'NotReadableError') {
+              reject(event.target.error)
+            }
+          }
+
+          reader.readAsText(file)
+        } else {
+          reject(this.$t('label.error.file.read'))
+        }
+      })
+    },
+    parseCSVLine (line) {
+      const result = []
+      let current = ''
+      let inQuotes = false
+
+      for (let i = 0; i < line.length; i++) {
+        const char = line[i]
+        if (char === '"') {
+          inQuotes = !inQuotes
+        } else if (char === ',' && !inQuotes) {
+          result.push(current)
+          current = ''
+        } else {
+          current += char
+        }
+      }
+      result.push(current)
+
+      return result.map(v => v.trim())
+    },
+    closeAction () {
+      this.$emit('close-action')
+    },
+    importNetworkACL () {
+      this.loading = true
+      const params = {
+        aclid: this.resource.id
+      }
+      this.csvData.forEach(function (values, index) {
+        for (const key in values) {
+          params['rules[' + index + '].' + key] = values[key]
+        }
+      })
+      postAPI('importNetworkACL', params).then(response => {
+        this.$pollJob({
+          jobId: response.importnetworkaclresponse.jobid,
+          title: this.$t('message.success.add.network.acl'),
+          successMethod: () => {
+            this.loading = false
+          },
+          errorMessage: this.$t('message.add.network.acl.failed'),
+          errorMethod: () => {
+            this.loading = false
+          },
+          loadingMessage: this.$t('message.add.network.acl.processing'),
+          catchMessage: this.$t('error.fetching.async.job.result'),
+          catchMethod: () => {
+            this.loading = false
+          }
+        })
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        this.loading = false
+      })
+    }
+  }
+}
+</script>
+
+<style scoped lang="less">
+  .csv-preview {
+  max-height: 400px;
+  overflow: auto;
+  border: 1px solid #d9d9d9;
+  border-radius: 4px;
+  padding: 10px;
+}
+</style>

ui/src/views/storage/CreateTemplate.vue

@@ -73,42 +73,32 @@
           <template #label>
             <tooltip-label :title="$t('label.domainid')" :tooltip="apiParams.domainid.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.domainid"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
-            :loading="domainLoading"
+            api="listDomains"
+            :apiParams="domainsApiParams"
+            resourceType="domain"
+            optionValueKey="id"
+            optionLabelKey="path"
+            defaultIcon="block-outlined"
+            allowClear="true"
             :placeholder="apiParams.domainid.description"
-            @change="val => { handleDomainChange(val) }">
-            <a-select-option v-for="(opt, optIndex) in this.domains" :key="optIndex" :label="opt.path || opt.name || opt.description" :value="opt.id">
-              <span>
-                <resource-icon v-if="opt && opt.icon" :image="opt.icon.base64image" size="1x" style="margin-right: 5px"/>
-                <block-outlined v-else style="margin-right: 5px" />
-                {{ opt.path || opt.name || opt.description }}
-              </span>
-            </a-select-option>
-          </a-select>
+            @change-option-value="handleDomainChange" />
         </a-form-item>
         <a-form-item name="account" ref="account" v-if="domainid">
           <template #label>
             <tooltip-label :title="$t('label.account')" :tooltip="apiParams.account.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.account"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.value.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
-            :placeholder="apiParams.account.description"
-            @change="val => { handleAccountChange(val) }">
-            <a-select-option v-for="(acc, index) in accounts" :value="acc.name" :key="index">
-              {{ acc.name }}
-            </a-select-option>
-          </a-select>
+            api="listAccounts"
+            :apiParams="accountsApiParams"
+            resourceType="account"
+            optionValueKey="name"
+            optionLabelKey="name"
+            defaultIcon="team-outlined"
+            allowClear="true"
+            :placeholder="apiParams.account.description" />
         </a-form-item>
 
       <a-form-item
@@ -199,13 +189,15 @@ import { getAPI, postAPI } from '@/api'
 import { mixinForm } from '@/utils/mixin'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 export default {
   name: 'CreateTemplate',
   mixins: [mixinForm],
   components: {
     ResourceIcon,
-    TooltipLabel
+    TooltipLabel,
+    InfiniteScrollSelect
   },
   props: {
     resource: {
@@ -219,9 +211,6 @@ export default {
       zones: [],
       osTypes: {},
       loading: false,
-      domains: [],
-      accounts: [],
-      domainLoading: false,
       domainid: null,
       account: null,
       architectureTypes: {}
@@ -230,6 +219,21 @@ export default {
   computed: {
     isAdminRole () {
       return this.$store.getters.userInfo.roletype === 'Admin'
+    },
+    domainsApiParams () {
+      return {
+        listall: true,
+        showicon: true,
+        details: 'min'
+      }
+    },
+    accountsApiParams () {
+      if (!this.domainid) {
+        return null
+      }
+      return {
+        domainid: this.domainid
+      }
     }
   },
   beforeCreate () {
@@ -256,9 +260,6 @@ export default {
       if (this.resource.intervaltype) {
         this.fetchSnapshotZones()
       }
-      if ('listDomains' in this.$store.getters.apis) {
-        this.fetchDomains()
-      }
       this.architectureTypes.opts = this.$fetchCpuArchitectureTypes()
     },
     fetchOsTypes () {
@@ -309,44 +310,16 @@ export default {
         }
       })
     },
-    fetchDomains () {
-      const params = {}
-      params.listAll = true
-      params.showicon = true
-      params.details = 'min'
-      this.domainLoading = true
-      getAPI('listDomains', params).then(json => {
-        this.domains = json.listdomainsresponse.domain
-      }).finally(() => {
-        this.domainLoading = false
-        this.handleDomainChange(null)
-      })
-    },
-    async handleDomainChange (domain) {
-      this.domainid = domain
+    handleDomainChange (domainId) {
+      this.domainid = domainId
       this.form.account = null
       this.account = null
-      if ('listAccounts' in this.$store.getters.apis) {
-        await this.fetchAccounts()
-      }
     },
-    fetchAccounts () {
-      return new Promise((resolve, reject) => {
-        getAPI('listAccounts', {
-          domainid: this.domainid
-        }).then(response => {
-          this.accounts = response?.listaccountsresponse?.account || []
-          resolve(this.accounts)
-        }).catch(error => {
-          this.$notifyError(error)
-        })
-      })
-    },
-    handleAccountChange (acc) {
-      if (acc) {
-        this.account = acc.name
+    handleAccountChange (accountName) {
+      if (accountName) {
+        this.account = accountName
       } else {
-        this.account = acc
+        this.account = null
       }
     },
     handleSubmit (e) {

ui/src/views/storage/UploadLocalVolume.vue

@@ -57,43 +57,33 @@
           <template #label>
             <tooltip-label :title="$t('label.zoneid')" :tooltip="apiParams.zoneid.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.zoneId"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }" >
-            <a-select-option :value="zone.id" v-for="zone in zones" :key="zone.id" :label="zone.name || zone.description">
-              <span>
-                <resource-icon v-if="zone.icon" :image="zone.icon.base64image" size="1x" style="margin-right: 5px"/>
-                <global-outlined v-else style="margin-right: 5px"/>
-                {{ zone.name || zone.description }}
-              </span>
-            </a-select-option>
-          </a-select>
+            api="listZones"
+            :apiParams="zonesApiParams"
+            resourceType="zone"
+            optionValueKey="id"
+            optionLabelKey="name"
+            defaultIcon="global-outlined"
+            selectFirstOption="true"
+            @change-option-value="handleZoneChange" />
         </a-form-item>
         <a-form-item name="diskofferingid" ref="diskofferingid">
           <template #label>
             <tooltip-label :title="$t('label.diskofferingid')" :tooltip="apiParams.diskofferingid.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.diskofferingid"
-            :loading="offeringLoading"
+            api="listDiskOfferings"
+            :apiParams="diskOfferingsApiParams"
+            resourceType="diskoffering"
+            optionValueKey="id"
+            optionLabelKey="displaytext"
+            defaultIcon="hdd-outlined"
+            :defaultOption="{ id: null, displaytext: ''}"
+            allowClear="true"
             :placeholder="apiParams.diskofferingid.description"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }" >
-            <a-select-option
-              v-for="(offering, index) in offerings"
-              :value="offering.id"
-              :key="index"
-              :label="offering.displaytext || offering.name">
-              {{ offering.displaytext || offering.name }}
-            </a-select-option>
-          </a-select>
+            @change-option="onChangeDiskOffering" />
         </a-form-item>
         <a-form-item ref="format" name="format">
           <template #label>
@@ -124,38 +114,33 @@
           <template #label>
             <tooltip-label :title="$t('label.domain')" :tooltip="apiParams.domainid.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.domainid"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
-            :loading="domainLoading"
+            api="listDomains"
+            :apiParams="domainsApiParams"
+            resourceType="domain"
+            optionValueKey="id"
+            optionLabelKey="path"
+            defaultIcon="block-outlined"
             :placeholder="$t('label.domainid')"
-            @change="val => { handleDomainChange(domainList[val].id) }">
-            <a-select-option v-for="(opt, optIndex) in domainList" :key="optIndex" :label="opt.path || opt.name || opt.description">
-              {{ opt.path || opt.name || opt.description }}
-            </a-select-option>
-          </a-select>
+            allowClear="true"
+            @change-option-value="handleDomainChange" />
         </a-form-item>
         <a-form-item name="account" ref="account" v-if="'listDomains' in $store.getters.apis">
           <template #label>
             <tooltip-label :title="$t('label.account')" :tooltip="apiParams.account.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.account"
-            showSearch
-            optionFilterProp="value"
-            :filterOption="(input, option) => {
-              return option.value.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
+            api="listAccounts"
+            :apiParams="accountsApiParams"
+            resourceType="account"
+            optionValueKey="name"
+            optionLabelKey="name"
+            defaultIcon="team-outlined"
+            allowClear="true"
             :placeholder="$t('label.account')"
-            @change="val => { handleAccountChange(val) }">
-            <a-select-option v-for="(acc, index) in accountList" :value="acc.name" :key="index">
-              {{ acc.name }}
-            </a-select-option>
-          </a-select>
+            @change-option-value="handleAccountChange" />
         </a-form-item>
         <div :span="24" class="action-button">
           <a-button @click="closeAction">{{ $t('label.cancel') }}</a-button>
@@ -173,27 +158,25 @@ import { axios } from '../../utils/request'
 import { mixinForm } from '@/utils/mixin'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 export default {
   name: 'UploadLocalVolume',
   mixins: [mixinForm],
   components: {
     ResourceIcon,
-    TooltipLabel
+    TooltipLabel,
+    InfiniteScrollSelect
   },
   data () {
     return {
       fileList: [],
-      zones: [],
-      domainList: [],
-      accountList: [],
-      offerings: [],
-      offeringLoading: false,
       formats: ['RAW', 'VHD', 'VHDX', 'OVA', 'QCOW2'],
       domainId: null,
       account: null,
       uploadParams: null,
-      domainLoading: false,
+      customDiskOffering: false,
+      isCustomizedDiskIOps: false,
       loading: false,
       uploadPercentage: 0
     }
@@ -201,9 +184,38 @@ export default {
   beforeCreate () {
     this.apiParams = this.$getApiParams('getUploadParamsForVolume')
   },
+  computed: {
+    zonesApiParams () {
+      return {
+        showicon: true
+      }
+    },
+    diskOfferingsApiParams () {
+      if (!this.form.zoneId) {
+        return null
+      }
+      return {
+        zoneid: this.form.zoneId,
+        listall: true
+      }
+    },
+    domainsApiParams () {
+      return {
+        listall: true,
+        details: 'min'
+      }
+    },
+    accountsApiParams () {
+      if (!this.form.domainid) {
+        return null
+      }
+      return {
+        domainid: this.form.domainid
+      }
+    }
+  },
   created () {
     this.initForm()
-    this.fetchData()
   },
   methods: {
     initForm () {
@@ -221,38 +233,18 @@ export default {
         zoneId: [{ required: true, message: this.$t('message.error.select') }]
       })
     },
-    listZones () {
-      getAPI('listZones', { showicon: true }).then(json => {
-        if (json && json.listzonesresponse && json.listzonesresponse.zone) {
-          this.zones = json.listzonesresponse.zone
-          this.zones = this.zones.filter(zone => zone.type !== 'Edge')
-          if (this.zones.length > 0) {
-            this.onZoneChange(this.zones[0].id)
-          }
-        }
-      })
-    },
-    onZoneChange (zoneId) {
+    handleZoneChange (zoneId) {
       this.form.zoneId = zoneId
-      this.zoneId = zoneId
-      this.fetchDiskOfferings(zoneId)
+      // InfiniteScrollSelect will auto-reload disk offerings when apiParams changes
     },
-    fetchDiskOfferings (zoneId) {
-      this.offeringLoading = true
-      this.offerings = [{ id: -1, name: '' }]
-      this.form.diskofferingid = undefined
-      getAPI('listDiskOfferings', {
-        zoneid: zoneId,
-        listall: true
-      }).then(json => {
-        for (var offering of json.listdiskofferingsresponse.diskoffering) {
-          if (offering.iscustomized) {
-            this.offerings.push(offering)
-          }
-        }
-      }).finally(() => {
-        this.offeringLoading = false
-      })
+    onChangeDiskOffering (offering) {
+      if (offering) {
+        this.customDiskOffering = offering.iscustomized || false
+        this.isCustomizedDiskIOps = offering.iscustomizediops || false
+      } else {
+        this.customDiskOffering = false
+        this.isCustomizedDiskIOps = false
+      }
     },
     handleRemove (file) {
       const index = this.fileList.indexOf(file)
@@ -266,53 +258,14 @@ export default {
       this.form.file = file
       return false
     },
-    handleDomainChange (domain) {
-      this.domainId = domain
-      if ('listAccounts' in this.$store.getters.apis) {
-        this.fetchAccounts()
-      }
+    handleDomainChange (domainId) {
+      this.form.domainid = domainId
+      this.domainId = domainId
+      this.form.account = null
     },
-    handleAccountChange (acc) {
-      if (acc) {
-        this.account = acc.name
-      } else {
-        this.account = acc
-      }
-    },
-    fetchData () {
-      this.listZones()
-      if ('listDomains' in this.$store.getters.apis) {
-        this.fetchDomains()
-      }
-    },
-    fetchDomains () {
-      this.domainLoading = true
-      getAPI('listDomains', {
-        listAll: true,
-        details: 'min'
-      }).then(response => {
-        this.domainList = response.listdomainsresponse.domain
-
-        if (this.domainList[0]) {
-          this.handleDomainChange(null)
-        }
-      }).catch(error => {
-        this.$notifyError(error)
-      }).finally(() => {
-        this.domainLoading = false
-      })
-    },
-    fetchAccounts () {
-      getAPI('listAccounts', {
-        domainid: this.domainId
-      }).then(response => {
-        this.accountList = response.listaccountsresponse.account || []
-        if (this.accountList && this.accountList.length === 0) {
-          this.handleAccountChange(null)
-        }
-      }).catch(error => {
-        this.$notifyError(error)
-      })
+    handleAccountChange (accountName) {
+      this.form.account = accountName
+      this.account = accountName
     },
     handleSubmit (e) {
       e.preventDefault()

ui/src/views/storage/UploadVolume.vue

@@ -47,21 +47,16 @@
           <template #label>
             <tooltip-label :title="$t('label.zoneid')" :tooltip="apiParams.zoneid.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.zoneId"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }" >
-            <a-select-option :value="zone.id" v-for="zone in zones" :key="zone.id" :label="zone.name || zone.description">
-              <span>
-                <resource-icon v-if="zone.icon" :image="zone.icon.base64image" size="1x" style="margin-right: 5px"/>
-                <global-outlined v-else style="margin-right: 5px"/>
-                {{ zone.name || zone.description }}
-              </span>
-            </a-select-option>
-          </a-select>
+            api="listZones"
+            :apiParams="zonesApiParams"
+            resourceType="zone"
+            optionValueKey="id"
+            optionLabelKey="name"
+            defaultIcon="global-outlined"
+            selectFirstOption="true"
+            @change-option-value="handleZoneChange" />
         </a-form-item>
         <a-form-item name="format" ref="format">
           <template #label>
@@ -83,23 +78,17 @@
           <template #label>
             <tooltip-label :title="$t('label.diskofferingid')" :tooltip="apiParams.diskofferingid.description || $t('label.diskoffering')"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.diskofferingid"
-            :loading="loading"
-            @change="id => onChangeDiskOffering(id)"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }" >
-            <a-select-option
-              v-for="(offering, index) in offerings"
-              :value="offering.id"
-              :key="index"
-              :label="offering.displaytext || offering.name">
-              {{ offering.displaytext || offering.name }}
-            </a-select-option>
-          </a-select>
+            api="listDiskOfferings"
+            :apiParams="diskOfferingsApiParams"
+            resourceType="diskoffering"
+            optionValueKey="id"
+            optionLabelKey="displaytext"
+            defaultIcon="hdd-outlined"
+            :defaultOption="{ id: null, displaytext: ''}"
+            allowClear="true"
+            @change-option="onChangeDiskOffering" />
         </a-form-item>
         <a-form-item name="checksum" ref="checksum">
           <template #label>
@@ -114,38 +103,33 @@
           <template #label>
             <tooltip-label :title="$t('label.domain')" :tooltip="apiParams.domainid.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.domainid"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
-            :loading="domainLoading"
+            api="listDomains"
+            :apiParams="domainsApiParams"
+            resourceType="domain"
+            optionValueKey="id"
+            optionLabelKey="path"
+            defaultIcon="block-outlined"
+            allowClear="true"
             :placeholder="$t('label.domainid')"
-            @change="val => { handleDomainChange(domainList[val].id) }">
-            <a-select-option v-for="(opt, optIndex) in domainList" :key="optIndex" :label="opt.path || opt.name || opt.description">
-              {{ opt.path || opt.name || opt.description }}
-            </a-select-option>
-          </a-select>
+            @change-option-value="handleDomainChange" />
         </a-form-item>
         <a-form-item name="account" ref="account" v-if="'listDomains' in $store.getters.apis">
           <template #label>
             <tooltip-label :title="$t('label.account')" :tooltip="apiParams.account.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.account"
-            showSearch
-            optionFilterProp="value"
-            :filterOption="(input, option) => {
-              return option.value.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
+            api="listAccounts"
+            :apiParams="accountsApiParams"
+            resourceType="account"
+            optionValueKey="name"
+            optionLabelKey="name"
+            defaultIcon="team-outlined"
             :placeholder="$t('label.account')"
-            @change="val => { handleAccountChange(val) }">
-            <a-select-option v-for="(acc, index) in accountList" :value="acc.name" :key="index">
-              {{ acc.name }}
-            </a-select-option>
-          </a-select>
+            allowClear="true"
+            @change-option-value="handleAccountChange" />
         </a-form-item>
         <div :span="24" class="action-button">
           <a-button @click="closeAction">{{ $t('label.cancel') }}</a-button>
@@ -158,31 +142,30 @@
 
 <script>
 import { ref, reactive, toRaw } from 'vue'
-import { getAPI, postAPI } from '@/api'
+import { postAPI } from '@/api'
 import { mixinForm } from '@/utils/mixin'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 export default {
   name: 'UploadVolume',
   mixins: [mixinForm],
   components: {
     ResourceIcon,
-    TooltipLabel
+    TooltipLabel,
+    InfiniteScrollSelect
   },
   data () {
     return {
-      zones: [],
-      domainList: [],
-      accountList: [],
       formats: ['RAW', 'VHD', 'VHDX', 'OVA', 'QCOW2'],
-      offerings: [],
       zoneSelected: '',
       selectedDiskOfferingId: null,
       domainId: null,
       account: null,
       uploadParams: null,
-      domainLoading: false,
+      customDiskOffering: false,
+      isCustomizedDiskIOps: false,
       loading: false,
       uploadPercentage: 0
     }
@@ -190,9 +173,36 @@ export default {
   beforeCreate () {
     this.apiParams = this.$getApiParams('uploadVolume')
   },
+  computed: {
+    zonesApiParams () {
+      return {
+        showicon: true
+      }
+    },
+    diskOfferingsApiParams () {
+      if (!this.form.zoneId) {
+        return null
+      }
+      return {
+        zoneid: this.form.zoneId,
+        listall: true
+      }
+    },
+    domainsApiParams () {
+      return {
+        listall: true,
+        details: 'min'
+      }
+    },
+    accountsApiParams () {
+      return {
+        domainid: this.form?.domainid || null,
+        showicon: true
+      }
+    }
+  },
   created () {
     this.initForm()
-    this.fetchData()
   },
   methods: {
     initForm () {
@@ -207,78 +217,28 @@ export default {
         format: [{ required: true, message: this.$t('message.error.select') }]
       })
     },
-    fetchData () {
-      this.loading = true
-      getAPI('listZones', { showicon: true }).then(json => {
-        this.zones = json.listzonesresponse.zone || []
-        this.zones = this.zones.filter(zone => zone.type !== 'Edge')
-        this.form.zoneId = this.zones[0].id || ''
-        this.fetchDiskOfferings(this.form.zoneId)
-      }).finally(() => {
-        this.loading = false
-      })
-      if ('listDomains' in this.$store.getters.apis) {
-        this.fetchDomains()
-      }
+    handleZoneChange (zoneId) {
+      this.form.zoneId = zoneId
+      // InfiniteScrollSelect will auto-reload disk offerings when apiParams changes
     },
-    fetchDiskOfferings (zoneId) {
-      this.loading = true
-      getAPI('listDiskOfferings', {
-        zoneid: zoneId,
-        listall: true
-      }).then(json => {
-        this.offerings = json.listdiskofferingsresponse.diskoffering || []
-      }).finally(() => {
-        this.loading = false
-      })
-    },
-    fetchDomains () {
-      this.domainLoading = true
-      getAPI('listDomains', {
-        listAll: true,
-        details: 'min'
-      }).then(response => {
-        this.domainList = response.listdomainsresponse.domain
-
-        if (this.domainList[0]) {
-          this.handleDomainChange(null)
-        }
-      }).catch(error => {
-        this.$notifyError(error)
-      }).finally(() => {
-        this.domainLoading = false
-      })
-    },
-    fetchAccounts () {
-      getAPI('listAccounts', {
-        domainid: this.domainId
-      }).then(response => {
-        this.accountList = response.listaccountsresponse.account || []
-        if (this.accountList && this.accountList.length === 0) {
-          this.handleAccountChange(null)
-        }
-      }).catch(error => {
-        this.$notifyError(error)
-      })
-    },
-    onChangeDiskOffering (id) {
-      const offering = this.offerings.filter(x => x.id === id)
-      this.customDiskOffering = offering[0]?.iscustomized || false
-      this.isCustomizedDiskIOps = offering[0]?.iscustomizediops || false
-    },
-    handleDomainChange (domain) {
-      this.domainId = domain
-      if ('listAccounts' in this.$store.getters.apis) {
-        this.fetchAccounts()
-      }
-    },
-    handleAccountChange (acc) {
-      if (acc) {
-        this.account = acc.name
+    onChangeDiskOffering (offering) {
+      if (offering) {
+        this.customDiskOffering = offering.iscustomized || false
+        this.isCustomizedDiskIOps = offering.iscustomizediops || false
       } else {
-        this.account = acc
+        this.customDiskOffering = false
+        this.isCustomizedDiskIOps = false
       }
     },
+    handleDomainChange (domainId) {
+      this.form.domainid = domainId
+      this.domainId = domainId
+      this.form.account = null
+    },
+    handleAccountChange (accountName) {
+      this.form.account = accountName
+      this.account = accountName
+    },
     handleSubmit (e) {
       e.preventDefault()
       if (this.loading) return

ui/src/views/tools/CreateWebhook.vue

@@ -67,39 +67,33 @@
               <info-circle-outlined style="color: rgba(0,0,0,.45)" />
             </a-tooltip>
           </template>
-          <a-select
+          <infinite-scroll-select
             id="domain-selection"
             v-model:value="form.domainid"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
-            :loading="domainLoading"
+            api="listDomains"
+            :apiParams="domainsApiParams"
+            resourceType="domain"
+            optionValueKey="id"
+            optionLabelKey="path"
+            defaultIcon="block-outlined"
+            :defaultOption="{ id: null, path: ''}"
+            allowClear="true"
             :placeholder="apiParams.domainid.description"
-            @change="val => { handleDomainChanged(val) }">
-            <a-select-option v-for="opt in domains" :key="opt.id" :label="opt.path || opt.name || opt.description || ''">
-              {{ opt.path || opt.name || opt.description }}
-            </a-select-option>
-          </a-select>
+            @change-option-value="handleDomainChanged" />
         </a-form-item>
         <a-form-item name="account" ref="account" v-if="isAdminOrDomainAdmin && ['Local'].includes(form.scope) && form.domainid">
           <template #label>
             <tooltip-label :title="$t('label.account')" :tooltip="apiParams.account.description"/>
           </template>
-          <a-select
+          <infinite-scroll-select
             v-model:value="form.account"
-            showSearch
-            optionFilterProp="label"
-            :filterOption="(input, option) => {
-              return option.value.toLowerCase().indexOf(input.toLowerCase()) >= 0
-            }"
-            :loading="accountLoading"
-            :placeholder="apiParams.account.description">
-            <a-select-option v-for="opt in accounts" :key="opt.id" :label="opt.name">
-              {{ opt.name }}
-            </a-select-option>
-          </a-select>
+            api="listAccounts"
+            :apiParams="accountsApiParams"
+            resourceType="account"
+            optionValueKey="name"
+            optionLabelKey="name"
+            defaultIcon="team-outlined"
+            :placeholder="apiParams.account.description" />
         </a-form-item>
         <a-form-item name="payloadurl" ref="payloadurl">
           <template #label>
@@ -155,26 +149,23 @@
 
 <script>
 import { ref, reactive, toRaw } from 'vue'
-import { getAPI, postAPI } from '@/api'
-import _ from 'lodash'
+import { postAPI } from '@/api'
 import { mixinForm } from '@/utils/mixin'
 import TooltipLabel from '@/components/widgets/TooltipLabel'
 import TestWebhookDeliveryView from '@/components/view/TestWebhookDeliveryView'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 export default {
   name: 'CreateWebhook',
   mixins: [mixinForm],
   components: {
     TooltipLabel,
-    TestWebhookDeliveryView
+    TestWebhookDeliveryView,
+    InfiniteScrollSelect
   },
   props: {},
   data () {
     return {
-      domains: [],
-      domainLoading: false,
-      accounts: [],
-      accountLoading: false,
       loading: false,
       testDeliveryAllowed: false,
       testDeliveryLoading: false
@@ -185,9 +176,6 @@ export default {
   },
   created () {
     this.initForm()
-    if (['Domain', 'Local'].includes(this.form.scope)) {
-      this.fetchDomainData()
-    }
   },
   computed: {
     isAdminOrDomainAdmin () {
@@ -201,6 +189,21 @@ export default {
         return this.form.payloadurl.toLowerCase().startsWith('https://')
       }
       return false
+    },
+    domainsApiParams () {
+      return {
+        listAll: true,
+        showicon: true,
+        details: 'min'
+      }
+    },
+    accountsApiParams () {
+      if (!this.form.domainid) {
+        return null
+      }
+      return {
+        domainid: this.form.domainid
+      }
     }
   },
   methods: {
@@ -228,46 +231,6 @@ export default {
     updateTestDeliveryLoading (value) {
       this.testDeliveryLoading = value
     },
-    fetchDomainData () {
-      this.domainLoading = true
-      this.domains = [
-        {
-          id: null,
-          name: ''
-        }
-      ]
-      this.form.domainid = null
-      this.form.account = null
-      getAPI('listDomains', {}).then(json => {
-        const listdomains = json.listdomainsresponse.domain
-        this.domains = this.domains.concat(listdomains)
-      }).finally(() => {
-        this.domainLoading = false
-        if (this.arrayHasItems(this.domains)) {
-          this.form.domainid = null
-        }
-      })
-    },
-    fetchAccountData () {
-      this.accounts = []
-      this.form.account = null
-      if (!this.form.domainid) {
-        return
-      }
-      this.accountLoading = true
-      var params = {
-        domainid: this.form.domainid
-      }
-      getAPI('listAccounts', params).then(json => {
-        const listAccounts = json.listaccountsresponse.account || []
-        this.accounts = listAccounts
-      }).finally(() => {
-        this.accountLoading = false
-        if (this.arrayHasItems(this.accounts)) {
-          this.form.account = this.accounts[0].id
-        }
-      })
-    },
     handleSubmit (e) {
       e.preventDefault()
       if (this.loading) return
@@ -300,10 +263,8 @@ export default {
           return
         }
         if (values.account) {
-          const accountItem = _.find(this.accounts, (option) => option.id === values.account)
-          if (accountItem) {
-            params.account = accountItem.name
-          }
+          // values.account is the account name (optionValueKey="name")
+          params.account = values.account
         }
         this.loading = true
         postAPI('createWebhook', params).then(json => {
@@ -331,14 +292,11 @@ export default {
       }, 1)
     },
     handleScopeChange (e) {
-      if (['Domain', 'Local'].includes(this.form.scope)) {
-        this.fetchDomainData()
-      }
+      this.form.domainid = null
+      this.form.account = null
     },
     handleDomainChanged (domainid) {
-      if (domainid) {
-        this.fetchAccountData()
-      }
+      this.form.account = null
     }
   }
 }

ui/src/views/tools/ManageVolumes.vue

@@ -372,22 +372,16 @@
               name="domain"
               ref="domain"
               :label="$t('label.domain')">
-              <a-select
-                @change="changeDomain"
+              <infinite-scroll-select
                 v-model:value="importForm.selectedDomain"
-                showSearch
-                optionFilterProp="label"
-                :filterOption="(input, option) => {
-                  return  option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-                }" >
-                <a-select-option v-for="domain in domains" :key="domain.name" :value="domain.id" :label="domain.path || domain.name || domain.description">
-                <span>
-                  <resource-icon v-if="domain && domain.icon" :image="domain.icon.base64image" size="1x" style="margin-right: 5px"/>
-                  <block-outlined v-else style="margin-right: 5px" />
-                  {{ domain.path || domain.name || domain.description }}
-                </span>
-                </a-select-option>
-              </a-select>
+                api="listDomains"
+                :apiParams="domainsApiParams"
+                resourceType="domain"
+                optionValueKey="id"
+                optionLabelKey="path"
+                defaultIcon="block-outlined"
+                allowClear="true"
+                @change-option-value="changeDomain" />
             </a-form-item>
 
             <a-form-item
@@ -395,22 +389,16 @@
               name="account"
               ref="account"
               :label="$t('label.account')">
-              <a-select
-                @change="changeAccount"
+              <infinite-scroll-select
                 v-model:value="importForm.selectedAccount"
-                showSearch
-                optionFilterProp="value"
-                :filterOption="(input, option) => {
-                    return option.value.toLowerCase().indexOf(input.toLowerCase()) >= 0
-                  }" >
-                <a-select-option v-for="account in accounts" :key="account.name" :value="account.name">
-                  <span>
-                    <resource-icon v-if="account && account.icon" :image="account.icon.base64image" size="1x" style="margin-right: 5px"/>
-                    <team-outlined v-else style="margin-right: 5px" />
-                    {{ account.name }}
-                  </span>
-                </a-select-option>
-              </a-select>
+                api="listAccounts"
+                :apiParams="accountsApiParams"
+                resourceType="account"
+                optionValueKey="name"
+                optionLabelKey="name"
+                defaultIcon="team-outlined"
+                allowClear="true"
+                @change-option-value="changeAccount" />
               <span v-if="importForm.accountError" class="required">{{ $t('label.required') }}</span>
             </a-form-item>
 
@@ -419,22 +407,16 @@
               name="project"
               ref="project"
               :label="$t('label.project')">
-              <a-select
-                @change="changeProject"
+              <infinite-scroll-select
                 v-model:value="importForm.selectedProject"
-                showSearch
-                optionFilterProp="label"
-                :filterOption="(input, option) => {
-                  return  option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
-                }" >
-                <a-select-option v-for="project in projects" :key="project.id" :value="project.id" :label="project.name">
-                <span>
-                  <resource-icon v-if="project && project.icon" :image="project.icon.base64image" size="1x" style="margin-right: 5px"/>
-                  <project-outlined v-else style="margin-right: 5px" />
-                  {{ project.name }}
-                </span>
-                </a-select-option>
-              </a-select>
+                api="listProjects"
+                :apiParams="projectsApiParams"
+                resourceType="project"
+                optionValueKey="id"
+                optionLabelKey="name"
+                defaultIcon="project-outlined"
+                allowClear="true"
+                @change-option-value="changeProject" />
               <span v-if="importForm.projectError" class="required">{{ $t('label.required') }}</span>
             </a-form-item>
 
@@ -480,6 +462,7 @@ import Status from '@/components/widgets/Status'
 import SearchView from '@/components/view/SearchView'
 import ResourceIcon from '@/components/view/ResourceIcon'
 import TooltipLabel from '@/components/widgets/TooltipLabel.vue'
+import InfiniteScrollSelect from '@/components/widgets/InfiniteScrollSelect.vue'
 
 export default {
   components: {
@@ -487,7 +470,8 @@ export default {
     Breadcrumb,
     Status,
     SearchView,
-    ResourceIcon
+    ResourceIcon,
+    InfiniteScrollSelect
   },
   name: 'ManageVolumes',
   data () {
@@ -607,7 +591,6 @@ export default {
     this.page.managed = parseInt(this.$route.query.managedpage || 1)
     this.initForm()
     this.fetchData()
-    this.fetchDomains()
   },
   computed: {
     isPageAllowed () {
@@ -629,6 +612,36 @@ export default {
     showCluster () {
       return this.poolscope !== 'zone'
     },
+    domainsApiParams () {
+      return {
+        listall: true,
+        details: 'min',
+        showicon: true
+      }
+    },
+    accountsApiParams () {
+      if (!this.importForm.selectedDomain) {
+        return null
+      }
+      return {
+        domainid: this.importForm.selectedDomain,
+        showicon: true,
+        state: 'Enabled',
+        isrecursive: false
+      }
+    },
+    projectsApiParams () {
+      if (!this.importForm.selectedDomain) {
+        return null
+      }
+      return {
+        domainid: this.importForm.selectedDomain,
+        state: 'Active',
+        showicon: true,
+        details: 'min',
+        isrecursive: false
+      }
+    },
     showHost () {
       return this.poolscope === 'host'
     },
@@ -970,53 +983,6 @@ export default {
       this.updateQuery('scope', value)
       this.fetchOptions(this.params.zones, 'zones', value)
     },
-    fetchDomains () {
-      getAPI('listDomains', {
-        response: 'json',
-        listAll: true,
-        showicon: true,
-        details: 'min'
-      }).then(response => {
-        this.domains = response.listdomainsresponse.domain || []
-      }).catch(error => {
-        this.$notifyError(error)
-      }).finally(() => {
-        this.loading = false
-      })
-    },
-    fetchAccounts () {
-      this.loading = true
-      getAPI('listAccounts', {
-        response: 'json',
-        domainId: this.importForm.selectedDomain,
-        showicon: true,
-        state: 'Enabled',
-        isrecursive: false
-      }).then(response => {
-        this.accounts = response.listaccountsresponse.account || []
-      }).catch(error => {
-        this.$notifyError(error)
-      }).finally(() => {
-        this.loading = false
-      })
-    },
-    fetchProjects () {
-      this.loading = true
-      getAPI('listProjects', {
-        response: 'json',
-        domainId: this.importForm.selectedDomain,
-        state: 'Active',
-        showicon: true,
-        details: 'min',
-        isrecursive: false
-      }).then(response => {
-        this.projects = response.listprojectsresponse.project || []
-      }).catch(error => {
-        this.$notifyError(error)
-      }).finally(() => {
-        this.loading = false
-      })
-    },
     changeAccountType () {
       this.importForm.selectedDomain = null
       this.importForm.selectedAccount = null
@@ -1029,8 +995,7 @@ export default {
       this.importForm.selectedProject = null
       this.importForm.selectedDiskoffering = null
       this.diskOfferings = {}
-      this.fetchAccounts()
-      this.fetchProjects()
+      // InfiniteScrollSelect will auto-reload when apiParams changes
     },
     changeAccount () {
       this.importForm.selectedProject = null

usage/src/main/java/com/cloud/usage/UsageManagerImpl.java

@@ -2168,6 +2168,11 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna
                         }
                     }
 
+                    if (_usageJobDao.getNextRecurringJob() == null) {
+                        // Own the usage processing immediately if no other node is owning it
+                        _usageJobDao.createNewJob(_hostname, _pid, UsageJobVO.JOB_TYPE_RECURRING);
+                    }
+
                     Long jobId = _usageJobDao.checkHeartbeat(_hostname, _pid, _aggregationDuration);
                     if (jobId != null) {
                         // if I'm taking over the job...see how long it's been since the last job, and if it's more than the

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -29,8 +29,10 @@ import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
@@ -42,7 +44,6 @@ import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import org.apache.cloudstack.utils.security.KeyStoreUtils;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.LogManager;
@@ -66,8 +67,8 @@ public class Script implements Callable<String> {
     private static final int DEFAULT_TIMEOUT = 3600 * 1000; /* 1 hour */
     private volatile boolean _isTimeOut = false;
 
-    private boolean _passwordCommand = false;
     private boolean avoidLoggingCommand = false;
+    private final Set<Integer> sensitiveArgIndices = new HashSet<>();
 
     private static final ScheduledExecutorService s_executors = Executors.newScheduledThreadPool(10, new NamedThreadFactory("Script"));
 
@@ -145,6 +146,11 @@ public class Script implements Callable<String> {
         _command.add(param);
     }
 
+    public void addSensitive(String param) {
+        _command.add(param);
+        sensitiveArgIndices.add(_command.size() - 1);
+    }
+
     public Script set(String name, String value) {
         _command.add(name);
         _command.add(value);
@@ -163,7 +169,7 @@ public class Script implements Callable<String> {
             if (sanitizeViCmdParameter(cmd, builder) || sanitizeRbdFileFormatCmdParameter(cmd, builder)) {
                 continue;
             }
-            if (obscureParam) {
+            if (obscureParam || sensitiveArgIndices.contains(i)) {
                 builder.append("******").append(" ");
                 obscureParam = false;
             } else {
@@ -172,7 +178,6 @@ public class Script implements Callable<String> {
 
             if ("-y".equals(cmd) || "-z".equals(cmd)) {
                 obscureParam = true;
-                _passwordCommand = true;
             }
         }
         return builder.toString();
@@ -240,8 +245,8 @@ public class Script implements Callable<String> {
     public String execute(OutputInterpreter interpreter) {
         String[] command = _command.toArray(new String[_command.size()]);
         String commandLine = buildCommandLine(command);
-        if (_logger.isDebugEnabled() && !avoidLoggingCommand) {
-            _logger.debug(String.format("Executing command [%s].", commandLine.split(KeyStoreUtils.KS_FILENAME)[0]));
+        if (_logger.isDebugEnabled() ) {
+            _logger.debug(String.format("Executing command [%s].", commandLine));
         }
 
         try {
@@ -264,48 +269,62 @@ public class Script implements Callable<String> {
             _thread = Thread.currentThread();
             ScheduledFuture<String> future = null;
             if (_timeout > 0) {
-                _logger.trace(String.format("Scheduling the execution of command [%s] with a timeout of [%s] milliseconds.", commandLine, _timeout));
+                _logger.trace(String.format(
+                        "Scheduling the execution of command [%s] with a timeout of [%s] milliseconds.",
+                        commandLine, _timeout));
                 future = s_executors.schedule(this, _timeout, TimeUnit.MILLISECONDS);
             }
 
             long processPid = _process.pid();
             Task task = null;
             if (interpreter != null && interpreter.drain()) {
-                _logger.trace(String.format("Executing interpreting task of process [%s] for command [%s].", processPid, commandLine));
+                _logger.trace(String.format("Executing interpreting task of process [%s] for command [%s].",
+                        processPid, commandLine));
                 task = new Task(interpreter, ir);
                 s_executors.execute(task);
             }
 
             while (true) {
-                _logger.trace(String.format("Attempting process [%s] execution for command [%s] with timeout [%s].", processPid, commandLine, _timeout));
+                _logger.trace(String.format("Attempting process [%s] execution for command [%s] with timeout [%s].",
+                        processPid, commandLine, _timeout));
                 try {
                     if (_process.waitFor(_timeout, TimeUnit.MILLISECONDS)) {
-                        _logger.trace(String.format("Process [%s] execution for command [%s] completed within timeout period [%s].", processPid, commandLine,
+                        _logger.trace(String.format(
+                                "Process [%s] execution for command [%s] completed within timeout period [%s].",
+                                processPid, commandLine,
                                 _timeout));
                         if (_process.exitValue() == 0) {
-                            _logger.debug(String.format("Successfully executed process [%s] for command [%s].", processPid, commandLine));
+                            _logger.debug(String.format("Successfully executed process [%s] for command [%s].",
+                                    processPid, commandLine));
                             if (interpreter != null) {
                                 if (interpreter.drain()) {
-                                    _logger.trace(String.format("Returning task result of process [%s] for command [%s].", processPid, commandLine));
+                                    _logger.trace(
+                                            String.format("Returning task result of process [%s] for command [%s].",
+                                                    processPid, commandLine));
                                     return task.getResult();
                                 }
-                                _logger.trace(String.format("Returning interpretation of process [%s] for command [%s].", processPid, commandLine));
+                                _logger.trace(
+                                        String.format("Returning interpretation of process [%s] for command [%s].",
+                                                processPid, commandLine));
                                 return interpreter.interpret(ir);
                             } else {
                                 // null return exitValue apparently
-                                _logger.trace(String.format("Process [%s] for command [%s] exited with value [%s].", processPid, commandLine,
+                                _logger.trace(String.format("Process [%s] for command [%s] exited with value [%s].",
+                                        processPid, commandLine,
                                         _process.exitValue()));
                                 return String.valueOf(_process.exitValue());
                             }
                         } else {
-                            _logger.warn(String.format("Execution of process [%s] for command [%s] failed.", processPid, commandLine));
+                            _logger.warn(String.format("Execution of process [%s] for command [%s] failed.",
+                                    processPid, commandLine));
                             break;
                         }
                     }
                 } catch (InterruptedException e) {
                     if (!_isTimeOut) {
-                        _logger.debug(String.format("Exception [%s] occurred; however, it was not a timeout. Therefore, proceeding with the execution of process [%s] for command "
-                                + "[%s].", e.getMessage(), processPid, commandLine), e);
+                        _logger.debug(String.format(
+                                "Exception [%s] occurred; however, it was not a timeout. Therefore, proceeding with the execution of process [%s] for command [%s].",
+                                e.getMessage(), processPid, commandLine), e);
                         continue;
                     }
                 } finally {
@@ -318,18 +337,17 @@ public class Script implements Callable<String> {
                 TimedOutLogger log = new TimedOutLogger(_process);
                 Task timedoutTask = new Task(log, ir);
 
-                _logger.trace(String.format("Running timed out task of process [%s] for command [%s].", processPid, commandLine));
+                _logger.trace(String.format("Running timed out task of process [%s] for command [%s].", processPid,
+                        commandLine));
                 timedoutTask.run();
-                if (!_passwordCommand) {
-                    _logger.warn(String.format("Process [%s] for command [%s] timed out. Output is [%s].", processPid, commandLine, timedoutTask.getResult()));
-                } else {
-                    _logger.warn(String.format("Process [%s] for command [%s] timed out.", processPid, commandLine));
-                }
+                _logger.warn(String.format("Process [%s] for command [%s] timed out. Output is [%s].", processPid,
+                            commandLine, timedoutTask.getResult()));
 
                 return ERR_TIMEOUT;
             }
 
-            _logger.debug(String.format("Exit value of process [%s] for command [%s] is [%s].", processPid, commandLine, _process.exitValue()));
+            _logger.debug(String.format("Exit value of process [%s] for command [%s] is [%s].", processPid,
+                    commandLine, _process.exitValue()));
 
             BufferedReader reader = new BufferedReader(new InputStreamReader(_process.getInputStream()), 128);
 
@@ -340,19 +358,24 @@ public class Script implements Callable<String> {
                 error = String.valueOf(_process.exitValue());
             }
 
-            _logger.warn(String.format("Process [%s] for command [%s] encountered the error: [%s].", processPid, commandLine, error));
+            _logger.warn(String.format("Process [%s] for command [%s] encountered the error: [%s].", processPid,
+                    commandLine, error));
 
             return error;
         } catch (SecurityException ex) {
-            _logger.warn(String.format("Exception [%s] occurred. This may be due to an attempt of executing command [%s] as non root.", ex.getMessage(), commandLine),
+            _logger.warn(String.format(
+                    "Exception [%s] occurred. This may be due to an attempt of executing command [%s] as non root.",
+                    ex.getMessage(), commandLine),
                     ex);
             return stackTraceAsString(ex);
         } catch (Exception ex) {
-            _logger.warn(String.format("Exception [%s] occurred when attempting to run command [%s].", ex.getMessage(), commandLine), ex);
+            _logger.warn(String.format("Exception [%s] occurred when attempting to run command [%s].",
+                    ex.getMessage(), commandLine), ex);
             return stackTraceAsString(ex);
         } finally {
             if (_process != null) {
-                _logger.trace(String.format("Destroying process [%s] for command [%s].", _process.pid(), commandLine));
+                _logger.trace(
+                        String.format("Destroying process [%s] for command [%s].", _process.pid(), commandLine));
                 IOUtils.closeQuietly(_process.getErrorStream());
                 IOUtils.closeQuietly(_process.getOutputStream());
                 IOUtils.closeQuietly(_process.getInputStream());
@@ -363,9 +386,10 @@ public class Script implements Callable<String> {
 
     public String executeIgnoreExitValue(OutputInterpreter interpreter, int exitValue) {
         String[] command = _command.toArray(new String[_command.size()]);
+        String commandLine = buildCommandLine(command);
 
         if (_logger.isDebugEnabled()) {
-            _logger.debug(String.format("Executing: %s", buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]));
+            _logger.debug(String.format("Executing: %s", commandLine));
         }
 
         try {
@@ -376,7 +400,7 @@ public class Script implements Callable<String> {
 
             _process = pb.start();
             if (_process == null) {
-                _logger.warn(String.format("Unable to execute: %s", buildCommandLine(command)));
+                _logger.warn(String.format("Unable to execute: %s", commandLine));
                 return String.format("Unable to execute the command: %s", command[0]);
             }
 
@@ -440,11 +464,8 @@ public class Script implements Callable<String> {
                 Task timedoutTask = new Task(log, ir);
 
                 timedoutTask.run();
-                if (!_passwordCommand) {
-                    _logger.warn(String.format("Timed out: %s.  Output is: %s", buildCommandLine(command), timedoutTask.getResult()));
-                } else {
-                    _logger.warn(String.format("Timed out: %s", buildCommandLine(command)));
-                }
+                _logger.warn(String.format("Timed out: %s.  Output is: %s", commandLine,
+                            timedoutTask.getResult()));
 
                 return ERR_TIMEOUT;
             }
@@ -468,7 +489,7 @@ public class Script implements Callable<String> {
             _logger.warn("Security Exception....not running as root?", ex);
             return stackTraceAsString(ex);
         } catch (Exception ex) {
-            _logger.warn(String.format("Exception: %s", buildCommandLine(command)), ex);
+            _logger.warn(String.format("Exception: %s", commandLine), ex);
             return stackTraceAsString(ex);
         } finally {
             if (_process != null) {
@@ -517,9 +538,9 @@ public class Script implements Callable<String> {
                 } catch (Exception ex) {
                     result = stackTraceAsString(ex);
                 } finally {
-                        done = true;
-                        notifyAll();
-                        IOUtils.closeQuietly(reader);
+                    done = true;
+                    notifyAll();
+                    IOUtils.closeQuietly(reader);
                 }
             }
         }

utils/src/main/java/com/cloud/utils/server/ServerProperties.java

@@ -17,10 +17,12 @@
 package com.cloud.utils.server;
 
 import com.cloud.utils.crypt.EncryptionSecretKeyChecker;
+import com.cloud.utils.StringUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Properties;
@@ -28,9 +30,20 @@ import java.util.Properties;
 public class ServerProperties {
     protected Logger logger = LogManager.getLogger(getClass());
 
+    public static final String HTTP_ENABLE = "http.enable";
+    public static final String HTTP_PORT = "http.port";
+    public static final String HTTPS_ENABLE = "https.enable";
+    public static final String HTTPS_PORT = "https.port";
+    public static final String KEYSTORE_FILE = "https.keystore";
+    public static final String PASSWORD_ENCRYPTION_TYPE = "password.encryption.type";
+
     private static Properties properties = new Properties();
     private static boolean loaded = false;
-    public static final String passwordEncryptionType = "password.encryption.type";
+
+    private static int httpPort = 8080;
+
+    private static boolean httpsEnable = false;
+    private static int httpsPort = 8443;
 
     public synchronized static Properties getServerProperties(InputStream inputStream) {
         if (!loaded) {
@@ -39,7 +52,7 @@ public class ServerProperties {
                 serverProps.load(inputStream);
 
                 EncryptionSecretKeyChecker checker = new EncryptionSecretKeyChecker();
-                checker.check(serverProps, passwordEncryptionType);
+                checker.check(serverProps, PASSWORD_ENCRYPTION_TYPE);
 
                 if (EncryptionSecretKeyChecker.useEncryption()) {
                     EncryptionSecretKeyChecker.decryptAnyProperties(serverProps);
@@ -50,10 +63,29 @@ public class ServerProperties {
                 IOUtils.closeQuietly(inputStream);
             }
 
+            httpPort = Integer.parseInt(serverProps.getProperty(ServerProperties.HTTP_PORT, "8080"));
+
+            boolean httpsEnabled = Boolean.parseBoolean(serverProps.getProperty(ServerProperties.HTTPS_ENABLE, "false"));
+            String keystoreFile = serverProps.getProperty(KEYSTORE_FILE);
+            httpsEnable =  httpsEnabled && StringUtils.isNotEmpty(keystoreFile) && new File(keystoreFile).exists();
+            httpsPort = Integer.parseInt(serverProps.getProperty(ServerProperties.HTTPS_PORT, "8443"));
+
             properties = serverProps;
             loaded = true;
         }
 
         return properties;
     }
+
+    public static int getHttpPort() {
+        return httpPort;
+    }
+
+    public static boolean isHttpsEnabled() {
+        return httpsEnable;
+    }
+
+    public static int getHttpsPort() {
+        return httpsPort;
+    }
 }

utils/src/main/java/com/cloud/utils/ssh/SshHelper.java

@@ -23,6 +23,8 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -40,6 +42,23 @@ public class SshHelper {
     private static final int DEFAULT_CONNECT_TIMEOUT = 180000;
     private static final int DEFAULT_KEX_TIMEOUT = 60000;
     private static final int DEFAULT_WAIT_RESULT_TIMEOUT = 120000;
+    private static final String MASKED_VALUE = "*****";
+
+    private static final Pattern[] SENSITIVE_COMMAND_PATTERNS = new Pattern[] {
+            Pattern.compile("(?i)(\\s+-p\\s+['\"])([^'\"]*)(['\"])"),
+            Pattern.compile("(?i)(\\s+-p\\s+)([^\\s]+)"),
+            Pattern.compile("(?i)(\\s+-p=['\"])([^'\"]*)(['\"])"),
+            Pattern.compile("(?i)(\\s+-p=)([^\\s]+)"),
+            Pattern.compile("(?i)(--password=['\"])([^'\"]*)(['\"])"),
+            Pattern.compile("(?i)(--password=)([^\\s]+)"),
+            Pattern.compile("(?i)(--password\\s+['\"])([^'\"]*)(['\"])"),
+            Pattern.compile("(?i)(--password\\s+)([^\\s]+)"),
+            Pattern.compile("(?i)(\\s+-u\\s+['\"][^,'\":]+[,:])([^'\"]*)(['\"])"),
+            Pattern.compile("(?i)(\\s+-u\\s+[^\\s,:]+[,:])([^\\s]+)"),
+            Pattern.compile("(?i)(\\s+-s\\s+['\"])([^'\"]*)(['\"])"),
+            Pattern.compile("(?i)(\\s+-s\\s+)([^\\s]+)"),
+
+    };
 
     protected static Logger LOGGER = LogManager.getLogger(SshHelper.class);
 
@@ -145,7 +164,7 @@ public class SshHelper {
     }
 
     public static void scpTo(String host, int port, String user, File pemKeyFile, String password, String remoteTargetDirectory, String[] localFiles, String fileMode,
-                             int connectTimeoutInMs, int kexTimeoutInMs) throws Exception {
+            int connectTimeoutInMs, int kexTimeoutInMs) throws Exception {
 
         com.trilead.ssh2.Connection conn = null;
         com.trilead.ssh2.SCPClient scpClient = null;
@@ -291,13 +310,16 @@ public class SshHelper {
             }
 
             if (sess.getExitStatus() == null) {
-                //Exit status is NOT available. Returning failure result.
-                LOGGER.error(String.format("SSH execution of command %s has no exit status set. Result output: %s", command, result));
+                // Exit status is NOT available. Returning failure result.
+                LOGGER.error(String.format("SSH execution of command %s has no exit status set. Result output: %s",
+                        sanitizeForLogging(command), sanitizeForLogging(result)));
                 return new Pair<Boolean, String>(false, result);
             }
 
             if (sess.getExitStatus() != null && sess.getExitStatus().intValue() != 0) {
-                LOGGER.error(String.format("SSH execution of command %s has an error status code in return. Result output: %s", command, result));
+                LOGGER.error(String.format(
+                        "SSH execution of command %s has an error status code in return. Result output: %s",
+                        sanitizeForLogging(command), sanitizeForLogging(result)));
                 return new Pair<Boolean, String>(false, result);
             }
             return new Pair<Boolean, String>(true, result);
@@ -366,4 +388,47 @@ public class SshHelper {
             throw new SshException(msg);
         }
     }
+
+    private static String sanitizeForLogging(String value) {
+        if (value == null) {
+            return null;
+        }
+        String masked = maskSensitiveValue(value);
+        String cleaned = com.cloud.utils.StringUtils.cleanString(masked);
+        if (StringUtils.isBlank(cleaned)) {
+            return masked;
+        }
+        return cleaned;
+    }
+
+    private static String maskSensitiveValue(String value) {
+        String masked = value;
+        for (Pattern pattern : SENSITIVE_COMMAND_PATTERNS) {
+            masked = replaceWithMask(masked, pattern);
+        }
+        return masked;
+    }
+
+    private static String replaceWithMask(String value, Pattern pattern) {
+        Matcher matcher = pattern.matcher(value);
+        if (!matcher.find()) {
+            return value;
+        }
+
+        StringBuffer buffer = new StringBuffer();
+        do {
+            StringBuilder replacement = new StringBuilder();
+            replacement.append(matcher.group(1));
+            if (matcher.groupCount() >= 3) {
+                replacement.append(MASKED_VALUE);
+                replacement.append(matcher.group(matcher.groupCount()));
+            } else {
+                replacement.append(MASKED_VALUE);
+            }
+            matcher.appendReplacement(buffer, Matcher.quoteReplacement(replacement.toString()));
+        } while (matcher.find());
+
+        matcher.appendTail(buffer);
+        return buffer.toString();
+    }
 }

utils/src/test/java/com/cloud/utils/script/ScriptTest.java

@@ -78,4 +78,34 @@ public class ScriptTest {
         String result = Script.getExecutableAbsolutePath("ls");
         Assert.assertTrue(List.of("/usr/bin/ls", "/bin/ls").contains(result));
     }
+
+    @Test
+    public void testBuildCommandLineWithSensitiveData() {
+        Script script = new Script("test.sh");
+        script.add("normal-arg");
+        script.addSensitive("sensitive-arg");
+        String commandLine = script.toString();
+        Assert.assertEquals("test.sh normal-arg ****** ", commandLine);
+    }
+
+    @Test
+    public void testBuildCommandLineWithMultipleSensitiveData() {
+        Script script = new Script("test.sh");
+        script.add("normal-arg");
+        script.addSensitive("sensitive-arg1");
+        script.add("another-normal-arg");
+        script.addSensitive("sensitive-arg2");
+        String commandLine = script.toString();
+        Assert.assertEquals("test.sh normal-arg ****** another-normal-arg ****** ", commandLine);
+    }
+
+    @Test
+    public void testBuildCommandLineWithLegacyPasswordOption() {
+        Script script = new Script("test.sh");
+        script.add("-y");
+        script.add("legacy-password");
+        String commandLine = script.toString();
+        Assert.assertEquals("test.sh -y ****** ", commandLine);
+    }
+
 }

utils/src/test/java/com/cloud/utils/ssh/SshHelperTest.java

@@ -21,6 +21,7 @@ package com.cloud.utils.ssh;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.reflect.Method;
 
 import org.junit.Assert;
 import org.junit.Test;
@@ -140,4 +141,63 @@ public class SshHelperTest {
 
         Mockito.verify(conn).openSession();
     }
+
+    @Test
+    public void sanitizeForLoggingMasksShortPasswordFlag() throws Exception {
+        String command = "/opt/cloud/bin/script -v 10.0.0.1 -p superSecret";
+        String sanitized = invokeSanitizeForLogging(command);
+
+        Assert.assertTrue("Sanitized command should retain flag", sanitized.contains("-p *****"));
+        Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("superSecret"));
+    }
+
+    @Test
+    public void sanitizeForLoggingMasksQuotedPasswordFlag() throws Exception {
+        String command = "/opt/cloud/bin/script -v 10.0.0.1 -p \"super Secret\"";
+        String sanitized = invokeSanitizeForLogging(command);
+
+        Assert.assertTrue("Sanitized command should retain quoted flag", sanitized.contains("-p *****"));
+        Assert.assertFalse("Sanitized command should not contain original password",
+                sanitized.contains("super Secret"));
+    }
+
+    @Test
+    public void sanitizeForLoggingMasksLongPasswordAssignments() throws Exception {
+        String command = "tool --password=superSecret";
+        String sanitized = invokeSanitizeForLogging(command);
+
+        Assert.assertTrue("Sanitized command should retain assignment", sanitized.contains("--password=*****"));
+        Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("superSecret"));
+    }
+
+    @Test
+    public void sanitizeForLoggingMasksUsernamePasswordPairs() throws Exception {
+        String command = "/opt/cloud/bin/vpn_l2tp.sh -u alice,topSecret";
+        String sanitized = invokeSanitizeForLogging(command);
+
+        Assert.assertTrue("Sanitized command should retain username and mask password",
+                sanitized.contains("-u alice,*****"));
+        Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("topSecret"));
+    }
+
+    @Test
+    public void sanitizeForLoggingMasksUsernamePasswordPairsWithColon() throws Exception {
+        String command = "curl -u alice:topSecret https://example.com";
+        String sanitized = invokeSanitizeForLogging(command);
+
+        Assert.assertTrue("Sanitized command should retain username and mask password",
+                sanitized.contains("-u alice:*****"));
+        Assert.assertFalse("Sanitized command should not contain original password", sanitized.contains("topSecret"));
+    }
+
+    @Test
+    public void sanitizeForLoggingHandlesNullValues() throws Exception {
+        Assert.assertNull(invokeSanitizeForLogging(null));
+    }
+
+    private String invokeSanitizeForLogging(String value) throws Exception {
+        Method method = SshHelper.class.getDeclaredMethod("sanitizeForLogging", String.class);
+        method.setAccessible(true);
+        return (String) method.invoke(null, value);
+    }
 }