Merge branch '4.22'

api/src/main/java/com/cloud/network/NetworkService.java

@@ -108,6 +108,10 @@ public interface NetworkService {
            PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType) throws
             InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException;
 
+    Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner,
+                               PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType, Pair<Integer, Integer> vrIfaceMTUs) throws
+            InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException;
+
     Pair<List<? extends Network>, Integer> searchForNetworks(ListNetworksCmd cmd);
 
     boolean deleteNetwork(long networkId, boolean forced);

api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java

@@ -118,6 +118,9 @@ public class ListHostsCmd extends BaseListCmd {
             since = "4.21.0")
     private String storageAccessGroup;
 
+    @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, description = "the host version", since = "4.20.3")
+    private String version;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -222,6 +225,10 @@ public class ListHostsCmd extends BaseListCmd {
         this.storageAccessGroup = storageAccessGroup;
     }
 
+    public String getVersion() {
+        return version;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java

@@ -45,6 +45,10 @@ public class ListMgmtsCmd extends BaseListCmd {
             since = "4.20.1.0")
     private Boolean peers;
 
+    @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING,
+            description = "the version of the management server", since = "4.20.3")
+    private String version;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -61,6 +65,10 @@ public class ListMgmtsCmd extends BaseListCmd {
         return BooleanUtils.toBooleanDefaultIfNull(peers, false);
     }
 
+    public String getVersion() {
+        return version;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java

@@ -59,7 +59,6 @@ public class ListBackupScheduleCmd extends BaseListProjectAndAccountResourcesCmd
     @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID,
             type = CommandType.UUID,
             entityType = UserVmResponse.class,
-            required = true,
             description = "ID of the Instance")
     private Long vmId;
 

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java

@@ -43,7 +43,7 @@ import java.util.List;
 public class DeployVnfApplianceCmd extends DeployVMCmd implements UserCmd {
 
     @Parameter(name = ApiConstants.VNF_CONFIGURE_MANAGEMENT, type = CommandType.BOOLEAN, required = false,
-            description = "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. " +
+            description = "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. " +
                     "Network rules are configured if management network is an isolated network or shared network with security groups.")
     private Boolean vnfConfigureManagement;
 

api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java

@@ -29,6 +29,7 @@ import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd;
 import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import java.util.List;
+import java.util.Map;
 
 public interface VnfTemplateManager {
 
@@ -42,11 +43,12 @@ public interface VnfTemplateManager {
 
     void updateVnfTemplate(long templateId, UpdateVnfTemplateCmd cmd);
 
-    void validateVnfApplianceNics(VirtualMachineTemplate template, List<Long> networkIds);
+    void validateVnfApplianceNics(VirtualMachineTemplate template, List<Long> networkIds, Map<Integer, Long> vmNetworkMap);
 
     SecurityGroup createSecurityGroupForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, DeployVnfApplianceCmd cmd);
 
     void createIsolatedNetworkRulesForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner,
                                                    UserVm vm, DeployVnfApplianceCmd cmd)
             throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException;
+
 }

api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java

@@ -16,6 +16,7 @@
 // under the License.
 package org.apache.cloudstack.storage.template;
 
+import com.cloud.agent.api.to.deployasis.OVFNetworkTO;
 import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.network.VNF;
 import com.cloud.storage.Storage;
@@ -124,6 +125,9 @@ public class VnfTemplateUtils {
     public static void validateApiCommandParams(BaseCmd cmd, VirtualMachineTemplate template) {
         if (cmd instanceof RegisterVnfTemplateCmd) {
             RegisterVnfTemplateCmd registerCmd = (RegisterVnfTemplateCmd) cmd;
+            if (registerCmd.isDeployAsIs() && CollectionUtils.isNotEmpty(registerCmd.getVnfNics())) {
+                throw new InvalidParameterValueException("VNF nics cannot be specified when register a deploy-as-is Template. Please wait until Template settings are read from OVA.");
+            }
             validateApiCommandParams(registerCmd.getVnfDetails(), registerCmd.getVnfNics(), registerCmd.getTemplateType());
         } else if (cmd instanceof UpdateVnfTemplateCmd) {
             UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd;
@@ -149,4 +153,18 @@ public class VnfTemplateUtils {
             }
         }
     }
+
+    public static void validateDeployAsIsTemplateVnfNics(List<OVFNetworkTO> ovfNetworks, List<VNF.VnfNic> vnfNics) {
+        if (CollectionUtils.isEmpty(vnfNics)) {
+            return;
+        }
+        if (CollectionUtils.isEmpty(ovfNetworks)) {
+            throw new InvalidParameterValueException("The list of networks read from OVA is empty. Please wait until the template is fully downloaded and processed.");
+        }
+        for (VNF.VnfNic vnfNic : vnfNics) {
+            if (vnfNic.getDeviceId() < ovfNetworks.size() && !vnfNic.isRequired()) {
+                throw new InvalidParameterValueException(String.format("The VNF nic [device ID: %s ] is required as it is defined in the OVA template.", vnfNic.getDeviceId()));
+            }
+        }
+    }
 }

client/src/main/java/org/apache/cloudstack/ACSRequestLog.java

@@ -18,6 +18,7 @@
 //
 package org.apache.cloudstack;
 
+import com.cloud.api.ApiServlet;
 import com.cloud.utils.StringUtils;
 import org.eclipse.jetty.server.NCSARequestLog;
 import org.eclipse.jetty.server.Request;
@@ -25,6 +26,7 @@ import org.eclipse.jetty.server.Response;
 import org.eclipse.jetty.util.DateCache;
 import org.eclipse.jetty.util.component.LifeCycle;
 
+import java.net.InetAddress;
 import java.util.Locale;
 import java.util.TimeZone;
 
@@ -51,9 +53,8 @@ public class ACSRequestLog extends NCSARequestLog {
             StringBuilder sb = buffers.get();
             sb.setLength(0);
 
-            sb.append(request.getHttpChannel().getEndPoint()
-                            .getRemoteAddress().getAddress()
-                            .getHostAddress())
+            InetAddress remoteAddress = ApiServlet.getClientAddress(request);
+            sb.append(remoteAddress.getHostAddress())
                     .append(" - - [")
                     .append(dateCache.format(request.getTimeStamp()))
                     .append("] \"")

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -24,15 +24,12 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.lang.management.ManagementFactory;
 import java.net.URL;
-import java.util.Arrays;
 import java.util.Properties;
 
-import com.cloud.api.ApiServer;
 import org.apache.commons.daemon.Daemon;
 import org.apache.commons.daemon.DaemonContext;
 import org.apache.commons.lang3.StringUtils;
 import org.eclipse.jetty.jmx.MBeanContainer;
-import org.eclipse.jetty.server.ForwardedRequestCustomizer;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.RequestLog;
@@ -193,7 +190,6 @@ public class ServerDaemon implements Daemon {
         httpConfig.setResponseHeaderSize(8192);
         httpConfig.setSendServerVersion(false);
         httpConfig.setSendDateHeader(false);
-        addForwardingCustomiser(httpConfig);
 
         // HTTP Connector
         createHttpConnector(httpConfig);
@@ -216,21 +212,6 @@ public class ServerDaemon implements Daemon {
         server.join();
     }
 
-    /**
-     * Adds a ForwardedRequestCustomizer to the HTTP configuration to handle forwarded headers.
-     * The header used for forwarding is determined by the ApiServer.listOfForwardHeaders property.
-     * Only non empty headers are considered and only the first of the comma-separated list is used.
-     * @param httpConfig the HTTP configuration to which the customizer will be added
-     */
-    private static void addForwardingCustomiser(HttpConfiguration httpConfig) {
-        ForwardedRequestCustomizer customiser = new ForwardedRequestCustomizer();
-        String header = Arrays.stream(ApiServer.listOfForwardHeaders.value().split(",")).findFirst().orElse(null);
-        if (com.cloud.utils.StringUtils.isNotEmpty(header)) {
-            customiser.setForwardedForHeader(header);
-        }
-        httpConfig.addCustomizer(customiser);
-    }
-
     @Override
     public void stop() throws Exception {
         server.stop();

debian/control

@@ -4,7 +4,7 @@ Priority: extra
 Maintainer: The Apache CloudStack Team <dev@cloudstack.apache.org>
 Build-Depends: debhelper (>= 9), openjdk-17-jdk | java17-sdk | java17-jdk | zulu-17 | openjdk-11-jdk | java11-sdk | java11-jdk | zulu-11, genisoimage,
  python-mysql.connector | python3-mysql.connector | mysql-connector-python-py3, maven (>= 3) | maven3,
- python (>= 2.7) | python2 (>= 2.7), python3 (>= 3), python-setuptools, python3-setuptools,
+ python3 (>= 3), python3-setuptools,
  nodejs (>= 12), lsb-release, dh-systemd | debhelper (>= 13)
 Standards-Version: 3.8.1
 Homepage: http://www.cloudstack.org/

engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java

@@ -79,6 +79,11 @@ public interface ResourceManager extends ResourceService, Configurable {
             ConfigKey.Kind.Select,
             "," + CPU.CPUArch.getTypesAsCSV());
 
+    ConfigKey<String> SystemVMDefaultHypervisor = new ConfigKey<String>(String.class,
+            "system.vm.default.hypervisor", "Advanced", "Any", "Hypervisor type used to create System VMs. Valid values are: XenServer, KVM, VMware, Hyperv, VirtualBox, " +
+            "Parralels, BareMetal, Ovm, LXC, Any", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "XenServer, KVM, VMware, Hyperv, " +
+            "VirtualBox, Parralels, BareMetal, Ovm, LXC, Any");
+
     /**
      * Register a listener for different types of resource life cycle events.
      * There can only be one type of listener per type of host.

engine/components-api/src/main/java/com/cloud/template/TemplateManager.java

@@ -153,6 +153,8 @@ public interface TemplateManager {
 
     TemplateType validateTemplateType(BaseCmd cmd, boolean isAdmin, boolean isCrossZones, Hypervisor.HypervisorType hypervisorType);
 
+    DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId);
+
     List<DatadiskTO> getTemplateDisksOnImageStore(VirtualMachineTemplate template, DataStoreRole role, String configurationId);
 
     static Boolean getValidateUrlIsResolvableBeforeRegisteringTemplateValue() {

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java

@@ -89,7 +89,7 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
             return;
         }
         for (Long vmId : vmIds) {
-            if (!notUpdated.containsKey(vmId)) {
+            if (MapUtils.isEmpty(notUpdated) || !notUpdated.containsKey(vmId)) {
                 logger.debug("VM state report is updated. {}, {}, power state: {}",
                         () -> hostCache.get(hostId), () -> vmCache.get(vmId), () -> instancePowerStates.get(vmId));
                 _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE,
@@ -158,8 +158,8 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat
                 // an update might have occurred that we should not override in case of out of band migration
                 instancePowerStates.put(instance.getId(), VirtualMachine.PowerState.PowerReportMissing);
             } else {
-                logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period yet",
-                        instance.getId(), milliSecondsSinceLastStateUpdate);
+                logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period ({} ms) yet",
+                        instance.getId(), milliSecondsSinceLastStateUpdate, milliSecondsGracefulPeriod);
             }
         }
         updateAndPublishVmPowerStates(hostId, instancePowerStates, startTime);

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -291,21 +291,41 @@ public class TemplateServiceImpl implements TemplateService {
         }
     }
 
-    protected boolean isSkipTemplateStoreDownload(VMTemplateVO template, Long zoneId) {
+    protected boolean shouldDownloadTemplateToStore(VMTemplateVO template, DataStore store) {
+        Long zoneId = store.getScope().getScopeId();
+        DataStore directedStore = _tmpltMgr.verifyHeuristicRulesForZone(template, zoneId);
+        if (directedStore != null && store.getId() != directedStore.getId()) {
+            logger.info("Template [{}] will not be download to image store [{}], as a heuristic rule is directing it to another store.",
+                    template.getUniqueName(), store.getName());
+            return false;
+        }
+
         if (template.isPublicTemplate()) {
-            return false;
+            logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is public.", template.getUniqueName(),
+                    store.getName());
+            return true;
         }
+
         if (template.isFeatured()) {
-            return false;
+            logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is featured.", template.getUniqueName(),
+                    store.getName());
+            return true;
         }
+
         if (TemplateType.SYSTEM.equals(template.getTemplateType())) {
-            return false;
+            logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is a system VM template.",
+                    template.getUniqueName(),store.getName());
+            return true;
         }
+
         if (zoneId != null &&  _vmTemplateStoreDao.findByTemplateZone(template.getId(), zoneId, DataStoreRole.Image) == null) {
-            logger.debug("Template {} is not present on any image store for the zone ID: {}, its download cannot be skipped", template, zoneId);
-            return false;
+            logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is not present on any image store of zone [{}].",
+                    template.getUniqueName(), store.getName(), zoneId);
+            return true;
         }
-        return true;
+
+        logger.info("Skipping download of template [{}] to image store [{}].", template.getUniqueName(), store.getName());
+        return false;
     }
 
     @Override
@@ -533,8 +553,7 @@ public class TemplateServiceImpl implements TemplateService {
                         // download.
                         for (VMTemplateVO tmplt : toBeDownloaded) {
                             // if this is private template, skip sync to a new image store
-                            if (isSkipTemplateStoreDownload(tmplt, zoneId)) {
-                                logger.info("Skip sync downloading private template {} to a new image store", tmplt);
+                            if (!shouldDownloadTemplateToStore(tmplt, store)) {
                                 continue;
                             }
 

engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java

@@ -19,6 +19,7 @@
 package org.apache.cloudstack.storage.image;
 
 import com.cloud.storage.template.TemplateProp;
+import com.cloud.template.TemplateManager;
 import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -70,6 +71,9 @@ public class TemplateServiceImplTest {
     @Mock
     TemplateObject templateInfoMock;
 
+    @Mock
+    DataStore dataStoreMock;
+
     @Mock
     DataStore sourceStoreMock;
 
@@ -82,6 +86,9 @@ public class TemplateServiceImplTest {
     @Mock
     StorageOrchestrationService storageOrchestrator;
 
+    @Mock
+    TemplateManager templateManagerMock;
+
     Map<String, TemplateProp> templatesInSourceStore = new HashMap<>();
 
     @Before
@@ -96,45 +103,45 @@ public class TemplateServiceImplTest {
         Mockito.doReturn(null).when(templateService).listTemplate(destStoreMock);
         Mockito.doReturn("install-path").when(templateInfoMock).getInstallPath();
         Mockito.doReturn(templateInfoMock).when(templateDataFactoryMock).getTemplate(2L, sourceStoreMock);
+        Mockito.doReturn(3L).when(dataStoreMock).getId();
+        Mockito.doReturn(zoneScopeMock).when(dataStoreMock).getScope();
     }
 
     @Test
-    public void testIsSkipTemplateStoreDownloadPublicTemplate() {
-        VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
-        Mockito.when(templateVO.isPublicTemplate()).thenReturn(true);
-        Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L));
+    public void shouldDownloadTemplateToStoreTestSkipsTemplateDirectedToAnotherStorage() {
+        DataStore destinedStore = Mockito.mock(DataStore.class);
+        Mockito.doReturn(dataStoreMock.getId() + 1L).when(destinedStore).getId();
+        Mockito.when(templateManagerMock.verifyHeuristicRulesForZone(tmpltMock, zoneScopeMock.getScopeId())).thenReturn(destinedStore);
+        Assert.assertFalse(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock));
     }
 
     @Test
-    public void testIsSkipTemplateStoreDownloadFeaturedTemplate() {
-        VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
-        Mockito.when(templateVO.isFeatured()).thenReturn(true);
-        Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L));
+    public void shouldDownloadTemplateToStoreTestDownloadsPublicTemplate() {
+        Mockito.when(tmpltMock.isPublicTemplate()).thenReturn(true);
+        Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock));
     }
 
     @Test
-    public void testIsSkipTemplateStoreDownloadSystemTemplate() {
-        VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
-        Mockito.when(templateVO.getTemplateType()).thenReturn(Storage.TemplateType.SYSTEM);
-        Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L));
+    public void shouldDownloadTemplateToStoreTestDownloadsFeaturedTemplate() {
+        Mockito.when(tmpltMock.isFeatured()).thenReturn(true);
+        Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock));
     }
 
     @Test
-    public void testIsSkipTemplateStoreDownloadPrivateNoRefTemplate() {
-        VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
-        long id = 1L;
-        Mockito.when(templateVO.getId()).thenReturn(id);
-        Mockito.when(templateDataStoreDao.findByTemplateZone(id, id, DataStoreRole.Image)).thenReturn(null);
-        Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, id));
+    public void shouldDownloadTemplateToStoreTestDownloadsSystemTemplate() {
+        Mockito.when(tmpltMock.getTemplateType()).thenReturn(Storage.TemplateType.SYSTEM);
+        Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock));
     }
 
     @Test
-    public void testIsSkipTemplateStoreDownloadPrivateExistingTemplate() {
-        VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);
-        long id = 1L;
-        Mockito.when(templateVO.getId()).thenReturn(id);
-        Mockito.when(templateDataStoreDao.findByTemplateZone(id, id, DataStoreRole.Image)).thenReturn(Mockito.mock(TemplateDataStoreVO.class));
-        Assert.assertTrue(templateService.isSkipTemplateStoreDownload(templateVO, id));
+    public void shouldDownloadTemplateToStoreTestDownloadsPrivateNoRefTemplate() {
+        Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock));
+    }
+
+    @Test
+    public void shouldDownloadTemplateToStoreTestSkipsPrivateExistingTemplate() {
+        Mockito.when(templateDataStoreDao.findByTemplateZone(tmpltMock.getId(), zoneScopeMock.getScopeId(), DataStoreRole.Image)).thenReturn(Mockito.mock(TemplateDataStoreVO.class));
+        Assert.assertFalse(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock));
     }
 
     @Test

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java

@@ -111,7 +111,9 @@ public class LibvirtDomainXMLParser {
                     def.defNetworkBasedDisk(diskPath, host, port, authUserName, poolUuid, diskLabel,
                         DiskDef.DiskBus.valueOf(bus.toUpperCase()),
                         DiskDef.DiskProtocol.valueOf(protocol.toUpperCase()), fmt);
-                    def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase()));
+                    if (StringUtils.isNotBlank(diskCacheMode)) {
+                        def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase()));
+                    }
                 } else {
                     String diskFmtType = getAttrValue("driver", "type", disk);
                     String diskCacheMode = getAttrValue("driver", "cache", disk);

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java

@@ -45,7 +45,7 @@ public final class CitrixUpdateHostPasswordCommandWrapper extends CommandWrapper
 
         Pair<Boolean, String> result;
         try {
-            logger.debug("Executing command in Host: " + cmdLine);
+            logger.debug("Executing password update command on host: {} for user: {}", hostIp, username);
             final String hostPassword = citrixResourceBase.getPwdFromQueue();
             result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine.toString());
         } catch (final Exception e) {

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

@@ -1172,9 +1172,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
 
             CallContext networkContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Network);
             try {
+                Long zoneId = zone.getId();
+                Integer publicMTU = NetworkService.VRPublicInterfaceMtu.valueIn(zoneId);
+                Integer privateMTU = NetworkService.VRPrivateInterfaceMtu.valueIn(zoneId);
                 network = networkService.createGuestNetwork(networkOffering.getId(), clusterName + "-network",
-                        owner.getAccountName() + "-network", owner, physicalNetwork, zone.getId(),
-                        ControlledEntity.ACLType.Account);
+                        owner.getAccountName() + "-network", owner, physicalNetwork, zoneId,
+                        ControlledEntity.ACLType.Account, new Pair<>(publicMTU, privateMTU));
                 if (!networkOffering.isForVpc() && NetworkOffering.RoutingMode.Dynamic == networkOffering.getRoutingMode()) {
                     bgpService.allocateASNumber(zone.getId(), asNumber, network.getId(), null);
                 }

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java

@@ -82,6 +82,9 @@ public interface SAML2AuthManager extends PluggableAPIAuthenticator, PluggableSe
     ConfigKey<Boolean> SAMLRequirePasswordLogin = new ConfigKey<Boolean>("Advanced", Boolean.class, "saml2.require.password", "true",
     "When enabled SAML2 will validate that the SAML login was performed with a password.  If disabled, other forms of authentication are allowed (two-factor, certificate, etc) on the SAML Authentication Provider", true);
 
+    ConfigKey<Boolean> EnableLoginAfterSAMLDisable = new ConfigKey<>("Advanced", Boolean.class, "enable.login.with.disabled.saml", "false", "When enabled, if SAML SSO is disabled, enables user to login with user and password, otherwise a user with SAML SSO disabled cannot login", true);
+
+
 
     SAMLProviderMetadata getSPMetadata();
     SAMLProviderMetadata getIdPMetadata(String entityId);

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java

@@ -451,8 +451,13 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage
                 user.setExternalEntity(entityId);
                 user.setSource(User.Source.SAML2);
             } else {
+                boolean enableLoginAfterSAMLDisable =  SAML2AuthManager.EnableLoginAfterSAMLDisable.value();
                 if (user.getSource().equals(User.Source.SAML2)) {
-                    user.setSource(User.Source.SAML2DISABLED);
+                    if(enableLoginAfterSAMLDisable) {
+                        user.setSource(User.Source.UNKNOWN);
+                    } else {
+                        user.setSource(User.Source.SAML2DISABLED);
+                    }
                 } else {
                     return false;
                 }
@@ -541,6 +546,6 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage
                 SAMLCloudStackRedirectionUrl, SAMLUserAttributeName,
                 SAMLIdentityProviderMetadataURL, SAMLDefaultIdentityProviderId,
                 SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature,
-                SAMLForceAuthn, SAMLUserSessionKeyPathAttribute, SAMLRequirePasswordLogin};
+                SAMLForceAuthn, SAMLUserSessionKeyPathAttribute, SAMLRequirePasswordLogin, EnableLoginAfterSAMLDisable};
     }
 }

server/src/main/java/com/cloud/api/ApiServlet.java

@@ -679,38 +679,34 @@ public class ApiServlet extends HttpServlet {
         }
         return false;
     }
-    boolean doUseForwardHeaders() {
+    static boolean doUseForwardHeaders() {
         return Boolean.TRUE.equals(ApiServer.useForwardHeader.value());
     }
 
-    String[] proxyNets() {
+    static String[] proxyNets() {
         return ApiServer.proxyForwardList.value().split(",");
     }
     //This method will try to get login IP of user even if servlet is behind reverseProxy or loadBalancer
-    public InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException {
-        String ip = null;
-        InetAddress pretender = InetAddress.getByName(request.getRemoteAddr());
-        if(doUseForwardHeaders()) {
-            if (NetUtils.isIpInCidrList(pretender, proxyNets())) {
+    public static InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException {
+        final String remote = request.getRemoteAddr();
+        if (doUseForwardHeaders()) {
+            final InetAddress remoteAddr = InetAddress.getByName(remote);
+            if (NetUtils.isIpInCidrList(remoteAddr, proxyNets())) {
                 for (String header : getClientAddressHeaders()) {
                     header = header.trim();
-                    ip = getCorrectIPAddress(request.getHeader(header));
+                    final String ip = getCorrectIPAddress(request.getHeader(header));
                     if (StringUtils.isNotBlank(ip)) {
-                        LOGGER.debug(String.format("found ip %s in header %s ", ip, header));
-                        break;
+                        LOGGER.debug("found ip {} in header {}", ip, header);
+                        return InetAddress.getByName(ip);
                     }
-                } // no address found in header so ip is blank and use remote addr
-            } // else not an allowed proxy address, ip is blank and use remote addr
+                }
+            }
         }
-        if (StringUtils.isBlank(ip)) {
-            LOGGER.trace(String.format("no ip found in headers, returning remote address %s.", pretender.getHostAddress()));
-            return pretender;
-        }
-
-        return InetAddress.getByName(ip);
+        LOGGER.trace("no ip found in headers, returning remote address {}.", remote);
+        return InetAddress.getByName(remote);
     }
 
-    private String[] getClientAddressHeaders() {
+    private static String[] getClientAddressHeaders() {
         return ApiServer.listOfForwardHeaders.value().split(",");
     }
 

server/src/main/java/com/cloud/api/query/QueryManagerImpl.java

@@ -2425,6 +2425,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         Long msId = cmd.getManagementServerId();
         final CPU.CPUArch arch = cmd.getArch();
         String storageAccessGroup = cmd.getStorageAccessGroup();
+        String version = cmd.getVersion();
 
         Filter searchFilter = new Filter(HostVO.class, "id", Boolean.TRUE, startIndex, pageSize);
 
@@ -2449,11 +2450,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             hostSearchBuilder.or("storageAccessGroupMiddle", hostSearchBuilder.entity().getStorageAccessGroups(), Op.LIKE);
             hostSearchBuilder.cp();
         }
+        hostSearchBuilder.and("version", hostSearchBuilder.entity().getVersion(), SearchCriteria.Op.EQ);
 
         if (keyword != null) {
             hostSearchBuilder.and().op("keywordName", hostSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE);
             hostSearchBuilder.or("keywordStatus", hostSearchBuilder.entity().getStatus(), SearchCriteria.Op.LIKE);
             hostSearchBuilder.or("keywordType", hostSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE);
+            hostSearchBuilder.or("keywordVersion", hostSearchBuilder.entity().getVersion(), SearchCriteria.Op.LIKE);
             hostSearchBuilder.cp();
         }
 
@@ -2484,6 +2487,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             sc.setParameters("keywordName", "%" + keyword + "%");
             sc.setParameters("keywordStatus", "%" + keyword + "%");
             sc.setParameters("keywordType", "%" + keyword + "%");
+            sc.setParameters("keywordVersion", "%" + keyword + "%");
         }
 
         if (id != null) {
@@ -2547,6 +2551,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
             sc.setParameters("storageAccessGroupMiddle", "%," + storageAccessGroup + ",%");
         }
 
+        if (version != null) {
+            sc.setParameters("version", version);
+        }
+
         Pair<List<HostVO>, Integer> uniqueHostPair = hostDao.searchAndCount(sc, searchFilter);
         Integer count = uniqueHostPair.second();
         List<Long> hostIds = uniqueHostPair.first().stream().map(HostVO::getId).collect(Collectors.toList());
@@ -5744,6 +5752,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
     protected Pair<List<ManagementServerJoinVO>, Integer> listManagementServersInternal(ListMgmtsCmd cmd) {
         Long id = cmd.getId();
         String name = cmd.getHostName();
+        String version = cmd.getVersion();
+        String keyword = cmd.getKeyword();
 
         SearchBuilder<ManagementServerJoinVO> sb = managementServerJoinDao.createSearchBuilder();
         SearchCriteria<ManagementServerJoinVO> sc = sb.create();
@@ -5753,6 +5763,12 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q
         if (name != null) {
             sc.addAnd("name", SearchCriteria.Op.EQ, name);
         }
+        if (version != null) {
+            sc.addAnd("version", SearchCriteria.Op.EQ, version);
+        }
+        if (keyword != null) {
+            sc.addAnd("version", SearchCriteria.Op.LIKE, "%" + keyword + "%");
+        }
         return managementServerJoinDao.searchAndCount(sc, null);
     }
 

server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java

@@ -76,6 +76,7 @@ import com.cloud.user.Account;
 import com.cloud.user.AccountService;
 import com.cloud.user.dao.UserDataDao;
 import com.cloud.utils.Pair;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
@@ -160,29 +161,50 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
         _count = "select count(distinct temp_zone_pair) from template_view WHERE ";
     }
 
-    private String getTemplateStatus(TemplateJoinVO template) {
-        String templateStatus = null;
-        if (template.getDownloadState() != Status.DOWNLOADED) {
-            templateStatus = "Processing";
-            if (template.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
-                if (template.getDownloadPercent() == 100) {
-                    templateStatus = "Installing Template";
-                } else {
-                    templateStatus = template.getDownloadPercent() + "% Downloaded";
-                }
-            } else if (template.getDownloadState() == Status.BYPASSED) {
-                templateStatus = "Bypassed Secondary Storage";
-            } else if (template.getErrorString() == null) {
-                templateStatus = template.getTemplateState().toString();
-            } else {
-                templateStatus = template.getErrorString().trim();
-            }
-        } else if (template.getDownloadState() == Status.DOWNLOADED) {
-            templateStatus = "Download Complete";
-        } else {
-            templateStatus = "Successfully Installed";
+    private enum TemplateStatus {
+        SUCCESSFULLY_INSTALLED("Successfully Installed"),
+        INSTALLING_TEMPLATE("Installing Template"),
+        INSTALLING_ISO("Installing ISO"),
+        BYPASSED_SECONDARY_STORAGE("Bypassed Secondary Storage"),
+        PROCESSING("Processing"),
+        DOWNLOADING("%d%% Downloaded"),
+        DOWNLOAD_COMPLETE("Download Complete");
+
+        private final String status;
+        TemplateStatus(String status) {
+            this.status = status;
         }
-        return templateStatus;
+        public String getStatus() {
+            return status;
+        }
+        // For statuses that have dynamic details (e.g. "75% Downloaded").
+        public String format(int percent) {
+            return String.format(status, percent);
+        }
+    }
+
+    private String getTemplateStatus(TemplateJoinVO template) {
+        if (template == null) {
+            return  null;
+        }
+        boolean isIso = Storage.ImageFormat.ISO == template.getFormat();
+        TemplateStatus templateStatus;
+        if (template.getDownloadState() == Status.DOWNLOADED) {
+            templateStatus =  isIso ? TemplateStatus.SUCCESSFULLY_INSTALLED : TemplateStatus.DOWNLOAD_COMPLETE;
+        } else if (template.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
+            if (template.getDownloadPercent() == 100) {
+                templateStatus = isIso ? TemplateStatus.INSTALLING_ISO : TemplateStatus.INSTALLING_TEMPLATE;
+            } else {
+                return TemplateStatus.DOWNLOADING.format(template.getDownloadPercent());
+            }
+        } else if (template.getDownloadState() == Status.BYPASSED) {
+            templateStatus = TemplateStatus.BYPASSED_SECONDARY_STORAGE;
+        } else if (StringUtils.isNotBlank(template.getErrorString())) {
+            return template.getErrorString().trim();
+        } else {
+            templateStatus = TemplateStatus.PROCESSING;
+        }
+        return templateStatus.getStatus();
     }
 
     @Override
@@ -503,24 +525,9 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation<Templa
         // If the user is an admin, add the template download status
         if (isAdmin || caller.getId() == iso.getAccountId()) {
             // add download status
-            if (iso.getDownloadState() != Status.DOWNLOADED) {
-                String isoStatus = "Processing";
-                if (iso.getDownloadState() == Status.DOWNLOADED) {
-                    isoStatus = "Download Complete";
-                } else if (iso.getDownloadState() == Status.DOWNLOAD_IN_PROGRESS) {
-                    if (iso.getDownloadPercent() == 100) {
-                        isoStatus = "Installing ISO";
-                    } else {
-                        isoStatus = iso.getDownloadPercent() + "% Downloaded";
-                    }
-                } else if (iso.getDownloadState() == Status.BYPASSED) {
-                    isoStatus = "Bypassed Secondary Storage";
-                } else {
-                    isoStatus = iso.getErrorString();
-                }
-                isoResponse.setStatus(isoStatus);
-            } else {
-                isoResponse.setStatus("Successfully Installed");
+            String templateStatus = getTemplateStatus(iso);
+            if (templateStatus != null) {
+                isoResponse.setStatus(templateStatus);
             }
             isoResponse.setUrl(iso.getUrl());
             List<TemplateDataStoreVO> isosInStore = _templateStoreDao.listByTemplateNotBypassed(iso.getId());

server/src/main/java/com/cloud/configuration/Config.java

@@ -542,13 +542,6 @@ public enum Config {
             "true",
             "Indicates whether or not to automatically reserver system VM standby capacity.",
             null),
-    SystemVMDefaultHypervisor("Advanced",
-            ManagementServer.class,
-            String.class,
-            "system.vm.default.hypervisor",
-            null,
-            "Hypervisor type used to create system vm, valid values are: XenServer, KVM, VMware, Hyperv, VirtualBox, Parralels, BareMetal, Ovm, LXC, Any",
-            null),
     SystemVMRandomPassword(
             "Advanced",
             ManagementServer.class,

server/src/main/java/com/cloud/network/NetworkServiceImpl.java

@@ -1886,6 +1886,18 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
                 null, null, null, null, null, null, null, null, null, null);
     }
 
+    @Override
+    @DB
+    @ActionEvent(eventType = EventTypes.EVENT_NETWORK_CREATE, eventDescription = "creating network")
+    public Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner,
+                                      PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Pair<Integer, Integer> vrIfaceMTUs) throws
+            InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException {
+        return _networkMgr.createGuestNetwork(networkOfferingId, name, displayText,
+                null, null, null, false, null, owner, null, physicalNetwork, zoneId,
+                aclType, null, null, null, null, true, null,
+                null, null, null, null, null, null, null, null, vrIfaceMTUs, null);
+    }
+
     void checkAndSetRouterSourceNatIp(Account owner, CreateNetworkCmd cmd, Network network) throws InsufficientAddressCapacityException, ResourceAllocationException {
         String sourceNatIp = cmd.getSourceNatIP();
         if (sourceNatIp == null) {

server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java

@@ -1747,8 +1747,9 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
             scvm.setParameters("networkId", routerJoinVO.getNetworkId());
             scvm.setParameters("state", VirtualMachine.State.Running);
             List<UserVmJoinVO> vms = userVmJoinDao.search(scvm, null);
-            boolean isDhcpSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dhcp);
-            boolean isDnsSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dns);
+            Provider provider = routerJoinVO.getVpcId() != 0 ? Provider.VPCVirtualRouter : Provider.VirtualRouter;
+            boolean isDhcpSupported = _ntwkSrvcDao.canProviderSupportServiceInNetwork(routerJoinVO.getNetworkId(), Service.Dhcp, provider);
+            boolean isDnsSupported = _ntwkSrvcDao.canProviderSupportServiceInNetwork(routerJoinVO.getNetworkId(), Service.Dns, provider);
             for (UserVmJoinVO vm : vms) {
                 vmsData.append("vmName=").append(vm.getName())
                         .append(",macAddress=").append(vm.getMacAddress())

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -352,7 +352,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
     private final HashMap<String, ResourceStateAdapter> _resourceStateAdapters = new HashMap<>();
 
     private final HashMap<Integer, List<ResourceListener>> _lifeCycleListeners = new HashMap<>();
-    private HypervisorType _defaultSystemVMHypervisor;
 
     private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 30; // seconds
 
@@ -2935,7 +2934,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
 
     @Override
     public boolean configure(final String name, final Map<String, Object> params) throws ConfigurationException {
-        _defaultSystemVMHypervisor = HypervisorType.getType(_configDao.getValue(Config.SystemVMDefaultHypervisor.toString()));
         _gson = GsonHelper.getGson();
 
         _hypervisorsInDC = _hostDao.createSearchBuilder(String.class);
@@ -2981,10 +2979,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
 
     @Override
     public HypervisorType getDefaultHypervisor(final long zoneId) {
-        HypervisorType defaultHyper = HypervisorType.None;
-        if (_defaultSystemVMHypervisor != HypervisorType.None) {
-            defaultHyper = _defaultSystemVMHypervisor;
-        }
+        HypervisorType systemVMDefaultHypervisor = HypervisorType.getType(ResourceManager.SystemVMDefaultHypervisor.value());
 
         final DataCenterVO dc = _dcDao.findById(zoneId);
         if (dc == null) {
@@ -2993,27 +2988,27 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         _dcDao.loadDetails(dc);
         final String defaultHypervisorInZone = dc.getDetail("defaultSystemVMHypervisorType");
         if (defaultHypervisorInZone != null) {
-            defaultHyper = HypervisorType.getType(defaultHypervisorInZone);
+            systemVMDefaultHypervisor = HypervisorType.getType(defaultHypervisorInZone);
         }
 
         final List<VMTemplateVO> systemTemplates = _templateDao.listAllSystemVMTemplates();
         boolean isValid = false;
         for (final VMTemplateVO template : systemTemplates) {
-            if (template.getHypervisorType() == defaultHyper) {
+            if (template.getHypervisorType() == systemVMDefaultHypervisor) {
                 isValid = true;
                 break;
             }
         }
 
         if (isValid) {
-            final List<ClusterVO> clusters = _clusterDao.listByDcHyType(zoneId, defaultHyper.toString());
+            final List<ClusterVO> clusters = _clusterDao.listByDcHyType(zoneId, systemVMDefaultHypervisor.toString());
             if (clusters.isEmpty()) {
                 isValid = false;
             }
         }
 
         if (isValid) {
-            return defaultHyper;
+            return systemVMDefaultHypervisor;
         } else {
             return HypervisorType.None;
         }
@@ -4578,7 +4573,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager,
         return new ConfigKey<?>[] {
                 KvmSshToAgentEnabled,
                 HOST_MAINTENANCE_LOCAL_STRATEGY,
-                SystemVmPreferredArchitecture
+                SystemVmPreferredArchitecture,
+                SystemVMDefaultHypervisor
         };
     }
 }

server/src/main/java/com/cloud/server/ManagementServerImpl.java

@@ -682,15 +682,15 @@ import com.cloud.alert.AlertManager;
 import com.cloud.alert.AlertVO;
 import com.cloud.alert.dao.AlertDao;
 import com.cloud.api.ApiDBUtils;
-import com.cloud.api.query.dao.ManagementServerJoinDao;
 import com.cloud.api.query.dao.StoragePoolJoinDao;
-import com.cloud.api.query.vo.ManagementServerJoinVO;
 import com.cloud.api.query.vo.StoragePoolJoinVO;
 import com.cloud.capacity.Capacity;
 import com.cloud.capacity.CapacityVO;
 import com.cloud.capacity.dao.CapacityDao;
 import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity;
 import com.cloud.cluster.ClusterManager;
+import com.cloud.cluster.ManagementServerHostVO;
+import com.cloud.cluster.dao.ManagementServerHostDao;
 import com.cloud.configuration.Config;
 import com.cloud.configuration.ConfigurationManagerImpl;
 import com.cloud.consoleproxy.ConsoleProxyManagementState;
@@ -792,7 +792,6 @@ import com.cloud.storage.GuestOSHypervisorVO;
 import com.cloud.storage.GuestOSVO;
 import com.cloud.storage.GuestOsCategory;
 import com.cloud.storage.ScopeType;
-import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.storage.Storage;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
@@ -809,6 +808,7 @@ import com.cloud.storage.dao.StoragePoolTagsDao;
 import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.secondary.SecondaryStorageVmManager;
+import com.cloud.storage.snapshot.SnapshotManager;
 import com.cloud.tags.ResourceTagVO;
 import com.cloud.tags.dao.ResourceTagDao;
 import com.cloud.template.TemplateManager;
@@ -1047,7 +1047,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
     @Inject
     private BackupManager backupManager;
     @Inject
-    protected ManagementServerJoinDao managementServerJoinDao;
+    protected ManagementServerHostDao managementServerHostDao;
     @Inject
     ClusterManager _clusterMgr;
 
@@ -5929,7 +5929,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
     @ActionEvent(eventType = EventTypes.EVENT_MANAGEMENT_SERVER_REMOVE, eventDescription = "removing Management Server")
     public boolean removeManagementServer(RemoveManagementServerCmd cmd) {
         final Long id = cmd.getId();
-        ManagementServerJoinVO managementServer = managementServerJoinDao.findById(id);
+        ManagementServerHostVO managementServer = managementServerHostDao.findById(id);
 
         if (managementServer == null) {
             throw new InvalidParameterValueException(String.format("Unable to find a Management Server with ID equal to [%s].", id));
@@ -5939,8 +5939,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
             throw new InvalidParameterValueException(String.format("Unable to remove Management Server with ID [%s]. It can only be removed when it is in the [%s] state, however currently it is in the [%s] state.", managementServer.getUuid(), ManagementServerHost.State.Down.name(), managementServer.getState().name()));
         }
 
-        managementServer.setRemoved(new Date());
-        return managementServerJoinDao.update(id, managementServer);
+        managementServerHostDao.remove(id);
+        return true;
 
     }
 

server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java

@@ -288,7 +288,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
 
 
         for (long zoneId : zonesIds) {
-            DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId);
+            DataStore imageStore = templateMgr.verifyHeuristicRulesForZone(template, zoneId);
 
             if (imageStore == null) {
                 List<DataStore> imageStores = getImageStoresThrowsExceptionIfNotFound(zoneId, profile);
@@ -299,6 +299,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
         }
     }
 
+    protected List<DataStore> getImageStoresThrowsExceptionIfNotFound(long zoneId, TemplateProfile profile) {
+        List<DataStore> imageStores = storeMgr.getImageStoresByZoneIds(zoneId);
+        if (imageStores == null || imageStores.size() == 0) {
+            throw new CloudRuntimeException(String.format("Unable to find image store to download the template [%s].", profile.getTemplate()));
+        }
+        return imageStores;
+    }
+
     protected void standardImageStoreAllocation(List<DataStore> imageStores, VMTemplateVO template) {
         Set<Long> zoneSet = new HashSet<Long>();
         Collections.shuffle(imageStores);
@@ -356,7 +364,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
                 }
 
                 Long zoneId = zoneIdList.get(0);
-                DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId);
+                DataStore imageStore = templateMgr.verifyHeuristicRulesForZone(template, zoneId);
                 List<TemplateOrVolumePostUploadCommand> payloads = new LinkedList<>();
 
                 if (imageStore == null) {

server/src/main/java/com/cloud/template/TemplateManagerImpl.java

@@ -121,6 +121,7 @@ import com.cloud.agent.api.to.DatadiskTO;
 import com.cloud.agent.api.to.DiskTO;
 import com.cloud.agent.api.to.NfsTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.agent.api.to.deployasis.OVFNetworkTO;
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.api.query.vo.UserVmJoinVO;
@@ -131,6 +132,7 @@ import com.cloud.dc.DataCenter;
 import com.cloud.dc.DataCenterVO;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.deploy.DeployDestination;
+import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao;
 import com.cloud.domain.Domain;
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.event.ActionEvent;
@@ -315,6 +317,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
     protected SnapshotHelper snapshotHelper;
     @Inject
     VnfTemplateManager vnfTemplateManager;
+    @Inject
+    TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao;
 
     @Inject
     private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao;
@@ -2217,6 +2221,11 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
             templateType = validateTemplateType(cmd, isAdmin, template.isCrossZones(), template.getHypervisorType());
             if (cmd instanceof UpdateVnfTemplateCmd) {
                 VnfTemplateUtils.validateApiCommandParams(cmd, template);
+                UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd;
+                if (template.isDeployAsIs() && CollectionUtils.isNotEmpty(updateCmd.getVnfNics())) {
+                    List<OVFNetworkTO> ovfNetworks = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(template.getId());
+                    VnfTemplateUtils.validateDeployAsIsTemplateVnfNics(ovfNetworks, updateCmd.getVnfNics());
+                }
                 vnfTemplateManager.updateVnfTemplate(template.getId(), (UpdateVnfTemplateCmd) cmd);
             }
             templateTag = ((UpdateTemplateCmd)cmd).getTemplateTag();
@@ -2415,6 +2424,17 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
                 TemplateType.USER, HypervisorType.External));
     }
 
+    @Override
+    public DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) {
+        HeuristicType heuristicType;
+        if (ImageFormat.ISO.equals(template.getFormat())) {
+            heuristicType = HeuristicType.ISO;
+        } else {
+            heuristicType = HeuristicType.TEMPLATE;
+        }
+        return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template);
+    }
+
     void validateDetails(VMTemplateVO template, Map<String, String> details) {
         if (MapUtils.isEmpty(details)) {
             return;

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -6284,7 +6284,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
     private void verifyTemplate(BaseDeployVMCmd cmd, VirtualMachineTemplate template, Long serviceOfferingId) {
         if (TemplateType.VNF.equals(template.getTemplateType())) {
-            vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds());
+            vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds(), cmd.getVmNetworkMap());
         } else if (cmd instanceof DeployVnfApplianceCmd) {
             throw new InvalidParameterValueException("Can't deploy VNF appliance from a non-VNF template");
         }

server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java

@@ -201,7 +201,14 @@ public class VnfTemplateManagerImpl extends ManagerBase implements VnfTemplateMa
     }
 
     @Override
-    public void validateVnfApplianceNics(VirtualMachineTemplate template, List<Long> networkIds) {
+    public void validateVnfApplianceNics(VirtualMachineTemplate template, List<Long> networkIds, Map<Integer, Long> vmNetworkMap) {
+        if (template.isDeployAsIs()) {
+            if (CollectionUtils.isNotEmpty(networkIds)) {
+                throw new InvalidParameterValueException("VNF nics mappings should be empty for deploy-as-is templates");
+            }
+            validateVnfApplianceNetworksMap(template, vmNetworkMap);
+            return;
+        }
         if (CollectionUtils.isEmpty(networkIds)) {
             throw new InvalidParameterValueException("VNF nics list is empty");
         }
@@ -213,6 +220,18 @@ public class VnfTemplateManagerImpl extends ManagerBase implements VnfTemplateMa
         }
     }
 
+    private void validateVnfApplianceNetworksMap(VirtualMachineTemplate template, Map<Integer, Long> vmNetworkMap) {
+        if (MapUtils.isEmpty(vmNetworkMap)) {
+            throw new InvalidParameterValueException("VNF networks map is empty");
+        }
+        List<VnfTemplateNicVO> vnfNics = vnfTemplateNicDao.listByTemplateId(template.getId());
+        for (VnfTemplateNicVO vnfNic : vnfNics) {
+            if (vnfNic.isRequired() && vmNetworkMap.size() <= vnfNic.getDeviceId()) {
+                throw new InvalidParameterValueException("VNF nic is required but not found: " + vnfNic);
+            }
+        }
+    }
+
     protected Set<Integer> getOpenPortsForVnfAppliance(VirtualMachineTemplate template) {
         Set<Integer> ports = new HashSet<>();
         VnfTemplateDetailVO accessMethodsDetail = vnfTemplateDetailsDao.findDetail(template.getId(), VNF.AccessDetail.ACCESS_METHODS.name().toLowerCase());

server/src/test/java/com/cloud/api/ApiServletTest.java

@@ -33,6 +33,7 @@ import org.apache.cloudstack.api.auth.APIAuthenticationType;
 import org.apache.cloudstack.api.auth.APIAuthenticator;
 import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd;
 import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -97,17 +98,20 @@ public class ApiServletTest {
     @Mock
     AccountService accountMgr;
 
-    @Mock ConfigKey<Boolean> useForwardHeader;
+    @Mock
+    ConfigDepotImpl mockConfigDepot;
+
     StringWriter responseWriter;
 
     ApiServlet servlet;
-    ApiServlet spyServlet;
+
+    private ConfigDepotImpl originalConfigDepot;
+
     @SuppressWarnings("unchecked")
     @Before
     public void setup() throws SecurityException, NoSuchFieldException,
-    IllegalArgumentException, IllegalAccessException, IOException, UnknownHostException {
+            IllegalArgumentException, IllegalAccessException, IOException {
         servlet = new ApiServlet();
-        spyServlet = Mockito.spy(servlet);
         responseWriter = new StringWriter();
         Mockito.when(response.getWriter()).thenReturn(
                 new PrintWriter(responseWriter));
@@ -131,6 +135,7 @@ public class ApiServletTest {
         apiServerField.setAccessible(true);
         apiServerField.set(servlet, apiServer);
 
+        setupConfigDepotMock();
     }
 
     /**
@@ -151,6 +156,33 @@ public class ApiServletTest {
         Field smsField = ApiDBUtils.class.getDeclaredField("s_ms");
         smsField.setAccessible(true);
         smsField.set(null, null);
+        restoreConfigDepot();
+    }
+
+    private void setupConfigDepotMock() throws NoSuchFieldException, IllegalAccessException {
+        Field depotField = ConfigKey.class.getDeclaredField("s_depot");
+        depotField.setAccessible(true);
+        originalConfigDepot = (ConfigDepotImpl) depotField.get(null);
+        depotField.set(null, mockConfigDepot);
+        Mockito.when(mockConfigDepot.getConfigStringValue(
+            Mockito.anyString(),
+            Mockito.any(ConfigKey.Scope.class),
+            Mockito.any()
+        )).thenReturn(null);
+    }
+
+    private void restoreConfigDepot() throws Exception {
+        Field depotField = ConfigKey.class.getDeclaredField("s_depot");
+        depotField.setAccessible(true);
+        depotField.set(null, originalConfigDepot);
+    }
+
+    private void setConfigValue(String configName, String value) {
+        Mockito.when(mockConfigDepot.getConfigStringValue(
+            Mockito.eq(configName),
+            Mockito.eq(ConfigKey.Scope.Global),
+            Mockito.isNull()
+        )).thenReturn(value);
     }
 
     @Test
@@ -261,43 +293,40 @@ public class ApiServletTest {
 
     @Test
     public void getClientAddressWithXForwardedFor() throws UnknownHostException {
-        String[] proxynet = {"127.0.0.0/8"};
-        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
-        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
+        setConfigValue("proxy.header.verify", "true");
+        setConfigValue("proxy.cidr", "127.0.0.0/8");
+        Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
         Mockito.when(request.getHeader(Mockito.eq("X-Forwarded-For"))).thenReturn("192.168.1.1");
-        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
     }
 
     @Test
     public void getClientAddressWithHttpXForwardedFor() throws UnknownHostException {
-        String[] proxynet = {"127.0.0.0/8"};
-        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
-        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
+        setConfigValue("proxy.header.verify", "true");
+        setConfigValue("proxy.cidr", "127.0.0.0/8");
         Mockito.when(request.getHeader(Mockito.eq("HTTP_X_FORWARDED_FOR"))).thenReturn("192.168.1.1");
-        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
     }
 
     @Test
     public void getClientAddressWithRemoteAddr() throws UnknownHostException {
-        String[] proxynet = {"127.0.0.0/8"};
-        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
-        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
-        Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request));
+        setConfigValue("proxy.header.verify", "true");
+        setConfigValue("proxy.cidr", "127.0.0.0/8");
+        Assert.assertEquals(InetAddress.getByName("127.0.0.1"), ApiServlet.getClientAddress(request));
     }
 
     @Test
     public void getClientAddressWithHttpClientIp() throws UnknownHostException {
-        String[] proxynet = {"127.0.0.0/8"};
-        Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet);
-        Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true);
+        setConfigValue("proxy.header.verify", "true");
+        setConfigValue("proxy.cidr", "127.0.0.0/8");
         Mockito.when(request.getHeader(Mockito.eq("HTTP_CLIENT_IP"))).thenReturn("192.168.1.1");
-        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request));
     }
 
     @Test
     public void getClientAddressDefault() throws UnknownHostException {
         Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1");
-        Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request));
+        Assert.assertEquals(InetAddress.getByName("127.0.0.1"), ApiServlet.getClientAddress(request));
     }
 
     @Test

server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java

@@ -49,7 +49,6 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.framework.events.Event;
 import org.apache.cloudstack.framework.events.EventDistributor;
 import org.apache.cloudstack.framework.messagebus.MessageBus;
-import org.apache.cloudstack.secstorage.heuristics.HeuristicType;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
 import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper;
@@ -339,7 +338,7 @@ public class HypervisorTemplateAdapterTest {
 
         Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
         Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class), Mockito.any(TemplateProfile.class));
-        Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
+        Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
         Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class));
 
         _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock);
@@ -355,7 +354,7 @@ public class HypervisorTemplateAdapterTest {
 
         Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
         Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class), Mockito.any(TemplateProfile.class));
-        Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
+        Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
         Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class));
 
         _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock);
@@ -371,7 +370,7 @@ public class HypervisorTemplateAdapterTest {
         List<Long> zoneIds = List.of(1L);
 
         Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds);
-        Mockito.doReturn(dataStoreMock).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
+        Mockito.doReturn(dataStoreMock).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong());
         Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull());
 
         _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock);
@@ -409,26 +408,6 @@ public class HypervisorTemplateAdapterTest {
         _adapter.getImageStoresThrowsExceptionIfNotFound(zoneId, templateProfileMock);
     }
 
-    @Test
-    public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() {
-        VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
-
-        Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.ISO);
-        _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L);
-
-        Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock);
-    }
-
-    @Test
-    public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() {
-        VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
-
-        Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.QCOW2);
-        _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L);
-
-        Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock);
-    }
-
     @Test
     public void isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() {
         DataStore dataStoreMock = Mockito.mock(DataStore.class);

server/src/test/java/com/cloud/template/TemplateManagerImplTest.java

@@ -24,6 +24,7 @@ import com.cloud.api.query.dao.SnapshotJoinDao;
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.configuration.Resource;
 import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao;
 import com.cloud.domain.dao.DomainDao;
 import com.cloud.event.dao.UsageEventDao;
 import com.cloud.exception.InvalidParameterValueException;
@@ -207,6 +208,8 @@ public class TemplateManagerImplTest {
     VnfTemplateManager vnfTemplateManager;
     @Inject
     SnapshotJoinDao snapshotJoinDao;
+    @Inject
+    TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao;
 
     @Inject
     HeuristicRuleHelper heuristicRuleHelperMock;
@@ -753,6 +756,26 @@ public class TemplateManagerImplTest {
         Assert.assertNull(type);
     }
 
+    @Test
+    public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() {
+        VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
+
+        Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(Storage.ImageFormat.ISO);
+        templateManager.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L);
+
+        Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock);
+    }
+
+    @Test
+    public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() {
+        VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class);
+
+        Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2);
+        templateManager.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L);
+
+        Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock);
+    }
+
     @Configuration
     @ComponentScan(basePackageClasses = {TemplateManagerImpl.class},
             includeFilters = {@ComponentScan.Filter(value = TestConfiguration.Library.class, type = FilterType.CUSTOM)},
@@ -959,6 +982,11 @@ public class TemplateManagerImplTest {
             return Mockito.mock(VnfTemplateManager.class);
         }
 
+        @Bean
+        public TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao() {
+            return Mockito.mock(TemplateDeployAsIsDetailsDao.class);
+        }
+
         @Bean
         public SnapshotHelper snapshotHelper() {
             return Mockito.mock(SnapshotHelper.class);

server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java

@@ -1164,7 +1164,7 @@ public class UserVmManagerImplTest {
         when(templateMock.isDeployAsIs()).thenReturn(false);
         when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2);
         when(templateMock.getUserDataId()).thenReturn(null);
-        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class));
+        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class));
 
         ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class);
         when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock);
@@ -1176,7 +1176,7 @@ public class UserVmManagerImplTest {
 
         UserVm result = userVmManagerImpl.createVirtualMachine(deployVMCmd);
         assertEquals(userVmVoMock, result);
-        Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null);
+        Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null, Collections.emptyMap());
         Mockito.verify(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(),
                 any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(),
                 any(), any(), any(), any(), eq(true), any(), any(), any());
@@ -1420,7 +1420,7 @@ public class UserVmManagerImplTest {
         when(templateMock.isDeployAsIs()).thenReturn(false);
         when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2);
         when(templateMock.getUserDataId()).thenReturn(null);
-        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class));
+        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class));
 
         ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class);
         when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock);
@@ -3880,7 +3880,7 @@ public class UserVmManagerImplTest {
         when(templateMock.isDeployAsIs()).thenReturn(false);
         when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2);
         when(templateMock.getUserDataId()).thenReturn(null);
-        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class));
+        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), any());
         when(_dcMock.isLocalStorageEnabled()).thenReturn(false);
         when(_dcMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic);
         Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(),
@@ -3918,7 +3918,7 @@ public class UserVmManagerImplTest {
         when(templateMock.isDeployAsIs()).thenReturn(false);
         when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2);
         when(templateMock.getUserDataId()).thenReturn(null);
-        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class));
+        Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), any());
         when(_dcMock.isLocalStorageEnabled()).thenReturn(false);
         when(_dcMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic);
         Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(),

server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java

@@ -231,6 +231,13 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches
         return null;
     }
 
+    @Override
+    public Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner,
+           PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Pair<Integer, Integer> vrIfaceMTUs)
+            throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException {
+        return null;
+    }
+
     /* (non-Javadoc)
      * @see com.cloud.network.NetworkService#searchForNetworks(com.cloud.api.commands.ListNetworksCmd)
      */

server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java

@@ -228,25 +228,25 @@ public class VnfTemplateManagerImplTest {
     @Test
     public void testValidateVnfApplianceNicsWithRequiredNics() {
         List<Long> networkIds = Arrays.asList(200L, 201L);
-        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds);
+        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null);
     }
 
     @Test
     public void testValidateVnfApplianceNicsWithAllNics() {
         List<Long> networkIds = Arrays.asList(200L, 201L, 202L);
-        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds);
+        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null);
     }
 
     @Test(expected = InvalidParameterValueException.class)
     public void testValidateVnfApplianceNicsWithEmptyList() {
         List<Long> networkIds = new ArrayList<>();
-        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds);
+        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null);
     }
 
     @Test(expected = InvalidParameterValueException.class)
     public void testValidateVnfApplianceNicsWithMissingNetworkId() {
         List<Long> networkIds = Arrays.asList(200L);
-        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds);
+        vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null);
     }
 
     @Test

tools/marvin/marvin/lib/base.py

@@ -1725,11 +1725,12 @@ class Template:
                 # If template is ready,
                 # template.status = Download Complete
                 # Downloading - x% Downloaded
+                # Processing - Initial status
                 # Error - Any other string
                 if template.status == 'Download Complete' and template.isready:
                     return
 
-                elif 'Downloaded' in template.status:
+                elif 'Downloaded' in template.status or template.status == 'Processing':
                     retries = retries - 1
                     continue
 

ui/public/locales/en.json

@@ -47,6 +47,7 @@
 "label.acl.rules": "ACL rules",
 "label.acl.reason.description": "Enter the reason behind an ACL rule.",
 "label.aclid": "ACL",
+"label.aclname": "ACL name",
 "label.acl.rule.name": "ACL rule name",
 "label.acquire.new.ip": "Acquire new IP",
 "label.acquire.new.secondary.ip": "Acquire new secondary IP",
@@ -439,6 +440,8 @@
 "label.auto.assign": "Automatically assign",
 "label.auto.assign.diskoffering.disk.size": "Automatically assign offering matching the disk size",
 "label.auto.assign.random.ip": "Automatically assign a random IP address",
+"label.auto.refresh.statistics": "Period between auto refreshes",
+"label.auto.refresh.statistics.none": "None",
 "label.automigrate.volume": "Auto migrate volume to another storage pool if required",
 "label.autoscale.vm.groups": "AutoScaling Groups",
 "label.autoscale.vm.profile": "AutoScale Instance Profile",
@@ -2768,7 +2771,7 @@
 "label.vnf.cidr.list": "CIDR from which access to the VNF appliance's Management interface should be allowed from",
 "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.",
 "label.vnf.configure.management": "Configure network rules for VNF's management interfaces",
-"label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances",
+"label.vnf.configure.management.tooltip": "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances",
 "label.vnf.detail.add": "Add VNF detail",
 "label.vnf.detail.remove": "Remove VNF detail",
 "label.vnf.details": "VNF Details",

ui/public/locales/pt_BR.json

@@ -288,6 +288,8 @@
 "label.author.name": "Nome do autor",
 "label.auto.assign.diskoffering.disk.size": "Atribuir automaticamente a oferta correspondente ao tamanho do disco",
 "label.auto.assign.random.ip": "Atribuir automaticamente um enderec\u0327o de IP aleat\u00f3rio",
+"label.auto.refresh.statistics": "Tempo entre atualiza\u00e7\u00f5es autom\u00e1ticas",
+"label.auto.refresh.statistics.none": "Nenhum",
 "label.autoscalingenabled": "Escalonamento autom\u00e1tico",
 "label.availability": "Disponibilidade",
 "label.available": "Dispon\u00edvel",
@@ -610,6 +612,12 @@
 "label.download.state": "Estado do download",
 "label.dpd": "Detec\u00e7\u00e3o de correspondente morto",
 "label.driver": "Driver",
+"label.duration.custom": "Personalizado",
+"label.duration.1hour": "1 hora",
+"label.duration.6hours": "6 horas",
+"label.duration.12hours": "12 horas",
+"label.duration.24hours": "24 horas",
+"label.duration.7days": "7 dias",
 "label.dynamicscalingenabled": "Escalonamento din\u00e2mico habilitado",
 "label.dynamicscalingenabled.tooltip": "VM s\u00f3 pode ser dinamicamente escalonada quando o escalonamento din\u00e2mico estiver habilitado no template, oferta de computa\u00e7\u00e3o e nas configura\u00e7\u00e3oes globais",
 "label.edit": "Editar",

ui/src/components/view/DeployVMFromBackup.vue

@@ -2140,7 +2140,9 @@ export default {
         this.owner.domainid = null
         this.owner.projectid = OwnerOptions.selectedProject
       }
-      this.resetData()
+      if (OwnerOptions.initialized) {
+        this.resetData()
+      }
     },
     fetchZones (zoneId, listZoneAllow) {
       this.zones = []

ui/src/components/view/StatsTab.vue

@@ -41,9 +41,9 @@
           <a-radio-group
             v-model:value="durationSelectorValue"
             buttonStyle="solid"
-            @change="handleDurationChange">
+            @change="updateVirtualMachineStats">
             <a-radio-button value="">
-              {{ $t('1 hour') }}
+              {{ $t('label.duration.1hour') }}
             </a-radio-button>
             <a-radio-button value="6hours" v-if="statsRetentionTime >= 60">
               {{ $t('label.duration.6hours') }}
@@ -62,6 +62,16 @@
             </a-radio-button>
           </a-radio-group>
           <InfoCircleOutlined class="info-icon" :title="$t('label.see.more.info.shown.charts')" @click="onClickShowResourceInfoModal('CHART')"/>
+          <span>{{$t('label.auto.refresh.statistics')}}</span>
+          <a-select
+            v-model:value="refreshTime"
+            style="width: 100px">
+            <a-select-option value="0">{{$t('label.auto.refresh.statistics.none')}}</a-select-option>
+            <a-select-option value="5000">5s</a-select-option>
+            <a-select-option value="30000">30s</a-select-option>
+            <a-select-option value="60000">1min</a-select-option>
+            <a-select-option value="300000">5min</a-select-option>
+          </a-select>
         </div>
         <div class="ant-tag" v-if="durationSelectorValue==='custom'">
           <a-button @click="openFilter()">
@@ -297,6 +307,8 @@ export default {
       selectedDiskUnitOfMeasurement: 'KiB',
       diskUnitsOfMeasurement: ['KiB', 'MiB', 'GiB'],
       chartLabels: [],
+      refreshTime: '0',
+      refreshIntervalId: null,
       resourceUsageHistory: {
         cpu: [],
         memory: {
@@ -334,6 +346,9 @@ export default {
   mounted () {
     this.fetchData()
   },
+  unmounted () {
+    window.clearInterval(this.refreshIntervalId)
+  },
   computed: {
     statsRetentionTime () {
       if (this.resourceType === 'Volume') {
@@ -371,6 +386,15 @@ export default {
         return
       }
       this.fetchData()
+    },
+    refreshTime: function () {
+      this.updateVirtualMachineStats()
+      if (this.refreshTime === '0') return window.clearInterval(this.refreshIntervalId)
+
+      window.clearInterval(this.refreshIntervalId)
+      this.refreshIntervalId = window.setInterval(() => {
+        this.updateVirtualMachineStats()
+      }, parseInt(this.refreshTime))
     }
   },
   methods: {
@@ -398,26 +422,10 @@ export default {
       this.resourceTypeToShowInfo = resource
       this.showResourceInfoModal = true
     },
-    handleDurationChange () {
-      var now = this.getEndDate()
-      var start = new Date(now)
-      switch (this.durationSelectorValue) {
-        case '6hours':
-          start.setHours(start.getHours() - 6)
-          break
-        case '12hours':
-          start.setHours(start.getHours() - 12)
-          break
-        case 'day':
-          start.setDate(start.getDate() - 1)
-          break
-        case 'week':
-          start.setDate(start.getDate() - 7)
-          break
-        default:
-          start.setHours(start.getHours() - 1)
-      }
-      this.handleSubmit({ startDate: start, endDate: now })
+    updateVirtualMachineStats () {
+      const start = this.getStartDate()
+      const end = this.getEndDate()
+      this.handleSubmit({ startDate: start, endDate: end })
     },
     handleSubmit (values) {
       if (values.startDate) {
@@ -437,9 +445,19 @@ export default {
       this.showFilterStatsModal = false
     },
     getStartDate () {
-      var now = new Date()
-      now.setHours(now.getHours() - 1)
-      return now
+      const now = new Date()
+      switch (this.durationSelectorValue) {
+        case '6hours':
+          return now.setHours(now.getHours() - 6)
+        case '12hours':
+          return now.setHours(now.getHours() - 12)
+        case 'day':
+          return now.setDate(now.getDate() - 1)
+        case 'week':
+          return now.setDate(now.getDate() - 7)
+        default:
+          return now.setHours(now.getHours() - 1)
+      }
     },
     getEndDate () {
       return new Date()

ui/src/views/compute/DeployVnfAppliance.vue

@@ -360,6 +360,7 @@
                   <div>
                     <vnf-nics-selection
                       :items="templateVnfNics"
+                      :templateNics="templateNics"
                       :networks="networks"
                       @update-vnf-nic-networks="($event) => updateVnfNicNetworks($event)" />
                   </div>
@@ -1303,7 +1304,8 @@ export default {
       return tabList
     },
     showVnfNicsSection () {
-      return this.networks && this.networks.length > 0 && this.vm.templateid && this.templateVnfNics && this.templateVnfNics.length > 0
+      return ((this.networks && this.networks.length > 0) || (this.templateNics && this.templateNics.length > 0)) &&
+        this.vm.templateid && this.templateVnfNics && this.templateVnfNics.length > 0
     },
     showVnfConfigureManagement () {
       const managementDeviceIds = []
@@ -1313,6 +1315,11 @@ export default {
         }
       }
       for (const deviceId of managementDeviceIds) {
+        if (this.templateNics && this.templateNics[deviceId] &&
+          ((this.templateNics[deviceId].selectednetworktype === 'Isolated' && this.templateNics[deviceId].selectednetworkvpcid === undefined) ||
+            (this.templateNics[deviceId].selectednetworktype === 'Shared' && this.templateNics[deviceId].selectednetworkwithsg))) {
+          return true
+        }
         if (this.vnfNicNetworks && this.vnfNicNetworks[deviceId] &&
           ((this.vnfNicNetworks[deviceId].type === 'Isolated' && this.vnfNicNetworks[deviceId].vpcid === undefined) ||
             (this.vnfNicNetworks[deviceId].type === 'Shared' && this.vnfNicNetworks[deviceId].service.filter(svc => svc.name === 'SecurityGroupProvider')))) {
@@ -2090,7 +2097,7 @@ export default {
         // All checked networks should be used and only once.
         // Required NIC must be associated to a network
         // DeviceID must be consequent
-        if (this.templateVnfNics && this.templateVnfNics.length > 0) {
+        if (this.templateVnfNics && this.templateVnfNics.length > 0 && (!this.templateNics || this.templateNics.length === 0)) {
           let nextDeviceId = 0
           const usedNetworkIds = []
           const keys = Object.keys(this.vnfNicNetworks)
@@ -2720,6 +2727,9 @@ export default {
             var network = this.options.networks[Math.min(i, this.options.networks.length - 1)]
             nic.selectednetworkid = network.id
             nic.selectednetworkname = network.name
+            nic.selectednetworktype = network.type
+            nic.selectednetworkvpcid = network.vpcid
+            nic.selectednetworkwithsg = network.service.filter(svc => svc.name === 'SecurityGroupProvider').length > 0
             this.nicToNetworkSelection.push({ nic: nic.id, network: network.id })
           }
         }

ui/src/views/compute/wizard/VnfNicsSelection.vue

@@ -50,6 +50,7 @@
       <template #network="{ record }">
         <a-form-item style="display: block" :name="'nic-' + record.deviceid">
           <a-select
+            disabled="templateNics && templateNics.length > 0"
             @change="updateNicNetworkValue($event, record.deviceid)"
             optionFilterProp="label"
             :filterOption="(input, option) => {
@@ -75,6 +76,10 @@ export default {
       type: Array,
       default: () => []
     },
+    templateNics: {
+      type: Array,
+      default: () => []
+    },
     networks: {
       type: Array,
       default: () => []