Merge 9a2ba84d42 into cd5bb09d0d

engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java

@@ -400,7 +400,9 @@ public class DefaultEndPointSelector implements EndPointSelector {
         }
         if (object instanceof TemplateInfo) {
             TemplateInfo tmplInfo = (TemplateInfo)object;
-            if (store.getScope().getScopeType() == ScopeType.ZONE && store.getScope().getScopeId() == null && tmplInfo.getTemplateType() == TemplateType.SYSTEM) {
+            if (tmplInfo.getTemplateType() == TemplateType.SYSTEM &&
+                (store.getScope().getScopeType() == ScopeType.REGION ||
+                 (store.getScope().getScopeType() == ScopeType.ZONE && store.getScope().getScopeId() == null))) {
                 return LocalHostEndpoint.getEndpoint(); // for bootstrap system vm template downloading to region image store
             }
         }

services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java

@@ -1249,6 +1249,10 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
     protected void addSecondaryStorageServerAddressToBuffer(StringBuilder buffer, List<DataStore> dataStores, String vmName) {
         List<String> addresses = new ArrayList<>();
         for (DataStore dataStore: dataStores) {
+            // S3 and other object stores may not have a URL, so better to skip them
+            if (dataStore == null || dataStore.getTO() == null || dataStore.getTO().getUrl() == null) {
+                continue;
+            }
             String url = dataStore.getTO().getUrl();
             String[] urlArray = url.split("/");
 

services/secondary-storage/controller/src/test/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImplTest.java

@@ -117,6 +117,45 @@ public class SecondaryStorageManagerImplTest {
         runAddSecondaryStorageServerAddressToBufferTest(addresses, StringUtils.join(List.of(randomIp1, randomIp2), ","));
     }
 
+    @Test
+    public void testAddSecondaryStorageServerAddressToBufferWithNullEntries() {
+        String randomIp1 = InetAddresses.fromInteger(secureRandom.nextInt()).getHostAddress();
+        String randomIp2 = InetAddresses.fromInteger(secureRandom.nextInt()).getHostAddress();
+
+        List<DataStore> dataStores = new ArrayList<>();
+
+        DataStore validStore1 = Mockito.mock(DataStore.class);
+        DataStoreTO validStoreTO1 = Mockito.mock(DataStoreTO.class);
+        when(validStoreTO1.getUrl()).thenReturn(String.format("http://%s", randomIp1));
+        when(validStore1.getTO()).thenReturn(validStoreTO1);
+        dataStores.add(validStore1);
+
+        dataStores.add(null);
+
+        DataStore nullToStore = Mockito.mock(DataStore.class);
+        when(nullToStore.getTO()).thenReturn(null);
+        dataStores.add(nullToStore);
+
+        DataStore nullUrlStore = Mockito.mock(DataStore.class);
+        DataStoreTO nullUrlStoreTO = Mockito.mock(DataStoreTO.class);
+        when(nullUrlStoreTO.getUrl()).thenReturn(null);
+        when(nullUrlStore.getTO()).thenReturn(nullUrlStoreTO);
+        dataStores.add(nullUrlStore);
+
+        DataStore validStore2 = Mockito.mock(DataStore.class);
+        DataStoreTO validStoreTO2 = Mockito.mock(DataStoreTO.class);
+        when(validStoreTO2.getUrl()).thenReturn(String.format("http://%s", randomIp2));
+        when(validStore2.getTO()).thenReturn(validStoreTO2);
+        dataStores.add(validStore2);
+
+        StringBuilder builder = new StringBuilder();
+        secondaryStorageManager.addSecondaryStorageServerAddressToBuffer(builder, dataStores, "VM");
+        String result = builder.toString();
+        result = result.contains("=") ? result.split("=")[1] : null;
+
+        assertEquals(StringUtils.join(List.of(randomIp1, randomIp2), ","), result);
+    }
+
     @Test
     public void testCreateSecondaryStorageVm_New() {
         long dataCenterId = 1L;

ui/public/locales/en.json

@@ -2527,7 +2527,7 @@
 "label.vnf.app.action.reinstall": "Reinstall VNF Appliance",
 "label.vnf.cidr.list": "CIDR from which access to the VNF appliance's Management interface should be allowed from",
 "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.",
-"label.vnf.configure.management": "Configure Firewall and Port Forwarding rules for VNF's management interfaces",
+"label.vnf.configure.management": "Configure network rules for VNF's management interfaces",
 "label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances",
 "label.vnf.detail.add": "Add VNF detail",
 "label.vnf.detail.remove": "Remove VNF detail",

ui/src/config/section/network.js

@@ -356,7 +356,10 @@ export default {
       permission: ['listVnfAppliances'],
       resourceType: 'UserVm',
       params: () => {
-        return { details: 'servoff,tmpl,nics', isvnf: true }
+        return {
+          details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics',
+          isvnf: true
+        }
       },
       columns: () => {
         const fields = ['name', 'state', 'ipaddress']

ui/src/views/compute/DeployVnfAppliance.vue

@@ -1305,7 +1305,7 @@ export default {
       for (const deviceId of managementDeviceIds) {
         if (this.vnfNicNetworks && this.vnfNicNetworks[deviceId] &&
           ((this.vnfNicNetworks[deviceId].type === 'Isolated' && this.vnfNicNetworks[deviceId].vpcid === undefined) ||
-            (this.vnfNicNetworks[deviceId].type === 'Shared' && this.zone.securitygroupsenabled))) {
+            (this.vnfNicNetworks[deviceId].type === 'Shared' && this.vnfNicNetworks[deviceId].service.filter(svc => svc.name === 'SecurityGroupProvider')))) {
           return true
         }
       }

ui/src/views/network/VnfAppliancesTab.vue

@@ -120,7 +120,7 @@ export default {
   methods: {
     fetchData () {
       var params = {
-        details: 'servoff,tmpl,nics',
+        details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics',
         isVnf: true,
         listAll: true
       }

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -40,9 +40,11 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import org.apache.cloudstack.utils.security.KeyStoreUtils;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -708,13 +710,31 @@ public class Script implements Callable<String> {
         return executeCommandForExitValue(0, command);
     }
 
+    private static void cleanupProcesses(AtomicReference<List<Process>> processesRef) {
+        List<Process> processes = processesRef.get();
+        if (CollectionUtils.isNotEmpty(processes)) {
+            for (Process process : processes) {
+                if (process == null) {
+                    continue;
+                }
+                LOGGER.trace(String.format("Cleaning up process [%s] from piped commands.", process.pid()));
+                IOUtils.closeQuietly(process.getErrorStream());
+                IOUtils.closeQuietly(process.getOutputStream());
+                IOUtils.closeQuietly(process.getInputStream());
+                process.destroyForcibly();
+            }
+        }
+    }
+
     public static Pair<Integer, String> executePipedCommands(List<String[]> commands, long timeout) {
         if (timeout <= 0) {
             timeout = DEFAULT_TIMEOUT;
         }
+        final AtomicReference<List<Process>> processesRef = new AtomicReference<>();
         Callable<Pair<Integer, String>> commandRunner = () -> {
             List<ProcessBuilder> builders = commands.stream().map(ProcessBuilder::new).collect(Collectors.toList());
             List<Process> processes = ProcessBuilder.startPipeline(builders);
+            processesRef.set(processes);
             Process last = processes.get(processes.size()-1);
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(last.getInputStream()))) {
                 String line;
@@ -741,6 +761,8 @@ public class Script implements Callable<String> {
             result.second(ERR_TIMEOUT);
         } catch (InterruptedException | ExecutionException e) {
             LOGGER.error("Error executing piped commands", e);
+        } finally {
+            cleanupProcesses(processesRef);
         }
         return result;
     }

utils/src/main/java/com/cloud/utils/storage/S3/S3Utils.java

@@ -114,6 +114,8 @@ public final class S3Utils {
             LOGGER.debug(format("Setting the end point for S3 client with access key %1$s to %2$s.", clientOptions.getAccessKey(), clientOptions.getEndPoint()));
 
             client.setEndpoint(clientOptions.getEndPoint());
+            // Enable path-style access for S3-compatible storage
+            client.setS3ClientOptions(com.amazonaws.services.s3.S3ClientOptions.builder().setPathStyleAccess(true).build());
         }
 
         TRANSFERMANAGER_ACCESSKEY_MAP.put(clientOptions.getAccessKey(), new TransferManager(client));