Merge branch '4.20' into 4.22

api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java

@@ -42,7 +42,7 @@ public abstract class BaseUpdateTemplateOrIsoCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = TemplateResponse.class, required = true, description = "The ID of the image file")
     private Long id;
 
-    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the image file")
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, length = 251, description = "The name of the image file")
     private String templateName;
 
     @Parameter(name = ApiConstants.OS_TYPE_ID,

api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java

@@ -78,6 +78,7 @@ public class UpdateNetworkOfferingCmd extends BaseCmd {
 
     @Parameter(name = ApiConstants.DOMAIN_ID,
             type = CommandType.STRING,
+            length = 4096,
             description = "The ID of the containing domain(s) as comma separated string, public for public offerings")
     private String domainIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java

@@ -75,6 +75,7 @@ public class UpdateDiskOfferingCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java

@@ -70,6 +70,7 @@ public class UpdateServiceOfferingCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java

@@ -65,6 +65,7 @@ public class UpdateVPCOfferingCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java

@@ -70,7 +70,7 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd {
     @Parameter(name = ApiConstants.IS_EXTRACTABLE, type = CommandType.BOOLEAN, description = "True if the ISO or its derivatives are extractable; default is false")
     private Boolean extractable;
 
-    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "The name of the ISO")
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, length = 251, description = "The name of the ISO")
     private String isoName;
 
     @Parameter(name = ApiConstants.OS_TYPE_ID,

engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java

@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.upgrade.dao;
+
+import java.io.InputStream;
+import java.sql.Connection;
+
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class Upgrade42020to42030 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate {
+
+    @Override
+    public String[] getUpgradableVersionRange() {
+        return new String[]{"4.20.2.0", "4.20.3.0"};
+    }
+
+    @Override
+    public String getUpgradedVersion() {
+        return "4.20.3.0";
+    }
+
+    @Override
+    public boolean supportsRollingUpgrade() {
+        return false;
+    }
+
+    @Override
+    public InputStream[] getPrepareScripts() {
+        final String scriptFile = "META-INF/db/schema-42020to42030.sql";
+        final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile);
+        if (script == null) {
+            throw new CloudRuntimeException("Unable to find " + scriptFile);
+        }
+
+        return new InputStream[] {script};
+    }
+
+    @Override
+    public void performDataMigration(Connection conn) {
+    }
+
+    @Override
+    public InputStream[] getCleanupScripts() {
+        return null;
+    }
+
+    @Override
+    public void updateSystemVmTemplates(Connection conn) {
+    }
+}

engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDao.java

@@ -31,4 +31,6 @@ public interface VMScheduledJobDao extends GenericDao<VMScheduledJobVO, Long> {
     int expungeJobsForSchedules(List<Long> scheduleId, Date dateAfter);
 
     int expungeJobsBefore(Date currentTimestamp);
+
+    VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp);
 }

engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java

@@ -39,6 +39,8 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
 
     private final SearchBuilder<VMScheduledJobVO> expungeJobForScheduleSearch;
 
+    private final SearchBuilder<VMScheduledJobVO> scheduleAndTimestampSearch;
+
     static final String SCHEDULED_TIMESTAMP = "scheduled_timestamp";
 
     static final String VM_SCHEDULE_ID = "vm_schedule_id";
@@ -58,6 +60,11 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
         expungeJobForScheduleSearch.and(VM_SCHEDULE_ID, expungeJobForScheduleSearch.entity().getVmScheduleId(), SearchCriteria.Op.IN);
         expungeJobForScheduleSearch.and(SCHEDULED_TIMESTAMP, expungeJobForScheduleSearch.entity().getScheduledTime(), SearchCriteria.Op.GTEQ);
         expungeJobForScheduleSearch.done();
+
+        scheduleAndTimestampSearch = createSearchBuilder();
+        scheduleAndTimestampSearch.and(VM_SCHEDULE_ID, scheduleAndTimestampSearch.entity().getVmScheduleId(), SearchCriteria.Op.EQ);
+        scheduleAndTimestampSearch.and(SCHEDULED_TIMESTAMP, scheduleAndTimestampSearch.entity().getScheduledTime(), SearchCriteria.Op.EQ);
+        scheduleAndTimestampSearch.done();
     }
 
     /**
@@ -92,4 +99,12 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
         sc.setParameters(SCHEDULED_TIMESTAMP, date);
         return expunge(sc);
     }
+
+    @Override
+    public VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp) {
+        SearchCriteria<VMScheduledJobVO> sc = scheduleAndTimestampSearch.create();
+        sc.setParameters(VM_SCHEDULE_ID, scheduleId);
+        sc.setParameters(SCHEDULED_TIMESTAMP, scheduledTimestamp);
+        return findOneBy(sc);
+    }
 }

engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql

@@ -0,0 +1,22 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+--;
+-- Schema upgrade from 4.20.2.0 to 4.20.3.0
+--;
+
+ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048);

engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql

@@ -25,3 +25,5 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'b
 
 -- Add vm_id column to cloud_usage.usage_volume table
 CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"');
+
+ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048);

framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java

@@ -89,6 +89,7 @@ import net.sf.cglib.proxy.NoOp;
 import net.sf.ehcache.Cache;
 import net.sf.ehcache.CacheManager;
 import net.sf.ehcache.Element;
+import org.springframework.util.ClassUtils;
 
 /**
  *  GenericDaoBase is a simple way to implement DAOs.  It DOES NOT
@@ -2058,16 +2059,22 @@ public abstract class GenericDaoBase<T, ID extends Serializable> extends Compone
 
     @DB()
     protected void setField(final Object entity, final ResultSet rs, ResultSetMetaData meta, final int index) throws SQLException {
-        Attribute attr = _allColumns.get(new Pair<String, String>(meta.getTableName(index), meta.getColumnName(index)));
+        String tableName = meta.getTableName(index);
+        String columnName = meta.getColumnName(index);
+        Attribute attr = _allColumns.get(new Pair<>(tableName, columnName));
         if (attr == null) {
             // work around for mysql bug to return original table name instead of view name in db view case
             Table tbl = entity.getClass().getSuperclass().getAnnotation(Table.class);
             if (tbl != null) {
-                attr = _allColumns.get(new Pair<String, String>(tbl.name(), meta.getColumnLabel(index)));
+                attr = _allColumns.get(new Pair<>(tbl.name(), meta.getColumnLabel(index)));
             }
         }
-        assert (attr != null) : "How come I can't find " + meta.getCatalogName(index) + "." + meta.getColumnName(index);
-        setField(entity, attr.field, rs, index);
+        if(attr == null) {
+            logger.warn(String.format("Failed to find attribute in the entity %s to map column %s.%s (%s)",
+                    ClassUtils.getUserClass(entity).getSimpleName(), tableName, columnName));
+        } else {
+            setField(entity, attr.field, rs, index);
+        }
     }
 
     @Override

server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java

@@ -162,7 +162,13 @@ public class VMSchedulerImpl extends ManagerBase implements VMScheduler, Configu
         }
 
         Date scheduledDateTime = Date.from(ts.toInstant());
-        VMScheduledJobVO scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime);
+        VMScheduledJobVO scheduledJob = vmScheduledJobDao.findByScheduleAndTimestamp(vmSchedule.getId(), scheduledDateTime);
+        if (scheduledJob != null) {
+            logger.trace("Job is already scheduled for schedule {} at {}", vmSchedule, scheduledDateTime);
+            return scheduledDateTime;
+        }
+
+        scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime);
         try {
             vmScheduledJobDao.persist(scheduledJob);
             ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, vm.getAccountId(), actionEventMap.get(vmSchedule.getAction()),

ui/public/locales/en.json

@@ -2752,7 +2752,7 @@
 "label.vnf.app.action.reinstall": "Reinstall VNF Appliance",
 "label.vnf.cidr.list": "CIDR from which access to the VNF appliance's Management interface should be allowed from",
 "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.",
-"label.vnf.configure.management": "Configure Firewall and Port Forwarding rules for VNF's management interfaces",
+"label.vnf.configure.management": "Configure network rules for VNF's management interfaces",
 "label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances",
 "label.vnf.detail.add": "Add VNF detail",
 "label.vnf.detail.remove": "Remove VNF detail",

ui/src/config/section/network.js

@@ -356,7 +356,10 @@ export default {
       permission: ['listVnfAppliances'],
       resourceType: 'UserVm',
       params: () => {
-        return { details: 'servoff,tmpl,nics', isvnf: true }
+        return {
+          details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics',
+          isvnf: true
+        }
       },
       columns: () => {
         const fields = ['name', 'state', 'ipaddress']

ui/src/utils/plugins.js

@@ -224,18 +224,19 @@ export const notifierPlugin = {
         if (error.response.status) {
           msg = `${i18n.global.t('message.request.failed')} (${error.response.status})`
         }
-        if (error.message) {
-          desc = error.message
-        }
-        if (error.response.headers && 'x-description' in error.response.headers) {
+        if (error.response.headers?.['x-description']) {
           desc = error.response.headers['x-description']
-        }
-        if (desc === '' && error.response.data) {
+        } else if (error.response.data) {
           const responseKey = _.findKey(error.response.data, 'errortext')
           if (responseKey) {
             desc = error.response.data[responseKey].errortext
+          } else if (typeof error.response.data === 'string') {
+            desc = error.response.data
           }
         }
+        if (!desc && error.message) {
+          desc = error.message
+        }
       }
       let countNotify = store.getters.countNotify
       countNotify++

ui/src/views/compute/DeployVnfAppliance.vue

@@ -1315,7 +1315,7 @@ export default {
       for (const deviceId of managementDeviceIds) {
         if (this.vnfNicNetworks && this.vnfNicNetworks[deviceId] &&
           ((this.vnfNicNetworks[deviceId].type === 'Isolated' && this.vnfNicNetworks[deviceId].vpcid === undefined) ||
-            (this.vnfNicNetworks[deviceId].type === 'Shared' && this.zone.securitygroupsenabled))) {
+            (this.vnfNicNetworks[deviceId].type === 'Shared' && this.vnfNicNetworks[deviceId].service.filter(svc => svc.name === 'SecurityGroupProvider')))) {
           return true
         }
       }

ui/src/views/image/RegisterOrUploadTemplate.vue

@@ -677,11 +677,7 @@ export default {
         this.$emit('refresh-data')
         this.closeAction()
       }).catch(e => {
-        this.$notification.error({
-          message: this.$t('message.upload.failed'),
-          description: `${this.$t('message.upload.template.failed.description')} -  ${e}`,
-          duration: 0
-        })
+        this.$notifyError(e)
       })
     },
     fetchCustomHypervisorName () {

ui/src/views/network/VnfAppliancesTab.vue

@@ -120,7 +120,7 @@ export default {
   methods: {
     fetchData () {
       var params = {
-        details: 'servoff,tmpl,nics',
+        details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics',
         isVnf: true,
         listAll: true
       }

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -39,9 +39,11 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
 import org.apache.cloudstack.utils.security.KeyStoreUtils;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -729,13 +731,31 @@ public class Script implements Callable<String> {
         return executeCommandForExitValue(0, command);
     }
 
+    private static void cleanupProcesses(AtomicReference<List<Process>> processesRef) {
+        List<Process> processes = processesRef.get();
+        if (CollectionUtils.isNotEmpty(processes)) {
+            for (Process process : processes) {
+                if (process == null) {
+                    continue;
+                }
+                LOGGER.trace(String.format("Cleaning up process [%s] from piped commands.", process.pid()));
+                IOUtils.closeQuietly(process.getErrorStream());
+                IOUtils.closeQuietly(process.getOutputStream());
+                IOUtils.closeQuietly(process.getInputStream());
+                process.destroyForcibly();
+            }
+        }
+    }
+
     public static Pair<Integer, String> executePipedCommands(List<String[]> commands, long timeout) {
         if (timeout <= 0) {
             timeout = DEFAULT_TIMEOUT;
         }
+        final AtomicReference<List<Process>> processesRef = new AtomicReference<>();
         Callable<Pair<Integer, String>> commandRunner = () -> {
             List<ProcessBuilder> builders = commands.stream().map(ProcessBuilder::new).collect(Collectors.toList());
             List<Process> processes = ProcessBuilder.startPipeline(builders);
+            processesRef.set(processes);
             Process last = processes.get(processes.size()-1);
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(last.getInputStream()))) {
                 String line;
@@ -762,6 +782,8 @@ public class Script implements Callable<String> {
             result.second(ERR_TIMEOUT);
         } catch (InterruptedException | ExecutionException e) {
             LOGGER.error("Error executing piped commands", e);
+        } finally {
+            cleanupProcesses(processesRef);
         }
         return result;
     }