Merge c6ee04f146 into bce3e54a7e

Co-authored-by: Daman Arora <daman.arora@shapeblue.com>

api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java

@@ -78,6 +78,7 @@ public class UpdateNetworkOfferingCmd extends BaseCmd {
 
     @Parameter(name = ApiConstants.DOMAIN_ID,
             type = CommandType.STRING,
+            length = 4096,
             description = "The ID of the containing domain(s) as comma separated string, public for public offerings")
     private String domainIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java

@@ -75,6 +75,7 @@ public class UpdateDiskOfferingCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java

@@ -69,6 +69,7 @@ public class UpdateServiceOfferingCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java

@@ -65,6 +65,7 @@ public class UpdateVPCOfferingCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDao.java

@@ -31,4 +31,6 @@ public interface VMScheduledJobDao extends GenericDao<VMScheduledJobVO, Long> {
     int expungeJobsForSchedules(List<Long> scheduleId, Date dateAfter);
 
     int expungeJobsBefore(Date currentTimestamp);
+
+    VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp);
 }

engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java

@@ -39,6 +39,8 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
 
     private final SearchBuilder<VMScheduledJobVO> expungeJobForScheduleSearch;
 
+    private final SearchBuilder<VMScheduledJobVO> scheduleAndTimestampSearch;
+
     static final String SCHEDULED_TIMESTAMP = "scheduled_timestamp";
 
     static final String VM_SCHEDULE_ID = "vm_schedule_id";
@@ -58,6 +60,11 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
         expungeJobForScheduleSearch.and(VM_SCHEDULE_ID, expungeJobForScheduleSearch.entity().getVmScheduleId(), SearchCriteria.Op.IN);
         expungeJobForScheduleSearch.and(SCHEDULED_TIMESTAMP, expungeJobForScheduleSearch.entity().getScheduledTime(), SearchCriteria.Op.GTEQ);
         expungeJobForScheduleSearch.done();
+
+        scheduleAndTimestampSearch = createSearchBuilder();
+        scheduleAndTimestampSearch.and(VM_SCHEDULE_ID, scheduleAndTimestampSearch.entity().getVmScheduleId(), SearchCriteria.Op.EQ);
+        scheduleAndTimestampSearch.and(SCHEDULED_TIMESTAMP, scheduleAndTimestampSearch.entity().getScheduledTime(), SearchCriteria.Op.EQ);
+        scheduleAndTimestampSearch.done();
     }
 
     /**
@@ -92,4 +99,12 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
         sc.setParameters(SCHEDULED_TIMESTAMP, date);
         return expunge(sc);
     }
+
+    @Override
+    public VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp) {
+        SearchCriteria<VMScheduledJobVO> sc = scheduleAndTimestampSearch.create();
+        sc.setParameters(VM_SCHEDULE_ID, scheduleId);
+        sc.setParameters(SCHEDULED_TIMESTAMP, scheduledTimestamp);
+        return findOneBy(sc);
+    }
 }

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateHostPasswordCommandWrapper.java

@@ -37,7 +37,8 @@ public final class LibvirtUpdateHostPasswordCommandWrapper extends CommandWrappe
         final String newPassword = command.getNewPassword();
 
         final Script script = libvirtUtilitiesHelper.buildScript(libvirtComputingResource.getUpdateHostPasswdPath());
-        script.add(username, newPassword);
+        script.add(username);
+        script.addSensitive(newPassword);
         final String result = script.execute();
 
         if (result != null) {

plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java

@@ -45,9 +45,10 @@ public final class CitrixUpdateHostPasswordCommandWrapper extends CommandWrapper
 
         Pair<Boolean, String> result;
         try {
-            logger.debug("Executing command in Host: " + cmdLine);
+            logger.debug("Executing command in Host: " + xenServerUtilitiesHelper.buildCommandLine(SCRIPT_CMD_PATH,
+                    VRScripts.UPDATE_HOST_PASSWD, username, "******"));
             final String hostPassword = citrixResourceBase.getPwdFromQueue();
-            result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine.toString());
+            result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine);
         } catch (final Exception e) {
             return new Answer(command, false, e.getMessage());
         }

server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java

@@ -162,7 +162,13 @@ public class VMSchedulerImpl extends ManagerBase implements VMScheduler, Configu
         }
 
         Date scheduledDateTime = Date.from(ts.toInstant());
-        VMScheduledJobVO scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime);
+        VMScheduledJobVO scheduledJob = vmScheduledJobDao.findByScheduleAndTimestamp(vmSchedule.getId(), scheduledDateTime);
+        if (scheduledJob != null) {
+            logger.trace("Job is already scheduled for schedule {} at {}", vmSchedule, scheduledDateTime);
+            return scheduledDateTime;
+        }
+
+        scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime);
         try {
             vmScheduledJobDao.persist(scheduledJob);
             ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, vm.getAccountId(), actionEventMap.get(vmSchedule.getAction()),

ui/src/utils/plugins.js

@@ -218,18 +218,19 @@ export const notifierPlugin = {
         if (error.response.status) {
           msg = `${i18n.global.t('message.request.failed')} (${error.response.status})`
         }
-        if (error.message) {
-          desc = error.message
-        }
-        if (error.response.headers && 'x-description' in error.response.headers) {
+        if (error.response.headers?.['x-description']) {
           desc = error.response.headers['x-description']
-        }
-        if (desc === '' && error.response.data) {
+        } else if (error.response.data) {
           const responseKey = _.findKey(error.response.data, 'errortext')
           if (responseKey) {
             desc = error.response.data[responseKey].errortext
+          } else if (typeof error.response.data === 'string') {
+            desc = error.response.data
           }
         }
+        if (!desc && error.message) {
+          desc = error.message
+        }
       }
       let countNotify = store.getters.countNotify
       countNotify++

ui/src/views/image/RegisterOrUploadTemplate.vue

@@ -638,11 +638,7 @@ export default {
         this.$emit('refresh-data')
         this.closeAction()
       }).catch(e => {
-        this.$notification.error({
-          message: this.$t('message.upload.failed'),
-          description: `${this.$t('message.upload.template.failed.description')} -  ${e}`,
-          duration: 0
-        })
+        this.$notifyError(e)
       })
     },
     fetchCustomHypervisorName () {

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -30,8 +30,10 @@ import java.net.URISyntaxException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
@@ -40,9 +42,10 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import org.apache.cloudstack.utils.security.KeyStoreUtils;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -64,8 +67,8 @@ public class Script implements Callable<String> {
     private static final int DEFAULT_TIMEOUT = 3600 * 1000; /* 1 hour */
     private volatile boolean _isTimeOut = false;
 
-    private boolean _passwordCommand = false;
     private boolean avoidLoggingCommand = false;
+    private final Set<Integer> sensitiveArgIndices = new HashSet<>();
 
     private static final ScheduledExecutorService s_executors = Executors.newScheduledThreadPool(10, new NamedThreadFactory("Script"));
 
@@ -143,6 +146,11 @@ public class Script implements Callable<String> {
         _command.add(param);
     }
 
+    public void addSensitive(String param) {
+        _command.add(param);
+        sensitiveArgIndices.add(_command.size() - 1);
+    }
+
     public Script set(String name, String value) {
         _command.add(name);
         _command.add(value);
@@ -161,7 +169,7 @@ public class Script implements Callable<String> {
             if (sanitizeViCmdParameter(cmd, builder) || sanitizeRbdFileFormatCmdParameter(cmd, builder)) {
                 continue;
             }
-            if (obscureParam) {
+            if (obscureParam || sensitiveArgIndices.contains(i)) {
                 builder.append("******").append(" ");
                 obscureParam = false;
             } else {
@@ -170,7 +178,6 @@ public class Script implements Callable<String> {
 
             if ("-y".equals(cmd) || "-z".equals(cmd)) {
                 obscureParam = true;
-                _passwordCommand = true;
             }
         }
         return builder.toString();
@@ -238,8 +245,8 @@ public class Script implements Callable<String> {
     public String execute(OutputInterpreter interpreter) {
         String[] command = _command.toArray(new String[_command.size()]);
         String commandLine = buildCommandLine(command);
-        if (_logger.isDebugEnabled() && !avoidLoggingCommand) {
-            _logger.debug(String.format("Executing command [%s].", commandLine.split(KeyStoreUtils.KS_FILENAME)[0]));
+        if (_logger.isDebugEnabled() ) {
+            _logger.debug(String.format("Executing command [%s].", commandLine));
         }
 
         try {
@@ -261,48 +268,62 @@ public class Script implements Callable<String> {
             _thread = Thread.currentThread();
             ScheduledFuture<String> future = null;
             if (_timeout > 0) {
-                _logger.trace(String.format("Scheduling the execution of command [%s] with a timeout of [%s] milliseconds.", commandLine, _timeout));
+                _logger.trace(String.format(
+                        "Scheduling the execution of command [%s] with a timeout of [%s] milliseconds.",
+                        commandLine, _timeout));
                 future = s_executors.schedule(this, _timeout, TimeUnit.MILLISECONDS);
             }
 
             long processPid = _process.pid();
             Task task = null;
             if (interpreter != null && interpreter.drain()) {
-                _logger.trace(String.format("Executing interpreting task of process [%s] for command [%s].", processPid, commandLine));
+                _logger.trace(String.format("Executing interpreting task of process [%s] for command [%s].",
+                        processPid, commandLine));
                 task = new Task(interpreter, ir);
                 s_executors.execute(task);
             }
 
             while (true) {
-                _logger.trace(String.format("Attempting process [%s] execution for command [%s] with timeout [%s].", processPid, commandLine, _timeout));
+                _logger.trace(String.format("Attempting process [%s] execution for command [%s] with timeout [%s].",
+                        processPid, commandLine, _timeout));
                 try {
                     if (_process.waitFor(_timeout, TimeUnit.MILLISECONDS)) {
-                        _logger.trace(String.format("Process [%s] execution for command [%s] completed within timeout period [%s].", processPid, commandLine,
+                        _logger.trace(String.format(
+                                "Process [%s] execution for command [%s] completed within timeout period [%s].",
+                                processPid, commandLine,
                                 _timeout));
                         if (_process.exitValue() == 0) {
-                            _logger.debug(String.format("Successfully executed process [%s] for command [%s].", processPid, commandLine));
+                            _logger.debug(String.format("Successfully executed process [%s] for command [%s].",
+                                    processPid, commandLine));
                             if (interpreter != null) {
                                 if (interpreter.drain()) {
-                                    _logger.trace(String.format("Returning task result of process [%s] for command [%s].", processPid, commandLine));
+                                    _logger.trace(
+                                            String.format("Returning task result of process [%s] for command [%s].",
+                                                    processPid, commandLine));
                                     return task.getResult();
                                 }
-                                _logger.trace(String.format("Returning interpretation of process [%s] for command [%s].", processPid, commandLine));
+                                _logger.trace(
+                                        String.format("Returning interpretation of process [%s] for command [%s].",
+                                                processPid, commandLine));
                                 return interpreter.interpret(ir);
                             } else {
                                 // null return exitValue apparently
-                                _logger.trace(String.format("Process [%s] for command [%s] exited with value [%s].", processPid, commandLine,
+                                _logger.trace(String.format("Process [%s] for command [%s] exited with value [%s].",
+                                        processPid, commandLine,
                                         _process.exitValue()));
                                 return String.valueOf(_process.exitValue());
                             }
                         } else {
-                            _logger.warn(String.format("Execution of process [%s] for command [%s] failed.", processPid, commandLine));
+                            _logger.warn(String.format("Execution of process [%s] for command [%s] failed.",
+                                    processPid, commandLine));
                             break;
                         }
                     }
                 } catch (InterruptedException e) {
                     if (!_isTimeOut) {
-                        _logger.debug(String.format("Exception [%s] occurred; however, it was not a timeout. Therefore, proceeding with the execution of process [%s] for command "
-                                + "[%s].", e.getMessage(), processPid, commandLine), e);
+                        _logger.debug(String.format(
+                                "Exception [%s] occurred; however, it was not a timeout. Therefore, proceeding with the execution of process [%s] for command [%s].",
+                                e.getMessage(), processPid, commandLine), e);
                         continue;
                     }
                 } finally {
@@ -315,18 +336,17 @@ public class Script implements Callable<String> {
                 TimedOutLogger log = new TimedOutLogger(_process);
                 Task timedoutTask = new Task(log, ir);
 
-                _logger.trace(String.format("Running timed out task of process [%s] for command [%s].", processPid, commandLine));
+                _logger.trace(String.format("Running timed out task of process [%s] for command [%s].", processPid,
+                        commandLine));
                 timedoutTask.run();
-                if (!_passwordCommand) {
-                    _logger.warn(String.format("Process [%s] for command [%s] timed out. Output is [%s].", processPid, commandLine, timedoutTask.getResult()));
-                } else {
-                    _logger.warn(String.format("Process [%s] for command [%s] timed out.", processPid, commandLine));
-                }
+                _logger.warn(String.format("Process [%s] for command [%s] timed out. Output is [%s].", processPid,
+                            commandLine, timedoutTask.getResult()));
 
                 return ERR_TIMEOUT;
             }
 
-            _logger.debug(String.format("Exit value of process [%s] for command [%s] is [%s].", processPid, commandLine, _process.exitValue()));
+            _logger.debug(String.format("Exit value of process [%s] for command [%s] is [%s].", processPid,
+                    commandLine, _process.exitValue()));
 
             BufferedReader reader = new BufferedReader(new InputStreamReader(_process.getInputStream()), 128);
 
@@ -337,19 +357,24 @@ public class Script implements Callable<String> {
                 error = String.valueOf(_process.exitValue());
             }
 
-            _logger.warn(String.format("Process [%s] for command [%s] encountered the error: [%s].", processPid, commandLine, error));
+            _logger.warn(String.format("Process [%s] for command [%s] encountered the error: [%s].", processPid,
+                    commandLine, error));
 
             return error;
         } catch (SecurityException ex) {
-            _logger.warn(String.format("Exception [%s] occurred. This may be due to an attempt of executing command [%s] as non root.", ex.getMessage(), commandLine),
+            _logger.warn(String.format(
+                    "Exception [%s] occurred. This may be due to an attempt of executing command [%s] as non root.",
+                    ex.getMessage(), commandLine),
                     ex);
             return stackTraceAsString(ex);
         } catch (Exception ex) {
-            _logger.warn(String.format("Exception [%s] occurred when attempting to run command [%s].", ex.getMessage(), commandLine), ex);
+            _logger.warn(String.format("Exception [%s] occurred when attempting to run command [%s].",
+                    ex.getMessage(), commandLine), ex);
             return stackTraceAsString(ex);
         } finally {
             if (_process != null) {
-                _logger.trace(String.format("Destroying process [%s] for command [%s].", _process.pid(), commandLine));
+                _logger.trace(
+                        String.format("Destroying process [%s] for command [%s].", _process.pid(), commandLine));
                 IOUtils.closeQuietly(_process.getErrorStream());
                 IOUtils.closeQuietly(_process.getOutputStream());
                 IOUtils.closeQuietly(_process.getInputStream());
@@ -360,9 +385,10 @@ public class Script implements Callable<String> {
 
     public String executeIgnoreExitValue(OutputInterpreter interpreter, int exitValue) {
         String[] command = _command.toArray(new String[_command.size()]);
+        String commandLine = buildCommandLine(command);
 
         if (_logger.isDebugEnabled()) {
-            _logger.debug(String.format("Executing: %s", buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]));
+            _logger.debug(String.format("Executing: %s", commandLine));
         }
 
         try {
@@ -373,7 +399,7 @@ public class Script implements Callable<String> {
 
             _process = pb.start();
             if (_process == null) {
-                _logger.warn(String.format("Unable to execute: %s", buildCommandLine(command)));
+                _logger.warn(String.format("Unable to execute: %s", commandLine));
                 return String.format("Unable to execute the command: %s", command[0]);
             }
 
@@ -437,11 +463,8 @@ public class Script implements Callable<String> {
                 Task timedoutTask = new Task(log, ir);
 
                 timedoutTask.run();
-                if (!_passwordCommand) {
-                    _logger.warn(String.format("Timed out: %s.  Output is: %s", buildCommandLine(command), timedoutTask.getResult()));
-                } else {
-                    _logger.warn(String.format("Timed out: %s", buildCommandLine(command)));
-                }
+                _logger.warn(String.format("Timed out: %s.  Output is: %s", commandLine,
+                            timedoutTask.getResult()));
 
                 return ERR_TIMEOUT;
             }
@@ -465,7 +488,7 @@ public class Script implements Callable<String> {
             _logger.warn("Security Exception....not running as root?", ex);
             return stackTraceAsString(ex);
         } catch (Exception ex) {
-            _logger.warn(String.format("Exception: %s", buildCommandLine(command)), ex);
+            _logger.warn(String.format("Exception: %s", commandLine), ex);
             return stackTraceAsString(ex);
         } finally {
             if (_process != null) {
@@ -708,13 +731,31 @@ public class Script implements Callable<String> {
         return executeCommandForExitValue(0, command);
     }
 
+    private static void cleanupProcesses(AtomicReference<List<Process>> processesRef) {
+        List<Process> processes = processesRef.get();
+        if (CollectionUtils.isNotEmpty(processes)) {
+            for (Process process : processes) {
+                if (process == null) {
+                    continue;
+                }
+                LOGGER.trace(String.format("Cleaning up process [%s] from piped commands.", process.pid()));
+                IOUtils.closeQuietly(process.getErrorStream());
+                IOUtils.closeQuietly(process.getOutputStream());
+                IOUtils.closeQuietly(process.getInputStream());
+                process.destroyForcibly();
+            }
+        }
+    }
+
     public static Pair<Integer, String> executePipedCommands(List<String[]> commands, long timeout) {
         if (timeout <= 0) {
             timeout = DEFAULT_TIMEOUT;
         }
+        final AtomicReference<List<Process>> processesRef = new AtomicReference<>();
         Callable<Pair<Integer, String>> commandRunner = () -> {
             List<ProcessBuilder> builders = commands.stream().map(ProcessBuilder::new).collect(Collectors.toList());
             List<Process> processes = ProcessBuilder.startPipeline(builders);
+            processesRef.set(processes);
             Process last = processes.get(processes.size()-1);
             try (BufferedReader reader = new BufferedReader(new InputStreamReader(last.getInputStream()))) {
                 String line;
@@ -741,6 +782,8 @@ public class Script implements Callable<String> {
             result.second(ERR_TIMEOUT);
         } catch (InterruptedException | ExecutionException e) {
             LOGGER.error("Error executing piped commands", e);
+        } finally {
+            cleanupProcesses(processesRef);
         }
         return result;
     }

utils/src/test/java/com/cloud/utils/script/ScriptTest.java

@@ -78,4 +78,34 @@ public class ScriptTest {
         String result = Script.getExecutableAbsolutePath("ls");
         Assert.assertTrue(List.of("/usr/bin/ls", "/bin/ls").contains(result));
     }
+
+    @Test
+    public void testBuildCommandLineWithSensitiveData() {
+        Script script = new Script("test.sh");
+        script.add("normal-arg");
+        script.addSensitive("sensitive-arg");
+        String commandLine = script.toString();
+        Assert.assertEquals("test.sh normal-arg ****** ", commandLine);
+    }
+
+    @Test
+    public void testBuildCommandLineWithMultipleSensitiveData() {
+        Script script = new Script("test.sh");
+        script.add("normal-arg");
+        script.addSensitive("sensitive-arg1");
+        script.add("another-normal-arg");
+        script.addSensitive("sensitive-arg2");
+        String commandLine = script.toString();
+        Assert.assertEquals("test.sh normal-arg ****** another-normal-arg ****** ", commandLine);
+    }
+
+    @Test
+    public void testBuildCommandLineWithLegacyPasswordOption() {
+        Script script = new Script("test.sh");
+        script.add("-y");
+        script.add("legacy-password");
+        String commandLine = script.toString();
+        Assert.assertEquals("test.sh -y ****** ", commandLine);
+    }
+
 }