CLOUDSTACK-10054:Volume download times out in 3600 seconds (#2244)

core/src/main/java/com/cloud/storage/template/IsoProcessor.java

@@ -37,6 +37,11 @@ public class IsoProcessor extends AdapterBase implements Processor {
 
     @Override
     public FormatInfo process(String templatePath, ImageFormat format, String templateName) {
+      return process(templatePath, format, templateName, 0);
+    }
+
+   @Override
+    public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) {
         if (format != null) {
             s_logger.debug("We don't handle conversion from " + format + " to ISO.");
             return null;

core/src/main/java/com/cloud/storage/template/OVAProcessor.java

@@ -42,11 +42,15 @@ import com.cloud.utils.script.Script;
 
 public class OVAProcessor extends AdapterBase implements Processor {
     private static final Logger s_logger = Logger.getLogger(OVAProcessor.class);
-
     StorageLayer _storage;
 
     @Override
     public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
+        return process(templatePath, format, templateName, 0);
+    }
+
+    @Override
+    public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException {
         if (format != null) {
             if (s_logger.isInfoEnabled()) {
                 s_logger.info("We currently don't handle conversion from " + format + " to OVA.");
@@ -66,8 +70,7 @@ public class OVAProcessor extends AdapterBase implements Processor {
         s_logger.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName);
         String templateFileFullPath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension();
         File templateFile = new File(templateFileFullPath);
-
+        Script command = new Script("tar", processTimeout, s_logger);
-        Script command = new Script("tar", 0, s_logger);
         command.add("--no-same-owner");
         command.add("--no-same-permissions");
         command.add("-xf", templateFileFullPath);

core/src/main/java/com/cloud/storage/template/Processor.java

@@ -44,6 +44,8 @@ public interface Processor extends Adapter {
      */
     FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException;
 
+    FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException;
+
     public static class FormatInfo {
         public ImageFormat format;
         public long size;

core/src/main/java/com/cloud/storage/template/QCOW2Processor.java

@@ -40,8 +40,13 @@ public class QCOW2Processor extends AdapterBase implements Processor {
 
     private StorageLayer _storage;
 
+   @Override
+   public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException  {
+     return process(templatePath, format, templateName, 0);
+   }
+
     @Override
-    public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
+    public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException {
         if (format != null) {
             s_logger.debug("We currently don't handle conversion from " + format + " to QCOW2.");
             return null;

core/src/main/java/com/cloud/storage/template/RawImageProcessor.java

@@ -45,8 +45,13 @@ public class RawImageProcessor extends AdapterBase implements Processor {
         return true;
     }
 
+   @Override
+   public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
+      return process(templatePath, format, templateName, 0);
+   }
+
     @Override
-    public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
+    public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException {
         if (format != null) {
             s_logger.debug("We currently don't handle conversion from " + format + " to raw image.");
             return null;

core/src/main/java/com/cloud/storage/template/TARProcessor.java

@@ -35,6 +35,11 @@ public class TARProcessor extends AdapterBase implements Processor {
 
     @Override
     public FormatInfo process(String templatePath, ImageFormat format, String templateName) {
+       return process(templatePath, format, templateName, 0);
+    }
+
+    @Override
+    public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) {
         if (format != null) {
             s_logger.debug("We currently don't handle conversion from " + format + " to TAR.");
             return null;

core/src/main/java/com/cloud/storage/template/VhdProcessor.java

@@ -58,6 +58,11 @@ public class VhdProcessor extends AdapterBase implements Processor {
 
     @Override
     public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
+     return process(templatePath, format, templateName, 0);
+    }
+
+    @Override
+    public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException {
         if (format != null) {
             s_logger.debug("We currently don't handle conversion from " + format + " to VHD.");
             return null;

core/src/main/java/com/cloud/storage/template/VmdkProcessor.java

@@ -44,6 +44,11 @@ public class VmdkProcessor extends AdapterBase implements Processor {
 
     @Override
     public FormatInfo process(String templatePath, ImageFormat format, String templateName) throws InternalErrorException {
+     return process(templatePath, format, templateName, 0);
+    }
+
+    @Override
+    public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException {
         if (format != null) {
             if (s_logger.isInfoEnabled()) {
                 s_logger.info("We currently don't handle conversion from " + format + " to VMDK.");

core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java

@@ -51,6 +51,8 @@ public class TemplateOrVolumePostUploadCommand {
 
     private String defaultMaxAccountSecondaryStorage;
 
+    private long processTimeout;
+
     private long accountId;
 
     private Integer nfsVersion;
@@ -206,4 +208,12 @@ public class TemplateOrVolumePostUploadCommand {
     public void setNfsVersion(Integer nfsVersion) {
         this.nfsVersion = nfsVersion;
     }
+
+    public void setProcessTimeout(long processTimeout) {
+        this.processTimeout = processTimeout;
+    }
+
+    public long getProcessTimeout() {
+        return processTimeout;
+    }
 }

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java

@@ -32,6 +32,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.command.DownloadCommand;
 import org.apache.cloudstack.storage.command.DeleteCommand;
 import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
@@ -434,13 +435,14 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
     @DB
     public Pair<Boolean, Long> getCommandHostDelegation(long hostId, Command cmd) {
         boolean needDelegation = false;
-
         if (cmd instanceof StorageSubSystemCommand) {
             Boolean fullCloneEnabled = VmwareFullClone.value();
             StorageSubSystemCommand c = (StorageSubSystemCommand)cmd;
             c.setExecuteInSequence(fullCloneEnabled);
         }
-
+        if (cmd instanceof DownloadCommand) {
+          cmd.setContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key(), String.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.value()));
+        }
         //NOTE: the hostid can be a hypervisor host, or a ssvm agent. For copycommand, if it's for volume upload, the hypervisor
         //type is empty, so we need to check the format of volume at first.
         if (cmd instanceof CopyCommand) {
@@ -514,11 +516,11 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co
             cmd.setContextParam("execid", String.valueOf(execLog.getId()));
             cmd.setContextParam("noderuninfo", String.format("%d-%d", _clusterMgr.getManagementNodeId(), _clusterMgr.getCurrentRunId()));
             cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout()));
+            cmd.setContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key(), String.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.value()));
 
             if (cmd instanceof BackupSnapshotCommand || cmd instanceof CreatePrivateTemplateFromVolumeCommand ||
                 cmd instanceof CreatePrivateTemplateFromSnapshotCommand || cmd instanceof CopyVolumeCommand || cmd instanceof CopyCommand ||
                 cmd instanceof CreateVolumeOVACommand || cmd instanceof PrepareOVAPackingCommand || cmd instanceof CreateVolumeFromSnapshotCommand) {
-
                 String workerName = _vmwareMgr.composeWorkerName();
                 long checkPointId = 1;
                 // FIXME: Fix                    long checkPointId = _checkPointMgr.pushCheckPoint(new VmwareCleanupMaid(hostDetails.get("guid"), workerName));

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java

@@ -42,6 +42,9 @@ public interface VmwareManager {
     static final ConfigKey<String> s_vmwareSearchExcludeFolder = new ConfigKey<String>("Advanced", String.class, "vmware.search.exclude.folders", null,
             "Comma seperated list of Datastore Folders to exclude from VMWare search", true, ConfigKey.Scope.Global);
 
+    static final ConfigKey<Integer> s_vmwareOVAPackageTimeout = new ConfigKey<Integer>(Integer.class, "vmware.package.ova.timeout", "Advanced", "3600",
+            "Vmware script timeout for ova packaging process", true, ConfigKey.Scope.Global, 1000);
+
     String composeWorkerName();
 
     String getSystemVMIsoFileNameOnDatastore();

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -136,6 +136,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
     private static final Logger s_logger = Logger.getLogger(VmwareManagerImpl.class);
 
     private static final long SECONDS_PER_MINUTE = 60;
+
     private int _timeout;
 
     private String _instance;
@@ -204,7 +205,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
     private int _additionalPortRangeSize;
     private int _routerExtraPublicNics = 2;
     private int _vCenterSessionTimeout = 1200000; // Timeout in milliseconds
-
     private String _rootDiskController = DiskControllerType.ide.toString();
 
     private final String _dataDiskController = DiskControllerType.osdefault.toString();
@@ -229,9 +229,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
 
     @Override
     public ConfigKey<?>[] getConfigKeys() {
-        return new ConfigKey<?>[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder};
+        return new ConfigKey<?>[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout};
     }
-
     @Override
     public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
         s_logger.info("Configure VmwareManagerImpl, manager name: " + name);

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManager.java

@@ -51,7 +51,7 @@ public interface VmwareStorageManager {
 
     boolean execute(VmwareHostService hostService, CreateEntityDownloadURLCommand cmd);
 
-    public void createOva(String path, String name);
+    public void createOva(String path, String name, int archiveTimeout);
 
-    public String createOvaForTemplate(TemplateObjectTO template);
+    public String createOvaForTemplate(TemplateObjectTO template, int archiveTimeout);
 }

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java

@@ -98,28 +98,32 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager {
     @Override
     public boolean execute(VmwareHostService hostService, CreateEntityDownloadURLCommand cmd) {
         DataTO data = cmd.getData();
+        int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()),
+                    Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier());
         if (data == null) {
             return false;
         }
 
         String newPath = null;
         if (data.getObjectType() == DataObjectType.VOLUME) {
-            newPath = createOvaForVolume((VolumeObjectTO)data);
+            newPath = createOvaForVolume((VolumeObjectTO)data, timeout);
         } else if (data.getObjectType() == DataObjectType.TEMPLATE) {
-            newPath = createOvaForTemplate((TemplateObjectTO)data);
+            newPath = createOvaForTemplate((TemplateObjectTO)data, timeout);
         }
         if (newPath != null) {
             cmd.setInstallPath(newPath);
+            return true;
         }
-        return true;
+        return false;
+
     }
 
     @Override
-    public void createOva(String path, String name) {
+    public void createOva(String path, String name, int archiveTimeout) {
         Script commandSync = new Script(true, "sync", 0, s_logger);
         commandSync.execute();
 
-        Script command = new Script(false, "tar", 0, s_logger);
+        Script command = new Script(false, "tar", archiveTimeout, s_logger);
         command.setWorkDir(path);
         command.add("-cf", name + ".ova");
         command.add(name + ".ovf");        // OVF file should be the first file in OVA archive
@@ -155,7 +159,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager {
     }
 
     @Override
-    public String createOvaForTemplate(TemplateObjectTO template) {
+    public String createOvaForTemplate(TemplateObjectTO template, int archiveTimeout) {
         DataStoreTO storeTO = template.getDataStore();
         if (!(storeTO instanceof NfsTO)) {
             s_logger.debug("Can only handle NFS storage, while creating OVA from template");
@@ -173,7 +177,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager {
                     s_logger.debug("OVA file found at: " + installFullPath);
                 } else {
                     if (new File(installFullPath + ".meta").exists()) {
-                        createOVAFromMetafile(installFullPath + ".meta");
+                        createOVAFromMetafile(installFullPath + ".meta", archiveTimeout);
                     } else {
                         String msg = "Unable to find OVA or OVA MetaFile to prepare template.";
                         s_logger.error(msg);
@@ -190,7 +194,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager {
 
     //Fang: new command added;
     // Important! we need to sync file system before we can safely use tar to work around a linux kernal bug(or feature)
-    public String createOvaForVolume(VolumeObjectTO volume) {
+    public String createOvaForVolume(VolumeObjectTO volume, int archiveTimeout) {
         DataStoreTO storeTO = volume.getDataStore();
         if (!(storeTO instanceof NfsTO)) {
             s_logger.debug("can only handle nfs storage, when create ova from volume");
@@ -215,15 +219,17 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager {
             } else {
                 Script commandSync = new Script(true, "sync", 0, s_logger);
                 commandSync.execute();
-
+                Script command = new Script(false, "tar", archiveTimeout, s_logger);
-                Script command = new Script(false, "tar", 0, s_logger);
                 command.setWorkDir(installFullPath);
                 command.add("-cf", volumeUuid + ".ova");
                 command.add(volumeUuid + ".ovf");        // OVF file should be the first file in OVA archive
                 command.add(volumeUuid + "-disk0.vmdk");
 
-                command.execute();
+                String result = command.execute();
-                return volumePath;
+                if (result != Script.ERR_TIMEOUT) {
+                    return volumePath;
+                }
+
             }
         } catch (Throwable e) {
             s_logger.info("Exception for createVolumeOVA");
@@ -1046,7 +1052,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager {
 
     // here we use a method to return the ovf and vmdk file names; Another way to do it:
     // create a new class, and like TemplateLocation.java and create templateOvfInfo.java to handle it;
-    private String createOVAFromMetafile(String metafileName) throws Exception {
+    private String createOVAFromMetafile(String metafileName, int archiveTimeout) throws Exception {
         File ova_metafile = new File(metafileName);
         Properties props = null;
         String ovaFileName = "";
@@ -1080,7 +1086,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager {
             s_logger.info("ova: " + ovaFileName + ", ovf:" + ovfFileName + ", vmdk:" + disks[0] + ".");
             Script commandSync = new Script(true, "sync", 0, s_logger);
             commandSync.execute();
-            Script command = new Script(false, "tar", 0, s_logger);
+            Script command = new Script(false, "tar", archiveTimeout, s_logger);
             command.setWorkDir(exportDir); // Fang: pass this in to the method?
             command.add("-cf", ovaFileName);
             command.add(ovfFileName); // OVF file should be the first file in OVA archive

plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java

@@ -145,8 +145,11 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe
     }
 
     protected Answer execute(CreateEntityDownloadURLCommand cmd) {
-        _storageMgr.execute(this, cmd);
+        boolean success = _storageMgr.execute(this, cmd);
-        return _resource.defaultAction(cmd);
+        if (success) {
+            return _resource.defaultAction(cmd);
+        }
+        return new Answer(cmd, false, "Failed to download");
     }
 
     private Answer execute(PrimaryStorageDownloadCommand cmd) {

plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageSubsystemCommandHandler.java

@@ -21,6 +21,8 @@ package com.cloud.storage.resource;
 import java.io.File;
 import java.util.EnumMap;
 
+import com.cloud.hypervisor.vmware.manager.VmwareManager;
+import com.cloud.utils.NumbersUtil;
 import org.apache.log4j.Logger;
 import org.apache.cloudstack.storage.command.CopyCmdAnswer;
 import org.apache.cloudstack.storage.command.CopyCommand;
@@ -95,6 +97,8 @@ public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemComman
         DataTO destData = cmd.getDestTO();
         DataStoreTO srcDataStore = srcData.getDataStore();
         DataStoreTO destDataStore = destData.getDataStore();
+        int timeout = NumbersUtil.parseInt(cmd.getContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key()),
+                        Integer.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.defaultValue()) * VmwareManager.s_vmwareOVAPackageTimeout.multiplier());
         //if copied between s3 and nfs cache, go to resource
         boolean needDelegation = false;
         if (destDataStore instanceof NfsTO && destDataStore.getRole() == DataStoreRole.ImageCache) {
@@ -112,11 +116,11 @@ public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemComman
                 String path = vol.getPath();
                 int index = path.lastIndexOf(File.separator);
                 String name = path.substring(index + 1);
-                storageManager.createOva(parentPath + File.separator + path, name);
+                storageManager.createOva(parentPath + File.separator + path, name, timeout);
                 vol.setPath(path + File.separator + name + ".ova");
             } else if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
                 // sync template from NFS cache to S3 in NFS migration to S3 case
-                storageManager.createOvaForTemplate((TemplateObjectTO)srcData);
+                storageManager.createOvaForTemplate((TemplateObjectTO)srcData, timeout);
             } else if (srcData.getObjectType() == DataObjectType.SNAPSHOT) {
                 // pack ova first
                 // sync snapshot from NFS cache to S3 in NFS migration to S3 case
@@ -126,7 +130,7 @@ public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemComman
                 int index = path.lastIndexOf(File.separator);
                 String name = path.substring(index + 1);
                 String snapDir = path.substring(0, index);
-                storageManager.createOva(parentPath + File.separator + snapDir, name);
+                storageManager.createOva(parentPath + File.separator + snapDir, name, timeout);
                 if (destData.getObjectType() == DataObjectType.TEMPLATE) {
                     //create template from snapshot on src at first, then copy it to s3
                     TemplateObjectTO cacheTemplate = (TemplateObjectTO)destData;
@@ -169,7 +173,7 @@ public class VmwareStorageSubsystemCommandHandler extends StorageSubsystemComman
                 int index = path.lastIndexOf(File.separator);
                 String name = path.substring(index + 1);
                 String dir = path.substring(0, index);
-                storageManager.createOva(parentPath + File.separator + dir, name);
+                storageManager.createOva(parentPath + File.separator + dir, name, timeout);
                 newSnapshot.setPath(newSnapshot.getPath() + ".ova");
                 newSnapshot.setDataStore(cmd.getCacheTO().getDataStore());
                 CopyCommand newCmd = new CopyCommand(newSnapshot, destData, cmd.getWait(), cmd.executeInSequence());

server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java

@@ -367,6 +367,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
                                                           dataObject.getDataStore().getRole().toString());
                 command.setLocalPath(volumeStore.getLocalDownloadPath());
                 //using the existing max upload size configuration
+                command.setProcessTimeout(NumbersUtil.parseLong(_configDao.getValue("vmware.package.ova.timeout"), 3600));
                 command.setMaxUploadSize(_configDao.getValue(Config.MaxUploadVolumeSize.key()));
                 command.setDefaultMaxAccountSecondaryStorage(_configDao.getValue(Config.DefaultMaxAccountSecondaryStorage.key()));
                 command.setAccountId(vol.getAccountId());

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java

@@ -87,6 +87,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
 
     private static final String HEADER_HOST = "X-Forwarded-Host";
 
+    private static long processTimeout;
+
     public HttpUploadServerHandler(NfsSecondaryStorageResource storageResource) {
         this.storageResource = storageResource;
     }
@@ -152,7 +154,6 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
                 Map<String, List<String>> uriAttributes = decoderQuery.parameters();
                 uuid = uriAttributes.get("uuid").get(0);
                 logger.info("URI: uuid=" + uuid);
-
                 UploadEntity uploadEntity = null;
                 try {
                     // Validate the request here
@@ -175,6 +176,7 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
                 }
                 //set the base directory to download the file
                 DiskFileUpload.baseDirectory = uploadEntity.getInstallPathPrefix();
+                this.processTimeout = uploadEntity.getProcessTimeout();
                 logger.info("base directory: " + DiskFileUpload.baseDirectory);
                 try {
                     //initialize the decoder
@@ -243,7 +245,7 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
                                 storageResource.updateStateMapWithError(uuid, errorString);
                                 return HttpResponseStatus.BAD_REQUEST;
                             }
-                            String status = storageResource.postUpload(uuid, fileUpload.getFile().getName());
+                            String status = storageResource.postUpload(uuid, fileUpload.getFile().getName(), processTimeout);
                             if (status != null) {
                                 responseContent.append(status);
                                 storageResource.updateStateMapWithError(uuid, status);

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java

@@ -251,6 +251,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
     protected String createTemplateFromSnapshotXenScript;
     private HashMap<String, UploadEntity> uploadEntityStateMap = new HashMap<String, UploadEntity>();
     private String _ssvmPSK = null;
+    private long processTimeout;
 
     public void setParentPath(String path) {
         _parent = path;
@@ -3345,6 +3346,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             throw new InvalidParameterValueException(errorMessage);
         } else {
             uuid = cmd.getEntityUUID();
+            processTimeout = cmd.getProcessTimeout();
             if (isOneTimePostUrlUsed(cmd)) {
                 uploadEntity = uploadEntityStateMap.get(uuid);
                 StringBuilder errorMessage = new StringBuilder("The one time post url is already used");
@@ -3366,6 +3368,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
                 uploadEntity = new UploadEntity(uuid, cmd.getEntityId(), UploadEntity.Status.IN_PROGRESS, cmd.getName(), absolutePath);
                 uploadEntity.setMetaDataPopulated(true);
                 uploadEntity.setResourceType(UploadEntity.ResourceType.valueOf(cmd.getType()));
+                uploadEntity.setProcessTimeout(processTimeout);
                 uploadEntity.setFormat(Storage.ImageFormat.valueOf(cmd.getImageFormat()));
                 //relative path with out ssvm mount info.
                 uploadEntity.setTemplatePath(absolutePath);
@@ -3447,7 +3450,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         return (int)Math.ceil(sizeInBytes * 1.0d / (1024 * 1024 * 1024));
     }
 
-    public String postUpload(String uuid, String filename) {
+    public String postUpload(String uuid, String filename, long processTimeout) {
         UploadEntity uploadEntity = uploadEntityStateMap.get(uuid);
         int installTimeoutPerGig = 180 * 60 * 1000;
 
@@ -3552,7 +3555,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
         for (Processor processor : processors.values()) {
             FormatInfo info = null;
             try {
-                info = processor.process(resourcePath, null, templateName);
+                 info = processor.process(resourcePath, null, templateName, processTimeout * 1000);
             } catch (InternalErrorException e) {
                 s_logger.error("Template process exception ", e);
                 return e.toString();

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java

@@ -90,7 +90,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
     private String _name;
     StorageLayer _storage;
     public Map<String, Processor> _processors;
-
+    private long _processTimeout;
     private Integer _nfsVersion;
 
     public class Completion implements DownloadCompleteCallback {
@@ -459,7 +459,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
 
             FormatInfo info = null;
             try {
-                info = processor.process(resourcePath, null, templateName);
+                info = processor.process(resourcePath, null, templateName, this._processTimeout);
             } catch (InternalErrorException e) {
                 s_logger.error("Template process exception ", e);
                 return e.toString();
@@ -677,6 +677,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
 
     @Override
     public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) {
+        int timeout = NumbersUtil.parseInt(cmd.getContextParam("vmware.package.ova.timeout"), 3600000);
+        this._processTimeout = timeout;
         ResourceType resourceType = cmd.getResourceType();
         if (cmd instanceof DownloadProgressCommand) {
             return handleDownloadProgressCmd(resource, (DownloadProgressCommand)cmd);

services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadEntity.java

@@ -34,6 +34,7 @@ public class UploadEntity {
     private int maxSizeInGB;
     private String description;
     private long contentLength;
+    private long processTimeout;
 
     public static enum ResourceType {
         VOLUME, TEMPLATE
@@ -60,6 +61,14 @@ public class UploadEntity {
         this.entityId=entityId;
     }
 
+    public void setProcessTimeout(long processTimeout) {
+        this.processTimeout = processTimeout;
+    }
+
+    public long getProcessTimeout() {
+        return processTimeout;
+    }
+
     public UploadEntity(){
 
     }

utils/src/main/java/com/cloud/utils/script/Script.java

@@ -37,7 +37,6 @@ import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.cloudstack.utils.security.KeyStoreUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.log4j.Logger;
 import org.joda.time.Duration;
@@ -203,7 +202,7 @@ public class Script implements Callable<String> {
         String[] command = _command.toArray(new String[_command.size()]);
 
         if (_logger.isDebugEnabled()) {
-            _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.defaultKeystoreFile)[0]);
+            _logger.debug("Executing: " + buildCommandLine(command));
         }
 
         try {
@@ -233,18 +232,23 @@ public class Script implements Callable<String> {
             }
 
             while (true) {
+                _logger.debug("Executing while with timeout : " + _timeout);
                 try {
-                    if (_process.waitFor() == 0) {
+                    //process execution completed within timeout period
-                        _logger.debug("Execution is successful.");
+                    if (_process.waitFor(_timeout, TimeUnit.MILLISECONDS)) {
-                        if (interpreter != null) {
+                        //process completed successfully
-                            return interpreter.drain() ? task.getResult() : interpreter.interpret(ir);
+                        if (_process.exitValue() == 0) {
-                        } else {
+                            _logger.debug("Execution is successful.");
-                            // null return exitValue apparently
+                            if (interpreter != null) {
-                            return String.valueOf(_process.exitValue());
+                                return interpreter.drain() ? task.getResult() : interpreter.interpret(ir);
+                            } else {
+                                // null return exitValue apparently
+                                return String.valueOf(_process.exitValue());
+                            }
+                        } else { //process failed
+                            break;
                         }
-                    } else {
+                    } //timeout
-                        break;
-                    }
                 } catch (InterruptedException e) {
                     if (!_isTimeOut) {
                         /*
@@ -254,24 +258,25 @@ public class Script implements Callable<String> {
                         _logger.debug("We are interrupted but it's not a timeout, just continue");
                         continue;
                     }
-
-                    TimedOutLogger log = new TimedOutLogger(_process);
-                    Task timedoutTask = new Task(log, ir);
-
-                    timedoutTask.run();
-                    if (!_passwordCommand) {
-                        _logger.warn("Timed out: " + buildCommandLine(command) + ".  Output is: " + timedoutTask.getResult());
-                    } else {
-                        _logger.warn("Timed out: " + buildCommandLine(command));
-                    }
-
-                    return ERR_TIMEOUT;
                 } finally {
                     if (future != null) {
                         future.cancel(false);
                     }
                     Thread.interrupted();
                 }
+
+                //timeout without completing the process
+                TimedOutLogger log = new TimedOutLogger(_process);
+                Task timedoutTask = new Task(log, ir);
+
+                timedoutTask.run();
+                if (!_passwordCommand) {
+                    _logger.warn("Timed out: " + buildCommandLine(command) + ".  Output is: " + timedoutTask.getResult());
+                } else {
+                    _logger.warn("Timed out: " + buildCommandLine(command));
+                }
+
+                return ERR_TIMEOUT;
             }
 
             _logger.debug("Exit value is " + _process.exitValue());
@@ -300,7 +305,7 @@ public class Script implements Callable<String> {
                 IOUtils.closeQuietly(_process.getErrorStream());
                 IOUtils.closeQuietly(_process.getOutputStream());
                 IOUtils.closeQuietly(_process.getInputStream());
-                _process.destroy();
+                _process.destroyForcibly();
             }
         }
     }