Bug : 4286 Adding extract templated command

client/tomcatconf/commands.properties.in

@@ -61,6 +61,7 @@ deleteTemplate=com.cloud.api.commands.DeleteTemplateCmd;15
 listTemplates=com.cloud.api.commands.ListTemplatesCmd;15
 updateTemplatePermissions=com.cloud.api.commands.UpdateTemplatePermissionsCmd;15
 listTemplatePermissions=com.cloud.api.commands.ListTemplatePermissionsCmd;15
+extractTemplate=com.cloud.api.commands.ExtractTemplateCmd;15
 
 #### iso commands
 attachIso=com.cloud.api.commands.AttachIsoCmd;15

client/tomcatconf/components.xml.in

@@ -170,6 +170,8 @@
         </manager>
         <manager name="download manager" class="com.cloud.storage.download.DownloadMonitorImpl">
         </manager>
+        <manager name="upload manager" class="com.cloud.storage.upload.UploadMonitorImpl">
+        </manager>
         <manager name="console proxy manager" class="com.cloud.consoleproxy.AgentBasedStandaloneConsoleProxyManager">
         </manager>
         <manager name="secondary storage vm manager" class="com.cloud.storage.secondary.SecondaryStorageManagerImpl">

core/src/com/cloud/agent/api/storage/AbstractUploadCommand.java

@@ -0,0 +1,52 @@
+package com.cloud.agent.api.storage;
+
+import com.cloud.storage.Storage.ImageFormat;
+
+public class AbstractUploadCommand  extends StorageCommand{
+
+
+    private String url;
+    private ImageFormat format;
+    private long accountId;
+    private String name;
+    
+    protected AbstractUploadCommand() {
+    }
+    
+    protected AbstractUploadCommand(String name, String url, ImageFormat format, long accountId) {
+        this.url = url;
+        this.format = format;
+        this.accountId = accountId;
+        this.name = name;
+    }
+    
+    protected AbstractUploadCommand(AbstractUploadCommand that) {
+        this(that.name, that.url, that.format, that.accountId);
+    }
+    
+    public String getUrl() {
+        return url;
+    }
+    
+    public String getName() {
+        return name;
+    }
+    
+    public ImageFormat getFormat() {
+        return format;
+    }
+    
+    public long getAccountId() {
+        return accountId;
+    }
+    
+    @Override
+    public boolean executeInSequence() {
+        return true;
+    }
+
+	public void setUrl(String url) {
+		this.url = url;
+	}
+
+}

core/src/com/cloud/agent/api/storage/UploadAnswer.java

@@ -0,0 +1,103 @@
+package com.cloud.agent.api.storage;
+
+import java.io.File;
+
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+
+public class UploadAnswer extends Answer {
+
+	
+	private String jobId;
+	private int uploadPct;
+	private String errorString;
+	private VMTemplateHostVO.Status uploadStatus;
+	private String uploadPath;
+	private String installPath;
+	public Long templateSize = 0L;
+	
+	public int getUploadPct() {
+		return uploadPct;
+	}
+	public String getErrorString() {
+		return errorString;
+	}
+	
+	public String getUploadStatusString() {
+		return uploadStatus.toString();
+	}
+	
+	public VMTemplateHostVO.Status getUploadStatus() {
+		return uploadStatus;
+	}
+	
+	public String getUploadPath() {
+		return uploadPath;
+	}
+	protected UploadAnswer() {
+		
+	}
+	
+	public String getJobId() {
+		return jobId;
+	}
+	public void setJobId(String jobId) {
+		this.jobId = jobId;
+	}
+	
+	public UploadAnswer(String jobId, int uploadPct, String errorString,
+			Status uploadStatus, String fileSystemPath, String installPath, long templateSize) {
+		super();
+		this.jobId = jobId;
+		this.uploadPct = uploadPct;
+		this.errorString = errorString;
+		this.uploadStatus = uploadStatus;
+		this.uploadPath = fileSystemPath;
+		this.installPath = fixPath(installPath);
+		this.templateSize = templateSize;
+	}
+
+   public UploadAnswer(String jobId, int uploadPct, Command command,
+            Status uploadStatus, String fileSystemPath, String installPath) {
+	    super(command);
+        this.jobId = jobId;
+        this.uploadPct = uploadPct;
+        this.uploadStatus = uploadStatus;
+        this.uploadPath = fileSystemPath;
+        this.installPath = installPath;
+    }
+		
+	private static String fixPath(String path){
+		if (path == null)
+			return path;
+		if (path.startsWith(File.separator)) {
+			path=path.substring(File.separator.length());
+		}
+		if (path.endsWith(File.separator)) {
+			path=path.substring(0, path.length()-File.separator.length());
+		}
+		return path;
+	}
+	
+	public void setUploadStatus(VMTemplateHostVO.Status uploadStatus) {
+		this.uploadStatus = uploadStatus;
+	}
+	
+	public String getInstallPath() {
+		return installPath;
+	}
+	public void setInstallPath(String installPath) {
+		this.installPath = fixPath(installPath);
+	}
+
+	public void setTemplateSize(long templateSize) {
+		this.templateSize = templateSize;
+	}
+	
+	public Long getTemplateSize() {
+		return templateSize;
+	}
+	
+}

core/src/com/cloud/agent/api/storage/UploadCommand.java

@@ -0,0 +1,115 @@
+package com.cloud.agent.api.storage;
+
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.agent.api.storage.AbstractUploadCommand;
+import com.cloud.agent.api.storage.DownloadCommand.PasswordAuth;
+
+
+public class UploadCommand extends AbstractUploadCommand {
+
+	private VMTemplateVO template;
+	private String url;
+	private String installPath;	
+	private boolean hvm;
+	private String description;
+	private String checksum;
+	private PasswordAuth auth;
+	private long templateSizeInBytes;
+	private long id;
+
+	public UploadCommand(VMTemplateVO template, String url, VMTemplateHostVO vmTemplateHost) {
+		
+		this.template = template;
+		this.url = url;
+		this.installPath = vmTemplateHost.getInstallPath();
+		this.checksum = template.getChecksum();
+		this.id = template.getId();
+		this.templateSizeInBytes = vmTemplateHost.getSize();
+		
+	}
+	
+	protected UploadCommand() {
+	}
+	
+	public UploadCommand(UploadCommand that) {
+		this.template = that.template;
+		this.url = that.url;
+		this.installPath = that.installPath;
+		this.checksum = that.getChecksum();
+		this.id = that.id;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+
+	public VMTemplateVO getTemplate() {
+		return template;
+	}
+
+	public void setTemplate(VMTemplateVO template) {
+		this.template = template;
+	}
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(String url) {
+		this.url = url;
+	}
+
+	public boolean isHvm() {
+		return hvm;
+	}
+
+	public void setHvm(boolean hvm) {
+		this.hvm = hvm;
+	}
+
+	public PasswordAuth getAuth() {
+		return auth;
+	}
+
+	public void setAuth(PasswordAuth auth) {
+		this.auth = auth;
+	}
+
+	public Long getTemplateSizeInBytes() {
+		return templateSizeInBytes;
+	}
+
+	public void setTemplateSizeInBytes(Long templateSizeInBytes) {
+		this.templateSizeInBytes = templateSizeInBytes;
+	}
+
+	public long getId() {
+		return id;
+	}
+
+	public void setId(long id) {
+		this.id = id;
+	}
+
+	public void setInstallPath(String installPath) {
+		this.installPath = installPath;
+	}
+
+	public void setDescription(String description) {
+		this.description = description;
+	}
+
+	public void setChecksum(String checksum) {
+		this.checksum = checksum;
+	}
+
+	public String getInstallPath() {
+		return installPath;
+	}
+
+	public String getChecksum() {
+		return checksum;
+	}
+}

core/src/com/cloud/agent/api/storage/UploadProgressCommand.java

@@ -0,0 +1,32 @@
+package com.cloud.agent.api.storage;
+
+public class UploadProgressCommand extends UploadCommand {
+
+	public static enum RequestType {GET_STATUS, ABORT, RESTART, PURGE, GET_OR_RESTART}
+	private String jobId;
+	private RequestType request;
+
+	protected UploadProgressCommand() {
+		super();
+	}
+	
+	public UploadProgressCommand(UploadCommand cmd, String jobId, RequestType req) {
+	    super(cmd);
+
+		this.jobId = jobId;
+		this.setRequest(req);
+	}
+
+	public String getJobId() {
+		return jobId;
+	}
+
+	public void setRequest(RequestType request) {
+		this.request = request;
+	}
+
+	public RequestType getRequest() {
+		return request;
+	}
+	
+}

core/src/com/cloud/event/EventTypes.java

@@ -78,7 +78,10 @@ public class EventTypes {
 	public static final String EVENT_TEMPLATE_COPY = "TEMPLATE.COPY";
 	public static final String EVENT_TEMPLATE_DOWNLOAD_START = "TEMPLATE.DOWNLOAD.START";
 	public static final String EVENT_TEMPLATE_DOWNLOAD_SUCCESS = "TEMPLATE.DOWNLOAD.SUCCESS";
-	public static final String EVENT_TEMPLATE_DOWNLOAD_FAILED = "TEMPLATE.DOWNLOAD.FAILED";
+	public static final String EVENT_TEMPLATE_DOWNLOAD_FAILED = "TEMPLATE.DOWNLOAD.FAILED";
+	public static final String EVENT_TEMPLATE_UPLOAD_FAILED = "TEMPLATE.UPLOAD.FAILED";
+	public static final String EVENT_TEMPLATE_UPLOAD_START = "TEMPLATE.UPLOAD.START";
+	public static final String EVENT_TEMPLATE_UPLOAD_SUCCESS = "TEMPLATE.UPLOAD.SUCCESS";
 	
 	// Volume Events
 	public static final String EVENT_VOLUME_CREATE = "VOLUME.CREATE";

core/src/com/cloud/server/ManagementServer.java

@@ -2186,5 +2186,13 @@ public interface ManagementServer {
 	boolean validateCustomVolumeSizeRange(long size) throws InvalidParameterValueException;
 	
 	boolean checkIfMaintenable(long hostId);
+	/**
+	 * Extracts the template to a particular location.
+	 * @param url - the url  where the template needs to be extracted to
+	 * @param zoneId - zone id of the template
+	 * @param template id - the id of the template
+	 *  
+	 */
+	void extractTemplate(String url, Long templateId, Long zoneId) throws URISyntaxException;
 
 }

core/src/com/cloud/storage/StorageResource.java

@@ -53,12 +53,14 @@ import com.cloud.agent.api.storage.ShareAnswer;
 import com.cloud.agent.api.storage.ShareCommand;
 import com.cloud.agent.api.storage.UpgradeDiskAnswer;
 import com.cloud.agent.api.storage.UpgradeDiskCommand;
+import com.cloud.agent.api.storage.UploadCommand;
 import com.cloud.host.Host;
 import com.cloud.resource.ServerResource;
 import com.cloud.resource.ServerResourceBase;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.template.DownloadManager;
 import com.cloud.storage.template.TemplateInfo;
+import com.cloud.storage.template.UploadManager;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.script.OutputInterpreter;
@@ -112,6 +114,7 @@ public abstract class StorageResource extends ServerResourceBase implements Serv
 	protected String _zfsScriptsDir;
 
 	protected DownloadManager _downloadManager;
+	protected UploadManager _uploadManager;
 
 	protected Map<Long, VolumeSnapshotRequest> _volumeHourlySnapshotRequests = new HashMap<Long, VolumeSnapshotRequest>();
     protected Map<Long, VolumeSnapshotRequest> _volumeDailySnapshotRequests = new HashMap<Long, VolumeSnapshotRequest>();
@@ -127,6 +130,8 @@ public abstract class StorageResource extends ServerResourceBase implements Serv
         	return execute((PrimaryStorageDownloadCommand)cmd);
         } else if (cmd instanceof DownloadCommand) {
             return execute((DownloadCommand)cmd);
+        }else if (cmd instanceof UploadCommand) {
+                return execute((UploadCommand)cmd);
         } else if (cmd instanceof GetStorageStatsCommand) {
             return execute((GetStorageStatsCommand)cmd);
         } else if (cmd instanceof UpgradeDiskCommand) {
@@ -159,6 +164,11 @@ public abstract class StorageResource extends ServerResourceBase implements Serv
     protected Answer execute(final PrimaryStorageDownloadCommand cmd) {
     	return Answer.createUnsupportedCommandAnswer(cmd);
     }
+
+	private Answer execute(UploadCommand cmd) {	
+		s_logger.warn(" Nitin got the cmd " +cmd);
+		return _uploadManager.handleUploadCommand(cmd);
+	}
     
     protected Answer execute(final DownloadCommand cmd) {
     	return _downloadManager.handleDownloadCommand(cmd);

core/src/com/cloud/storage/VMTemplateHostVO.java

@@ -59,7 +59,10 @@ public class VMTemplateHostVO implements VMTemplateStorageResourceAssoc {
 	private Date lastUpdated = null;
 	
 	@Column (name="download_pct")
-	private int downloadPercent;
+	private int downloadPercent;
+	
+	@Column (name="upload_pct")
+	private int uploadPercent;
 	
 	@Column (name="size")
 	private long size;
@@ -67,15 +70,25 @@ public class VMTemplateHostVO implements VMTemplateStorageResourceAssoc {
 	@Column (name="download_state")
 	@Enumerated(EnumType.STRING)
 	private Status downloadState;
+	
+	@Column (name="upload_state")
+	@Enumerated(EnumType.STRING)
+	private Status uploadState;
 	
 	@Column (name="local_path")
 	private String localDownloadPath;
 	
 	@Column (name="error_str")
 	private String errorString;
+	
+	@Column (name="upload_error_str")
+	private String upload_errorString;
 	
 	@Column (name="job_id")
 	private String jobId;
+	
+	@Column (name="upload_job_id")
+	private String uploadJobId;
 	
 	@Column (name="pool_id")
 	private Long poolId;
@@ -85,7 +98,10 @@ public class VMTemplateHostVO implements VMTemplateStorageResourceAssoc {
 	
 	@Column (name="url")
 	private String downloadUrl;
-	
+
+	@Column (name="upload_url")
+	private String uploadUrl;
+		
 	@Column(name="is_copy")
 	private boolean isCopy = false;
     
@@ -262,5 +278,45 @@ public class VMTemplateHostVO implements VMTemplateStorageResourceAssoc {
 
 	public boolean isCopy() {
 		return isCopy;
+	}
+
+	public int getUploadPercent() {
+		return uploadPercent;
+	}
+
+	public void setUploadPercent(int uploadPercent) {
+		this.uploadPercent = uploadPercent;
+	}
+
+	public Status getUploadState() {
+		return uploadState;
+	}
+
+	public void setUploadState(Status uploadState) {
+		this.uploadState = uploadState;
+	}
+
+	public String getUpload_errorString() {
+		return upload_errorString;
+	}
+
+	public void setUpload_errorString(String uploadErrorString) {
+		upload_errorString = uploadErrorString;
+	}
+
+	public String getUploadUrl() {
+		return uploadUrl;
+	}
+
+	public void setUploadUrl(String uploadUrl) {
+		this.uploadUrl = uploadUrl;
+	}
+
+	public String getUploadJobId() {
+		return uploadJobId;
+	}
+
+	public void setUploadJobId(String uploadJobId) {
+		this.uploadJobId = uploadJobId;
 	}
 }

core/src/com/cloud/storage/VMTemplateStorageResourceAssoc.java

@@ -24,7 +24,7 @@ import java.util.Date;
  *
  */
 public interface VMTemplateStorageResourceAssoc {
-	public static enum Status  {UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED}
+	public static enum Status  {UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS}
 
 	public String getInstallPath();
 

core/src/com/cloud/storage/dao/VMTemplateHostDao.java

@@ -18,9 +18,11 @@
 
 package com.cloud.storage.dao;
 
+import java.util.Date;
 import java.util.List;
 
 import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
 import com.cloud.utils.db.GenericDao;
 
 public interface VMTemplateHostDao extends GenericDao<VMTemplateHostVO, Long> {
@@ -41,6 +43,9 @@ public interface VMTemplateHostDao extends GenericDao<VMTemplateHostVO, Long> {
     List<VMTemplateHostVO> listByTemplatePool(long templateId, long poolId);
 
     void update(VMTemplateHostVO instance);
+    
+    void updateUploadStatus(long hostId, long templateId, int uploadPercent, Status uploadState,
+			String jobId, String uploadUrl );
 
     List<VMTemplateHostVO> listByTemplateStatus(long templateId, VMTemplateHostVO.Status downloadState);
 
@@ -53,4 +58,6 @@ public interface VMTemplateHostDao extends GenericDao<VMTemplateHostVO, Long> {
     List<VMTemplateHostVO> listDestroyed(long hostId);
 
     boolean templateAvailable(long templateId, long hostId);
+
+	List<VMTemplateHostVO> listByTemplateUploadStatus(long templateId,Status UploadState);
 }

core/src/com/cloud/storage/dao/VMTemplateHostDaoImpl.java

@@ -49,16 +49,26 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 	protected final SearchBuilder<VMTemplateHostVO> PoolTemplateSearch;
 	protected final SearchBuilder<VMTemplateHostVO> HostTemplatePoolSearch;
 	protected final SearchBuilder<VMTemplateHostVO> TemplateStatusSearch;
-	protected final SearchBuilder<VMTemplateHostVO> TemplateStatesSearch;
+	protected final SearchBuilder<VMTemplateHostVO> TemplateStatesSearch;
+	protected final SearchBuilder<VMTemplateHostVO> TemplateUploadStatusSearch;
 	
 	protected static final String UPDATE_TEMPLATE_HOST_REF =
 		"UPDATE template_host_ref SET download_state = ?, download_pct= ?, last_updated = ? "
 	+   ", error_str = ?, local_path = ?, job_id = ? "
+	+   "WHERE host_id = ? and template_id = ?";
+	
+	protected static final String UPDATE_UPLOAD_INFO =
+		"UPDATE template_host_ref SET upload_state = ?, upload_pct= ?, last_updated = ? "
+	+   ", upload_error_str = ?, upload_job_id = ? "
 	+   "WHERE host_id = ? and template_id = ?";
 	
 	protected static final String DOWNLOADS_STATE_DC=
 		"SELECT * FROM template_host_ref t, host h where t.host_id = h.id and h.data_center_id=? "
 	+	" and t.template_id=? and t.download_state = ?" ;
+
+	protected static final String UPLOADS_STATE_DC=
+		"SELECT * FROM template_host_ref t, host h where t.host_id = h.id and h.data_center_id=? "
+	+	" and t.template_id=? and t.upload_state = ?" ;
 	
 	protected static final String DOWNLOADS_STATE_DC_POD=
 		"SELECT * FROM template_host_ref t, host h where t.host_id = h.id and h.data_center_id=? and h.pod_id=? "
@@ -67,7 +77,12 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 	protected static final String DOWNLOADS_STATE=
 		"SELECT * FROM template_host_ref t "
 	+	" where t.template_id=? and t.download_state=?";
-	
+	
+
+	protected static final String UPLOADS_STATE=
+		"SELECT * FROM template_host_ref t "
+	+	" where t.template_id=? and t.upload_state=?";
+		
 	public VMTemplateHostDaoImpl () {
 		HostSearch = createSearchBuilder();
 		HostSearch.and("host_id", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
@@ -98,6 +113,11 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 		TemplateStatusSearch.and("template_id", TemplateStatusSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
 		TemplateStatusSearch.and("download_state", TemplateStatusSearch.entity().getDownloadState(), SearchCriteria.Op.EQ);
 		TemplateStatusSearch.done();
+		
+		TemplateUploadStatusSearch = createSearchBuilder();
+		TemplateUploadStatusSearch.and("template_id", TemplateUploadStatusSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
+		TemplateUploadStatusSearch.and("upload_state", TemplateUploadStatusSearch.entity().getUploadState(), SearchCriteria.Op.EQ);
+		TemplateUploadStatusSearch.done();
 		
 		TemplateStatesSearch = createSearchBuilder();
 		TemplateStatesSearch.and("template_id", TemplateStatesSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
@@ -129,6 +149,27 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 		} catch (Exception e) {
 			s_logger.warn("Exception: ", e);
 		}
+	}
+	
+	public void updateUploadStatus(long hostId, long templateId, int uploadPercent, Status uploadState,
+			String uploadJobId, String uploadUrl ) {
+        Transaction txn = Transaction.currentTxn();
+		PreparedStatement pstmt = null;
+		try {
+			Date now = new Date();
+			String sql = UPDATE_UPLOAD_INFO;
+			pstmt = txn.prepareAutoCloseStatement(sql);
+			pstmt.setString(1, uploadState.toString());
+			pstmt.setInt(2, uploadPercent);
+			pstmt.setString(3, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), now));
+			pstmt.setString(4, uploadJobId);
+			pstmt.setLong(5, hostId);
+			pstmt.setLong(6, templateId);
+			pstmt.setString(7, uploadUrl);
+			pstmt.executeUpdate();
+		} catch (Exception e) {
+			s_logger.warn("Exception: ", e);
+		}
 	}
 
 	@Override
@@ -160,6 +201,14 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
 	    sc.setParameters("template_id", templateId);
 	    return findOneBy(sc);
 	}
+
+	@Override
+	public List<VMTemplateHostVO> listByTemplateUploadStatus(long templateId, VMTemplateHostVO.Status uploadState) {
+		SearchCriteria<VMTemplateHostVO> sc = TemplateUploadStatusSearch.create();
+		sc.setParameters("template_id", templateId);
+		sc.setParameters("upload_state", uploadState.toString());
+		return listBy(sc);
+	}
 	
 	@Override
 	public List<VMTemplateHostVO> listByTemplateStatus(long templateId, VMTemplateHostVO.Status downloadState) {

core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java

@@ -48,6 +48,7 @@ import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig;
 import com.cloud.agent.api.storage.DeleteTemplateCommand;
 import com.cloud.agent.api.storage.DownloadCommand;
 import com.cloud.agent.api.storage.DownloadProgressCommand;
+import com.cloud.agent.api.storage.UploadCommand;
 import com.cloud.host.Host;
 import com.cloud.host.Host.Type;
 import com.cloud.resource.ServerResource;
@@ -58,6 +59,8 @@ import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.template.DownloadManager;
 import com.cloud.storage.template.DownloadManagerImpl;
 import com.cloud.storage.template.TemplateInfo;
+import com.cloud.storage.template.UploadManager;
+import com.cloud.storage.template.UploadManagerImpl;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.component.ComponentLocator;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -85,6 +88,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
     Random _rand = new Random(System.currentTimeMillis());
     
     DownloadManager _dlMgr;
+    UploadManager _upldMgr;
 	private String _configSslScr;
 	private String _configAuthScr;
 	private String _publicIp;
@@ -111,6 +115,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             return _dlMgr.handleDownloadCommand((DownloadProgressCommand)cmd);
         } else if (cmd instanceof DownloadCommand) {
             return _dlMgr.handleDownloadCommand((DownloadCommand)cmd);
+        }else if (cmd instanceof UploadCommand) {        	
+            return _upldMgr.handleUploadCommand((UploadCommand)cmd);
         } else if (cmd instanceof GetStorageStatsCommand) {
         	return execute((GetStorageStatsCommand)cmd);
         } else if (cmd instanceof CheckHealthCommand) {
@@ -413,6 +419,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
             _params.put(StorageLayer.InstanceConfigKey, _storage);
             _dlMgr = new DownloadManagerImpl();
             _dlMgr.configure("DownloadManager", _params);
+            _upldMgr = new UploadManagerImpl();
+            _upldMgr.configure("UploadManager", params);
         } catch (ConfigurationException e) {
             s_logger.warn("Caught problem while configuring DownloadManager", e);
             return false;

core/src/com/cloud/storage/template/FtpTemplateUploader.java

@@ -0,0 +1,224 @@
+package com.cloud.storage.template;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Date;
+
+import org.apache.log4j.Logger;
+
+
+public class FtpTemplateUploader implements TemplateUploader {
+	
+	public static final Logger s_logger = Logger.getLogger(FtpTemplateUploader.class.getName());
+	public TemplateUploader.Status status = TemplateUploader.Status.NOT_STARTED;
+	public String errorString = "";
+	public long totalBytes = 0;
+	public long templateSizeinBytes;
+	private String sourcePath;
+	private String ftpUrl;	
+	private UploadCompleteCallback completionCallback;
+	private boolean resume;
+    private BufferedInputStream inputStream = null;
+    private BufferedOutputStream outputStream = null;
+	private static final int CHUNK_SIZE = 1024*1024; //1M
+	
+	public FtpTemplateUploader(String sourcePath, String url, UploadCompleteCallback callback, long templateSizeinBytes){
+		
+		this.sourcePath = sourcePath;
+		this.ftpUrl = url;
+		this.completionCallback = callback;
+		this.templateSizeinBytes = templateSizeinBytes;
+		s_logger.warn("Nitin in  FtpTemplateUploader " +url + " "+sourcePath);
+	}
+	
+	public long upload(UploadCompleteCallback callback )
+		   {
+		
+				switch (status) {
+				case ABORTED:
+				case UNRECOVERABLE_ERROR:
+				case UPLOAD_FINISHED:
+					return 0;
+				default:
+		
+				}
+				
+	             Date start = new Date();
+				 s_logger.warn("Nitin in  FtpTemplateUploader ");
+		         StringBuffer sb = new StringBuffer();
+		         // check for authentication else assume its anonymous access.
+		        /* if (user != null && password != null)
+		         {
+		            sb.append( user );
+		            sb.append( ':' );
+		            sb.append( password );
+		            sb.append( '@' );
+		         }*/
+		         sb.append( ftpUrl );
+		         /*sb.append( '/' );
+		         sb.append( fileName ); filename where u want to dld it */
+		         /*ftp://10.91.18.14/
+		          * type ==> a=ASCII mode, i=image (binary) mode, d= file directory
+		          * listing
+		          */
+		         sb.append( ";type=i" );
+
+		         try
+		         {
+		            URL url = new URL( sb.toString() );
+		            URLConnection urlc = url.openConnection();
+
+		            outputStream = new BufferedOutputStream( urlc.getOutputStream() );
+		            inputStream = new BufferedInputStream( new FileInputStream( new File(sourcePath) ) );	            
+
+		            status = TemplateUploader.Status.IN_PROGRESS;
+
+		            int bytes = 0;
+		            byte[] block = new byte[CHUNK_SIZE];
+		            boolean done=false;
+		            while (!done && status != Status.ABORTED ) {
+		            	if ( (bytes = inputStream.read(block, 0, CHUNK_SIZE)) > -1) {
+		            		outputStream.write(block,0, bytes);		            			            				            			            		
+		            		totalBytes += bytes;
+		            	} else {
+		            		done = true;
+		            	}
+		            }		            
+		            status = TemplateUploader.Status.UPLOAD_FINISHED;
+		            s_logger.warn("Nitin in  FtpTemplateUploader " +status);
+		            return totalBytes;
+		         } catch (MalformedURLException e) {
+		        	status = TemplateUploader.Status.UNRECOVERABLE_ERROR;
+		 			errorString = e.getMessage();
+		 			s_logger.error("Nitin in  FtpTemplateUploader " +errorString);
+				} catch (IOException e) {
+					status = TemplateUploader.Status.UNRECOVERABLE_ERROR;
+		 			errorString = e.getMessage();
+		 			s_logger.error("Nitin in  FtpTemplateUploader " +errorString);
+				}
+		         finally
+		         {
+		           try		         
+	               {
+		            if (inputStream != null){		               
+		            	inputStream.close();
+		            }
+		            if (outputStream != null){		               
+		                  outputStream.close();
+		            }
+	               }catch (IOException ioe){
+	            	   s_logger.error(" Caught exception while closing the resources" ); 		                  
+	               }
+				   if (callback != null) {
+					   callback.uploadComplete(status);
+				   }
+		         }
+
+				return 0;
+		   }
+
+	@Override
+	public void run() {
+		try {
+			upload(completionCallback);
+		} catch (Throwable t) {
+			s_logger.warn("Caught exception during upload "+ t.getMessage(), t);
+			errorString = "Failed to install: " + t.getMessage();
+			status = TemplateUploader.Status.UNRECOVERABLE_ERROR;
+		}
+		
+	}
+
+	@Override
+	public Status getStatus() {
+		return status;
+	}
+
+	@Override
+	public String getUploadError() {
+		return errorString;
+	}
+
+	@Override
+	public String getUploadLocalPath() {
+		return null;
+	}
+
+	@Override
+	public int getUploadPercent() {
+		if (templateSizeinBytes == 0) {
+			return 0;
+		}		
+		return (int)(100.0*totalBytes/templateSizeinBytes);
+	}
+
+	@Override
+	public long getUploadTime() {
+		// TODO Auto-generated method stub
+		return 0;
+	}
+
+	@Override
+	public long getUploadedBytes() {
+		return totalBytes;
+	}
+
+	@Override
+	public boolean isInited() {
+		return false;
+	}
+
+	@Override
+	public void setResume(boolean resume) {
+		this.resume = resume;
+		
+	}
+
+	@Override
+	public void setStatus(Status status) {
+		this.status = status;		
+	}
+
+	@Override
+	public void setUploadError(String string) {
+		errorString = string;		
+	}
+
+	@Override
+	public boolean stopUpload() {
+		switch (getStatus()) {
+		case IN_PROGRESS:
+			try {
+				if(outputStream != null) {
+					outputStream.close();
+				}
+				if (inputStream != null){				
+					inputStream.close();					
+				}
+			} catch (IOException e) {
+				s_logger.error(" Caught exception while closing the resources" );
+			}
+			status = TemplateUploader.Status.ABORTED;
+			return true;
+		case UNKNOWN:
+		case NOT_STARTED:
+		case RECOVERABLE_ERROR:
+		case UNRECOVERABLE_ERROR:
+		case ABORTED:
+			status = TemplateUploader.Status.ABORTED;
+		case UPLOAD_FINISHED:
+			return true;
+
+		default:
+			return true;
+		}
+	}
+	
+
+}

core/src/com/cloud/storage/template/TemplateUploader.java

@@ -0,0 +1,77 @@
+package com.cloud.storage.template;
+
+import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback;
+import com.cloud.storage.template.TemplateUploader.Status;
+
+public interface TemplateUploader extends Runnable{
+
+	/**
+	 * Callback used to notify completion of upload
+	 * @author nitin
+	 *
+	 */
+	public interface UploadCompleteCallback {
+		void uploadComplete( Status status);
+
+	}
+
+	public static enum Status  {UNKNOWN, NOT_STARTED, IN_PROGRESS, ABORTED, UNRECOVERABLE_ERROR, RECOVERABLE_ERROR, UPLOAD_FINISHED, POST_UPLOAD_FINISHED}
+
+	
+	/**
+	 * Initiate upload
+	 * @param callback completion callback to be called after upload is complete
+	 * @return bytes uploaded
+	 */
+	public long upload(UploadCompleteCallback callback);
+	
+	/**
+	 * @return
+	 */
+	public boolean stopUpload();
+	
+	/**
+	 * @return percent of file uploaded
+	 */
+	public int getUploadPercent();
+
+	/**
+	 * Get the status of the upload
+	 * @return status of upload
+	 */
+	public TemplateUploader.Status getStatus();
+
+
+	/**
+	 * Get time taken to upload so far
+	 * @return time in seconds taken to upload
+	 */
+	public long getUploadTime();
+
+	/**
+	 * Get bytes uploaded
+	 * @return bytes uploaded so far
+	 */
+	public long getUploadedBytes();
+
+	/**
+	 * Get the error if any
+	 * @return error string if any
+	 */
+	public String getUploadError();
+
+	/** Get local path of the uploaded file
+	 * @return local path of the file uploaded
+	 */
+	public String getUploadLocalPath();
+
+	public void setStatus(TemplateUploader.Status status);
+
+	public void setUploadError(String string);
+
+	public void setResume(boolean resume);
+	
+	public boolean isInited();	
+	
+
+}

core/src/com/cloud/storage/template/UploadManager.java

@@ -0,0 +1,68 @@
+package com.cloud.storage.template;
+
+import java.util.List;
+import java.util.Map;
+
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.storage.StorageResource;
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.utils.component.Manager;
+
+public interface UploadManager extends Manager {
+
+
+	/**
+	 * Get the status of a upload job
+	 * @param jobId job Id
+	 * @return status of the upload job
+	 */
+	public TemplateUploader.Status getUploadStatus(String jobId);
+	
+	/**
+	 * Get the status of a upload job
+	 * @param jobId job Id
+	 * @return status of the upload job
+	 */
+	public VMTemplateHostVO.Status getUploadStatus2(String jobId);
+
+	/**
+	 * Get the upload percent of a upload job
+	 * @param jobId job Id
+	 * @return
+	 */
+	public int getUploadPct(String jobId);
+
+	/**
+	 * Get the upload error if any
+	 * @param jobId job Id
+	 * @return
+	 */
+	public String getUploadError(String jobId);
+
+	/**
+	 * Get the local path for the upload
+	 * @param jobId job Id
+	 * @return
+	public String getUploadLocalPath(String jobId);
+     */
+	
+	/** Handle upload commands from the management server
+	 * @param cmd cmd from server
+	 * @return answer representing status of upload.
+	 */
+	public UploadAnswer handleUploadCommand(UploadCommand cmd);		
+	
+	public String setRootDir(String rootDir, StorageResource storage);
+    
+    public String getPublicTemplateRepo();
+
+
+	String uploadPublicTemplate(long id, String url, String name,
+			ImageFormat format, Long accountId, String descr,
+			String cksum, String installPathPrefix, String user,
+			String password, long maxTemplateSizeInBytes);
+	
+}

core/src/com/cloud/storage/template/UploadManagerImpl.java

@@ -0,0 +1,597 @@
+package com.cloud.storage.template;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadProgressCommand;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.storage.StorageLayer;
+import com.cloud.storage.StorageResource;
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback;
+import com.cloud.storage.template.TemplateUploader.Status;
+import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.UUID;
+import com.cloud.utils.component.Adapters;
+import com.cloud.utils.component.ComponentLocator;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.script.Script;
+
+public class UploadManagerImpl implements UploadManager {
+
+
+   public class Completion implements UploadCompleteCallback {
+        private final String jobId;
+
+        public Completion(String jobId) {
+            this.jobId = jobId;
+        }
+
+        @Override
+        public void uploadComplete(Status status) {
+            setUploadStatus(jobId, status);
+        }
+    }
+   
+   private static class UploadJob {
+       private final TemplateUploader td;
+       private final String jobId;
+       private final String tmpltName;
+       private final ImageFormat format;
+       private String tmpltPath;
+       private String description;
+       private String checksum;
+       private Long accountId;
+       private String installPathPrefix;
+       private long templatesize;
+       private long id;
+
+       public UploadJob(TemplateUploader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix) {
+           super();
+           this.td = td;
+           this.jobId = jobId;
+           this.tmpltName = tmpltName;
+           this.format = format;
+           this.accountId = accountId;
+           this.description = descr;
+           this.checksum = cksum;
+           this.installPathPrefix = installPathPrefix;
+           this.templatesize = 0;
+           this.id = id;
+       }
+
+       public TemplateUploader getTd() {
+           return td;
+       }
+
+       public String getDescription() {
+           return description;
+       }
+
+       public String getChecksum() {
+           return checksum;
+       }
+
+       public UploadJob(TemplateUploader td, String jobId, UploadCommand cmd) {
+           this.td = td;
+           this.jobId = jobId;
+           this.tmpltName = cmd.getName();
+           this.format = cmd.getFormat();           
+       }
+
+       public TemplateUploader getTemplateUploader() {
+           return td;
+       }
+
+       public String getJobId() {
+           return jobId;
+       }
+
+       public String getTmpltName() {
+           return tmpltName;
+       }
+
+       public ImageFormat getFormat() {
+           return format;
+       }
+
+       public Long getAccountId() {
+           return accountId;
+       }
+
+       public long getId() {
+           return id;
+       }
+
+       public void setTmpltPath(String tmpltPath) {
+           this.tmpltPath = tmpltPath;
+       }
+
+       public String getTmpltPath() {
+           return tmpltPath;
+       }
+
+       public String getInstallPathPrefix() {
+           return installPathPrefix;
+       }
+
+       public void cleanup() {
+       }
+
+       public void setTemplatesize(long templatesize) {
+           this.templatesize = templatesize;
+       }
+
+       public long getTemplatesize() {
+           return templatesize;
+       }
+   }
+   public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class);
+   private ExecutorService threadPool;
+   private final Map<String, UploadJob> jobs = new ConcurrentHashMap<String, UploadJob>();
+	private String parentDir;
+	private Adapters<Processor> _processors;
+	private String publicTemplateRepo;
+	private StorageLayer _storage;
+	private int installTimeoutPerGig;
+	private boolean _sslCopy;
+	private String _name;
+	private boolean hvm;
+   
+	@Override
+	public String uploadPublicTemplate(long id, String url, String name,
+			ImageFormat format, Long accountId, String descr,
+			String cksum, String installPathPrefix, String userName,
+			String passwd, long templateSizeInBytes) {		
+		
+        UUID uuid = new UUID();
+        String jobId = uuid.toString();
+
+        String completePath = parentDir + File.separator + installPathPrefix;
+        s_logger.debug("Starting upload from " + completePath);
+        
+        URI uri;
+		try {
+		    uri = new URI(url);
+		} catch (URISyntaxException e) {
+		    s_logger.error("URI is incorrect: " + url);
+		    throw new CloudRuntimeException("URI is incorrect: " + url);
+		}
+		TemplateUploader tu;
+		if ((uri != null) && (uri.getScheme() != null)) {
+		    if (uri.getScheme().equalsIgnoreCase("ftp")) {
+		        tu = new FtpTemplateUploader(completePath, url, new Completion(jobId), templateSizeInBytes);                                
+		    } else {
+		    	s_logger.error("Scheme is not supported " + url);
+		        throw new CloudRuntimeException("Scheme is not supported " + url);
+		    }
+		} else {
+		    s_logger.error("Unable to download from URL: " + url);
+		    throw new CloudRuntimeException("Unable to download from URL: " + url);
+		}
+		UploadJob uj = new UploadJob(tu, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix);
+		jobs.put(jobId, uj);
+		threadPool.execute(tu);
+
+		return jobId;
+				
+	}
+
+	@Override
+	public String getUploadError(String jobId) {
+        UploadJob uj = jobs.get(jobId);
+        if (uj != null) {
+            return uj.getTemplateUploader().getUploadError();
+        }
+        return null;
+	}
+
+	@Override
+	public int getUploadPct(String jobId) {
+		UploadJob uj = jobs.get(jobId);
+        if (uj != null) {
+            return uj.getTemplateUploader().getUploadPercent();
+        }
+        return 0;
+	}
+
+	@Override
+	public Status getUploadStatus(String jobId) {
+        UploadJob job = jobs.get(jobId);
+        if (job != null) {
+            TemplateUploader tu = job.getTemplateUploader();
+            if (tu != null) {
+                return tu.getStatus();
+            }
+        }
+        return Status.UNKNOWN;
+	}
+	
+    public static VMTemplateHostVO.Status convertStatus(Status tds) {
+        switch (tds) {
+        case ABORTED:
+            return VMTemplateHostVO.Status.NOT_UPLOADED;
+        case UPLOAD_FINISHED:
+            return VMTemplateHostVO.Status.UPLOAD_IN_PROGRESS;
+        case IN_PROGRESS:
+            return VMTemplateHostVO.Status.UPLOAD_IN_PROGRESS;
+        case NOT_STARTED:
+            return VMTemplateHostVO.Status.NOT_UPLOADED;
+        case RECOVERABLE_ERROR:
+            return VMTemplateHostVO.Status.NOT_UPLOADED;
+        case UNKNOWN:
+            return VMTemplateHostVO.Status.UNKNOWN;
+        case UNRECOVERABLE_ERROR:
+            return VMTemplateHostVO.Status.UPLOAD_ERROR;
+        case POST_UPLOAD_FINISHED:
+            return VMTemplateHostVO.Status.UPLOADED;
+        default:
+            return VMTemplateHostVO.Status.UNKNOWN;
+        }
+    }
+
+    @Override
+    public com.cloud.storage.VMTemplateHostVO.Status getUploadStatus2(String jobId) {
+        return convertStatus(getUploadStatus(jobId));
+    }
+	@Override
+	public String getPublicTemplateRepo() {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+    private UploadAnswer handleUploadProgressCmd(UploadProgressCommand cmd) {
+        String jobId = cmd.getJobId();
+        UploadAnswer answer;
+        UploadJob uj = null;
+        if (jobId != null)
+            uj = jobs.get(jobId);
+        if (uj == null) {           
+           return new UploadAnswer(null, 0, "Cannot find job", com.cloud.storage.VMTemplateHostVO.Status.UNKNOWN, "", "", 0);            
+        }
+        TemplateUploader td = uj.getTemplateUploader();
+        switch (cmd.getRequest()) {
+        case GET_STATUS:
+            break;
+        case ABORT:
+            td.stopUpload();
+            sleep();
+            break;
+        /*case RESTART:
+            td.stopUpload();
+            sleep();
+            threadPool.execute(td);
+            break;*/
+        case PURGE:
+            td.stopUpload();
+            answer = new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId), getUploadTemplateSize(jobId));
+            jobs.remove(jobId);
+            return answer;
+        default:
+            break; // TODO
+        }
+        return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+                getUploadTemplateSize(jobId));
+    }	
+	
+    @Override
+    public UploadAnswer handleUploadCommand(UploadCommand cmd) {
+    	s_logger.warn(" handliing the upload " +cmd.getInstallPath() + " " + cmd.getId());
+        if (cmd instanceof UploadProgressCommand) {
+            return handleUploadProgressCmd((UploadProgressCommand) cmd);
+        }
+        /*
+        if (cmd.getUrl() == null) {
+            return new UploadAnswer(null, 0, "Template is corrupted on storage due to an invalid url , cannot Upload", com.cloud.storage.VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR, "", "", 0);
+        }
+
+        if (cmd.getName() == null) {
+            return new UploadAnswer(null, 0, "Invalid Name", com.cloud.storage.VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR, "", "", 0);
+        }*/
+
+       // String installPathPrefix = null;
+       // installPathPrefix = publicTemplateRepo;
+
+        String user = null;
+        String password = null;                
+        String jobId = uploadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), 
+        									cmd.getFormat(), cmd.getAccountId(), cmd.getDescription(),
+        									cmd.getChecksum(), cmd.getInstallPath(), user, password,
+        									cmd.getTemplateSizeInBytes());
+        sleep();
+        if (jobId == null) {
+            return new UploadAnswer(null, 0, "Internal Error", com.cloud.storage.VMTemplateStorageResourceAssoc.Status.UPLOAD_ERROR, "", "", 0);
+        }
+        return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
+                getUploadTemplateSize(jobId));
+    }
+
+	private String getInstallPath(String jobId) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	private String getUploadLocalPath(String jobId) {
+		// TODO Auto-generated method stub
+		return null;
+	}
+
+	private long getUploadTemplateSize(String jobId){
+		UploadJob uj = jobs.get(jobId);
+        if (uj != null) {
+            return uj.getTemplatesize();
+        }
+        return 0;
+	}
+
+	@Override
+	public String setRootDir(String rootDir, StorageResource storage) {
+        this.publicTemplateRepo = rootDir + publicTemplateRepo;
+        return null;
+	}
+
+	@Override
+	public boolean configure(String name, Map<String, Object> params)
+			throws ConfigurationException {
+        _name = name;
+
+        String value = null;
+
+        _storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey);
+        if (_storage == null) {
+            value = (String) params.get(StorageLayer.ClassConfigKey);
+            if (value == null) {
+                throw new ConfigurationException("Unable to find the storage layer");
+            }
+
+            Class<StorageLayer> clazz;
+            try {
+                clazz = (Class<StorageLayer>) Class.forName(value);
+            } catch (ClassNotFoundException e) {
+                throw new ConfigurationException("Unable to instantiate " + value);
+            }
+            _storage = ComponentLocator.inject(clazz);
+        }
+        String useSsl = (String)params.get("sslcopy");
+        if (useSsl != null) {
+        	_sslCopy = Boolean.parseBoolean(useSsl);
+        	
+        }
+        configureFolders(name, params);
+        String inSystemVM = (String)params.get("secondary.storage.vm");
+        if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
+        	s_logger.info("UploadManager: starting additional services since we are inside system vm");
+        	startAdditionalServices();
+        	blockOutgoingOnPrivate();
+        }
+
+        value = (String) params.get("install.timeout.pergig");
+        this.installTimeoutPerGig = NumbersUtil.parseInt(value, 15 * 60) * 1000;
+
+        value = (String) params.get("install.numthreads");
+        final int numInstallThreads = NumbersUtil.parseInt(value, 10);        
+
+        String scriptsDir = (String) params.get("template.scripts.dir");
+        if (scriptsDir == null) {
+            scriptsDir = "scripts/storage/secondary";
+        }
+
+        List<Processor> processors = new ArrayList<Processor>();
+        _processors = new Adapters<Processor>("processors", processors);
+        Processor processor = new VhdProcessor();
+        
+        processor.configure("VHD Processor", params);
+        processors.add(processor);
+        
+        processor = new IsoProcessor();
+        processor.configure("ISO Processor", params);
+        processors.add(processor);
+        
+        processor = new QCOW2Processor();
+        processor.configure("QCOW2 Processor", params);
+        processors.add(processor);
+        // Add more processors here.
+        threadPool = Executors.newFixedThreadPool(numInstallThreads);
+        return true;
+	}
+	
+	protected void configureFolders(String name, Map<String, Object> params) throws ConfigurationException {
+        parentDir = (String) params.get("template.parent");
+        if (parentDir == null) {
+            throw new ConfigurationException("Unable to find the parent root for the templates");
+        }
+
+        String value = (String) params.get("public.templates.root.dir");
+        if (value == null) {
+            value = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR;
+        }
+               
+		if (value.startsWith(File.separator)) {
+            publicTemplateRepo = value;
+        } else {
+            publicTemplateRepo = parentDir + File.separator + value;
+        }
+        
+        if (!publicTemplateRepo.endsWith(File.separator)) {
+            publicTemplateRepo += File.separator;
+        }
+        
+        publicTemplateRepo += TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR;
+        
+        if (!_storage.mkdirs(publicTemplateRepo)) {
+            throw new ConfigurationException("Unable to create public templates directory");
+        }
+    }
+
+	@Override
+	public String getName() {
+		return _name;
+	}
+
+	@Override
+	public boolean start() {
+		return true;
+	}
+
+	@Override
+	public boolean stop() {
+		return true;
+	}
+
+    /**
+     * Get notified of change of job status. Executed in context of uploader thread
+     * 
+     * @param jobId
+     *            the id of the job
+     * @param status
+     *            the status of the job
+     */
+    public void setUploadStatus(String jobId, Status status) {
+        UploadJob uj = jobs.get(jobId);
+        if (uj == null) {
+            s_logger.warn("setUploadStatus for jobId: " + jobId + ", status=" + status + " no job found");
+            return;
+        }
+        TemplateUploader tu = uj.getTemplateUploader();
+        s_logger.warn("Upload Completion for jobId: " + jobId + ", status=" + status);
+        s_logger.warn("UploadedBytes=" + tu.getUploadedBytes() + ", error=" + tu.getUploadError() + ", pct=" + tu.getUploadPercent());
+
+        switch (status) {
+        case ABORTED:
+        case NOT_STARTED:
+        case UNRECOVERABLE_ERROR:
+            // TODO
+            uj.cleanup();
+            break;
+        case UNKNOWN:
+            return;
+        case IN_PROGRESS:
+            s_logger.info("Resuming jobId: " + jobId + ", status=" + status);
+            tu.setResume(true);
+            threadPool.execute(tu);
+            break;
+        case RECOVERABLE_ERROR:
+            threadPool.execute(tu);
+            break;
+        case UPLOAD_FINISHED:
+            tu.setUploadError("Upload success, starting install ");
+            String result = postUpload(jobId);
+            if (result != null) {
+                s_logger.error("Failed post upload script: " + result);
+                tu.setStatus(Status.UNRECOVERABLE_ERROR);
+                tu.setUploadError("Failed post upload script: " + result);
+            } else {
+            	s_logger.warn("Upload completed successfully at " + new SimpleDateFormat().format(new Date()));
+                tu.setStatus(Status.POST_UPLOAD_FINISHED);
+                tu.setUploadError("Upload completed successfully at " + new SimpleDateFormat().format(new Date()));
+            }
+            uj.cleanup();
+            break;
+        default:
+            break;
+        }
+    }
+
+	private String postUpload(String jobId) {
+		return null;
+	}
+
+    private void sleep() {
+        try {
+            Thread.sleep(3000);
+        } catch (InterruptedException e) {
+            // ignore
+        }
+    }
+
+    private void blockOutgoingOnPrivate() {
+    	Script command = new Script("/bin/bash", s_logger);
+    	String intf = "eth1";
+    	command.add("-c");
+    	command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" +
+    			"iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;");
+
+    	String result = command.execute();
+    	if (result != null) {
+    		s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result );
+    		return;
+    	}		
+	}
+
+  private void startAdditionalServices() {
+    	
+    	Script command = new Script("/bin/bash", s_logger);
+		command.add("-c");
+    	command.add("service httpd stop ");
+    	String result = command.execute();
+    	if (result != null) {
+    		s_logger.warn("Error in stopping httpd service err=" + result );
+    	}
+    	String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
+    	String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF;
+    	
+    	command = new Script("/bin/bash", s_logger);
+		command.add("-c");
+    	command.add("iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j DROP;" +
+			        "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j HTTP;" +
+			        "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j DROP;" +
+			        "iptables -D INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j HTTP;" +
+			        "iptables -F HTTP;" +
+			        "iptables -X HTTP;" +
+			        "iptables -N HTTP;" +
+    			    "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j DROP;" +
+    			    "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j DROP;" +
+    			    "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j HTTP;" +
+    	            "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j HTTP;");
+
+    	result = command.execute();
+    	if (result != null) {
+    		s_logger.warn("Error in opening up httpd port err=" + result );
+    		return;
+    	}
+    	
+    	command = new Script("/bin/bash", s_logger);
+		command.add("-c");
+    	command.add("service httpd start ");
+    	result = command.execute();
+    	if (result != null) {
+    		s_logger.warn("Error in starting httpd service err=" + result );
+    		return;
+    	}
+    	command = new Script("mkdir", s_logger);
+		command.add("-p");
+    	command.add("/var/www/html/copy/template");
+    	result = command.execute();
+    	if (result != null) {
+    		s_logger.warn("Error in creating directory =" + result );
+    		return;
+    	}
+    	
+    	command = new Script("/bin/bash", s_logger);
+		command.add("-c");
+    	command.add("ln -sf " + publicTemplateRepo + " /var/www/html/copy/template");
+    	result = command.execute();
+    	if (result != null) {
+    		s_logger.warn("Error in linking  err=" + result );
+    		return;
+    	}
+	}
+}    

server/src/com/cloud/api/commands/ExtractTemplateCmd.java

@@ -0,0 +1,85 @@
+package com.cloud.api.commands;
+
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.api.BaseCmd;
+import com.cloud.api.ServerApiException;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.user.Account;
+import com.cloud.utils.Pair;
+
+public class ExtractTemplateCmd extends BaseCmd {
+
+	public static final Logger s_logger = Logger.getLogger(ExtractTemplateCmd.class.getName());
+
+    private static final String s_name = "extracttemplateresponse";
+    private static final List<Pair<Enum, Boolean>> s_properties = new ArrayList<Pair<Enum, Boolean>>();
+
+    static {        
+        s_properties.add(new Pair<Enum, Boolean>(BaseCmd.Properties.URL, Boolean.TRUE));
+        s_properties.add(new Pair<Enum, Boolean>(BaseCmd.Properties.ID, Boolean.TRUE));
+        s_properties.add(new Pair<Enum, Boolean>(BaseCmd.Properties.ZONE_ID, Boolean.TRUE));
+        s_properties.add(new Pair<Enum, Boolean>(BaseCmd.Properties.ACCOUNT_OBJ, Boolean.FALSE));
+    }
+    
+	@Override
+	public List<Pair<String, Object>> execute(Map<String, Object> params) {
+		String url		   = (String) params.get(BaseCmd.Properties.URL.getName());
+		Long templateId    = (Long) params.get(BaseCmd.Properties.ID.getName());
+		Long zoneId		   = (Long) params.get(BaseCmd.Properties.ZONE_ID.getName());
+		Account account = (Account) params.get(BaseCmd.Properties.ACCOUNT_OBJ.getName());				
+		
+        VMTemplateVO template = getManagementServer().findTemplateById(templateId.longValue());
+        if (template == null) {
+            throw new ServerApiException(BaseCmd.INTERNAL_ERROR, "Unable to find template with id " + templateId);
+        }
+		
+        if(url.toLowerCase().contains("file://")){
+        	throw new ServerApiException(BaseCmd.PARAM_ERROR, "file:// type urls are currently unsupported");
+        }
+        
+        boolean isAdmin;
+    	if (account == null) {
+    		// Admin API call
+    		isAdmin = true;
+    	} else {
+    		// User API call
+    		isAdmin = isAdmin(account.getType());
+    	}
+    	
+    	if(!isAdmin){
+    		if (template.getAccountId() != account.getId()){
+    			throw new ServerApiException(BaseCmd.PARAM_ERROR, "Unable to find template with ID: " + templateId + " for account: " + account.getAccountName());
+    		}
+    	}
+    	
+        try {
+			getManagementServer().extractTemplate(url, templateId, zoneId);
+		} catch (Exception e) {			
+			s_logger.error(e.getMessage(), e);
+            throw new ServerApiException(BaseCmd.INTERNAL_ERROR, "Internal Error Extracting the template " + e.getMessage());
+		}				
+				
+		List<Pair<String, Object>> response = new ArrayList<Pair<String, Object>>();
+		response.add(new Pair<String, Object>("template", templateId));
+		response.add(new Pair<String, Object>("url", url));
+		response.add(new Pair<String, Object>("zoneid", zoneId));
+		return response;
+	}
+
+	@Override
+	public String getName() {
+		return s_name;
+	}
+
+	@Override
+	public List<Pair<Enum, Boolean>> getProperties() {
+		return s_properties;
+	}
+
+}

server/src/com/cloud/server/ManagementServerImpl.java

@@ -4690,6 +4690,42 @@ public class ManagementServerImpl implements ManagementServer {
         return _vlanDao.findById(vlanDbId);
     }
 
+    @Override
+    public void extractTemplate(String url, Long templateId, Long zoneId) throws URISyntaxException{
+    
+        URI uri = new URI(url);
+        if ( (uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp") )) {
+           throw new IllegalArgumentException("Unsupported scheme for url: " + url);
+        }
+        String host = uri.getHost();
+        
+        try {
+        	InetAddress hostAddr = InetAddress.getByName(host);
+        	if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress() ) {
+        		throw new IllegalArgumentException("Illegal host specified in url");
+        	}
+        	if (hostAddr instanceof Inet6Address) {
+        		throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
+        	}
+        } catch (UnknownHostException uhe) {
+        	throw new IllegalArgumentException("Unable to resolve " + host);
+        }
+        
+    	if (_dcDao.findById(zoneId) == null) {
+    		throw new IllegalArgumentException("Please specify a valid zone.");
+    	}
+        
+        VMTemplateVO template = findTemplateById(templateId);
+        
+        VMTemplateHostVO tmpltHostRef = findTemplateHostRef(templateId, zoneId);
+        if (tmpltHostRef != null && tmpltHostRef.getDownloadState() != com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED){
+        	throw new IllegalArgumentException("The template hasnt been downloaded ");
+        }
+        
+        _tmpltMgr.extract(template, url, tmpltHostRef, zoneId);
+        
+    }
+    
     @Override
     public Long createTemplate(long userId, Long zoneId, String name, String displayText, boolean isPublic, boolean featured, String format, String diskType, String url, String chksum, boolean requiresHvm, int bits, boolean enablePassword, long guestOSId, boolean bootable) throws InvalidParameterValueException,IllegalArgumentException, ResourceAllocationException {
         try

server/src/com/cloud/storage/upload/NotUploadedState.java

@@ -0,0 +1,23 @@
+package com.cloud.storage.upload;
+
+import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+
+public class NotUploadedState extends UploadActiveState {
+	
+	public NotUploadedState(UploadListener uploadListener) {
+		super(uploadListener);
+	}	
+
+	@Override
+	public String getName() {
+		return Status.NOT_UPLOADED.toString();
+	}
+
+	@Override
+	public void onEntry(String prevState, UploadEvent event, Object evtObj) {
+		super.onEntry(prevState, event, evtObj);
+		getUploadListener().scheduleStatusCheck(RequestType.GET_STATUS);
+	}
+	
+}

server/src/com/cloud/storage/upload/UploadAbandonedState.java

@@ -0,0 +1,27 @@
+package com.cloud.storage.upload;
+
+import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+
+public class UploadAbandonedState extends UploadInactiveState {
+
+	public UploadAbandonedState(UploadListener dl) {
+		super(dl);
+	}
+
+	@Override
+	public String getName() {
+		return Status.ABANDONED.toString();
+	}
+
+	@Override
+	public void onEntry(String prevState, UploadEvent event, Object evtObj) {
+		super.onEntry(prevState, event, evtObj);
+		if (!prevState.equalsIgnoreCase(getName())){
+			getUploadListener().updateDatabase(Status.ABANDONED, "Upload canceled");
+			getUploadListener().cancelStatusTask();
+			getUploadListener().cancelTimeoutTask();
+			getUploadListener().sendCommand(RequestType.ABORT);
+		}
+	}
+}

server/src/com/cloud/storage/upload/UploadActiveState.java

@@ -0,0 +1,95 @@
+package com.cloud.storage.upload;
+
+import org.apache.log4j.Level;
+
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.download.DownloadState.DownloadEvent;
+
+public abstract class UploadActiveState extends UploadState {
+	
+	public UploadActiveState(UploadListener ul) {
+		super(ul);
+	}
+	
+	@Override
+	public  String handleAbort(){
+		return Status.ABANDONED.toString();
+	}
+	
+	@Override
+	public  String handleDisconnect(){
+
+		return Status.UPLOAD_ERROR.toString();
+	}
+
+
+	@Override
+	public String handleAnswer(UploadAnswer answer) {
+		if (s_logger.isDebugEnabled()) {
+			s_logger.debug("handleAnswer, answer status=" + answer.getUploadStatus() + ", curr state=" + getName());
+		}
+		switch (answer.getUploadStatus()) {
+		case UPLOAD_IN_PROGRESS:
+			getUploadListener().scheduleStatusCheck(RequestType.GET_STATUS);
+			return Status.UPLOAD_IN_PROGRESS.toString();
+		case UPLOADED:
+			getUploadListener().scheduleImmediateStatusCheck(RequestType.PURGE);
+			getUploadListener().cancelTimeoutTask();
+			return Status.UPLOADED.toString();
+		case NOT_UPLOADED:
+			getUploadListener().scheduleStatusCheck(RequestType.GET_STATUS);
+			return Status.NOT_UPLOADED.toString();
+		case UPLOAD_ERROR:
+			getUploadListener().cancelStatusTask();
+			getUploadListener().cancelTimeoutTask();
+			return Status.UPLOAD_ERROR.toString();
+		case UNKNOWN:
+			getUploadListener().cancelStatusTask();
+			getUploadListener().cancelTimeoutTask();
+			return Status.UPLOAD_ERROR.toString();
+		default:
+			return null;
+		}
+	}
+
+	@Override
+	public String handleTimeout(long updateMs) {
+		if (s_logger.isTraceEnabled()) {
+			getUploadListener().log("handleTimeout, updateMs=" + updateMs + ", curr state= " + getName(), Level.TRACE);
+		}
+		String newState = this.getName();
+		if (updateMs > 5*UploadListener.STATUS_POLL_INTERVAL){
+			newState=Status.UPLOAD_ERROR.toString();
+			getUploadListener().log("timeout: transitioning to upload error state, currstate=" + getName(), Level.DEBUG );
+		} else if (updateMs > 3*UploadListener.STATUS_POLL_INTERVAL) {
+			getUploadListener().cancelStatusTask();
+			getUploadListener().scheduleImmediateStatusCheck(RequestType.GET_STATUS);
+			getUploadListener().scheduleTimeoutTask(3*UploadListener.STATUS_POLL_INTERVAL);
+			getUploadListener().log(getName() + " first timeout: checking again ", Level.DEBUG );
+		} else {
+			getUploadListener().scheduleTimeoutTask(3*UploadListener.STATUS_POLL_INTERVAL);
+		}
+		return newState;
+	}
+	
+	@Override
+	public  void onEntry(String prevState, UploadEvent event, Object evtObj) {
+		if (s_logger.isTraceEnabled()) {
+			getUploadListener().log("onEntry, prev state= " + prevState + ", curr state=" + getName() + ", event=" + event, Level.TRACE);
+		}
+		
+		if (event == UploadEvent.UPLOAD_ANSWER) {
+			getUploadListener().updateDatabase((UploadAnswer)evtObj);
+			getUploadListener().setLastUpdated();
+		}
+		
+	}
+	
+	@Override
+	public  void onExit() {
+	}
+}

server/src/com/cloud/storage/upload/UploadCompleteState.java

@@ -0,0 +1,30 @@
+package com.cloud.storage.upload;
+
+import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+
+public class UploadCompleteState extends UploadInactiveState {
+
+	public UploadCompleteState(UploadListener ul) {
+		super(ul);
+	}
+
+	@Override
+	public String getName() {
+		return Status.UPLOADED.toString();
+
+	}
+
+
+	@Override
+	public void onEntry(String prevState, UploadEvent event, Object evtObj) {
+		super.onEntry(prevState, event, evtObj);
+		if (! prevState.equals(getName())) {
+			if (event == UploadEvent.UPLOAD_ANSWER){
+				getUploadListener().scheduleImmediateStatusCheck(RequestType.PURGE);
+			}
+			getUploadListener().setUploadInactive(Status.UPLOADED);
+		}
+		
+	}
+}

server/src/com/cloud/storage/upload/UploadErrorState.java

@@ -0,0 +1,73 @@
+package com.cloud.storage.upload;
+
+import org.apache.log4j.Level;
+
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+
+public class UploadErrorState extends UploadInactiveState {
+
+	public UploadErrorState(UploadListener ul) {
+		super(ul);
+	}
+
+	@Override
+	public String handleAnswer(UploadAnswer answer) {
+		switch (answer.getUploadStatus()) {
+		case UPLOAD_IN_PROGRESS:
+			getUploadListener().scheduleStatusCheck(RequestType.GET_STATUS);
+			return Status.UPLOAD_IN_PROGRESS.toString();
+		case UPLOADED:
+			getUploadListener().scheduleImmediateStatusCheck(RequestType.PURGE);
+			getUploadListener().cancelTimeoutTask();
+			return Status.UPLOADED.toString();
+		case NOT_UPLOADED:
+			getUploadListener().scheduleStatusCheck(RequestType.GET_STATUS);
+			return Status.NOT_UPLOADED.toString();
+		case UPLOAD_ERROR:
+			getUploadListener().cancelStatusTask();
+			getUploadListener().cancelTimeoutTask();
+			return Status.UPLOAD_ERROR.toString();
+		case UNKNOWN:
+			getUploadListener().cancelStatusTask();
+			getUploadListener().cancelTimeoutTask();
+			return Status.UPLOAD_ERROR.toString();
+		default:
+			return null;
+		}
+	}
+
+
+
+	@Override
+	public String handleAbort() {
+		return Status.ABANDONED.toString();
+	}
+
+
+	@Override
+	public String getName() {
+		return Status.UPLOAD_ERROR.toString();
+	}
+
+
+	@Override
+	public void onEntry(String prevState, UploadEvent event, Object evtObj) {
+		super.onEntry(prevState, event, evtObj);
+		if (event==UploadEvent.DISCONNECT){
+			getUploadListener().logDisconnect();
+			getUploadListener().cancelStatusTask();
+			getUploadListener().cancelTimeoutTask();
+			getUploadListener().updateDatabase(Status.UPLOAD_ERROR, "Storage agent or storage VM disconnected");  
+			getUploadListener().log("Entering upload error state because the storage host disconnected", Level.WARN);
+		} else if (event==UploadEvent.TIMEOUT_CHECK){
+			getUploadListener().updateDatabase(Status.UPLOAD_ERROR, "Timeout waiting for response from storage host");
+			getUploadListener().log("Entering upload error state: timeout waiting for response from storage host", Level.WARN);
+		}
+		getUploadListener().setUploadInactive(Status.UPLOAD_ERROR);
+	}
+
+
+
+}

server/src/com/cloud/storage/upload/UploadInProgressState.java

@@ -0,0 +1,23 @@
+package com.cloud.storage.upload;
+
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+
+public class UploadInProgressState extends UploadActiveState {
+
+	public UploadInProgressState(UploadListener dl) {
+		super(dl);
+	}
+
+	@Override
+	public String getName() {
+		return Status.UPLOAD_IN_PROGRESS.toString();
+	}
+
+	@Override
+	public void onEntry(String prevState, UploadEvent event, Object evtObj) {
+		super.onEntry(prevState, event, evtObj);
+		if (!prevState.equals(getName()))
+			getUploadListener().logUploadStart();
+	}
+
+}

server/src/com/cloud/storage/upload/UploadInactiveState.java

@@ -0,0 +1,35 @@
+package com.cloud.storage.upload;
+
+import com.cloud.agent.api.storage.DownloadAnswer;
+import com.cloud.agent.api.storage.UploadAnswer;
+
+public abstract class UploadInactiveState extends UploadState {
+
+	public UploadInactiveState(UploadListener ul) {
+		super(ul);
+	}
+
+	@Override
+	public String handleAnswer(UploadAnswer answer) {
+		// ignore and stay put
+		return getName();
+	}
+
+	@Override
+	public String handleAbort() {
+		// ignore and stay put
+		return getName();
+	}
+
+	@Override
+	public String handleDisconnect() {
+		//ignore and stay put
+		return getName();
+	}
+
+	@Override
+	public String handleTimeout(long updateMs) {
+		// ignore and stay put
+		return getName();
+	}
+}

server/src/com/cloud/storage/upload/UploadListener.java

@@ -0,0 +1,369 @@
+package com.cloud.storage.upload;
+
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.Listener;
+import com.cloud.agent.api.AgentControlAnswer;
+import com.cloud.agent.api.AgentControlCommand;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.StartupCommand;
+import com.cloud.agent.api.StartupStorageCommand;
+import com.cloud.agent.api.storage.DownloadCommand;
+import com.cloud.agent.api.storage.DownloadProgressCommand;
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.agent.api.storage.UploadProgressCommand;
+import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
+import com.cloud.event.EventTypes;
+import com.cloud.event.EventVO;
+import com.cloud.host.HostVO;
+import com.cloud.storage.Storage;
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.VMTemplateHostDao;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.download.DownloadState.DownloadEvent;
+import com.cloud.storage.upload.UploadMonitorImpl;
+import com.cloud.storage.upload.UploadState.UploadEvent;
+import com.cloud.utils.exception.CloudRuntimeException;
+
+public class UploadListener implements Listener {
+	
+
+	private static final class StatusTask extends TimerTask {
+		private final UploadListener ul;
+		private final RequestType reqType;
+		
+		public StatusTask( UploadListener ul,  RequestType req) {
+			this.reqType = req;
+			this.ul = ul;
+		}
+
+		@Override
+		public void run() {
+		  ul.sendCommand(reqType);
+
+		}
+	}
+	
+	private static final class TimeoutTask extends TimerTask {
+		private final UploadListener ul;
+		
+		public TimeoutTask( UploadListener ul) {
+			this.ul = ul;
+		}
+
+		@Override
+		public void run() {
+		  ul.checkProgress();
+		}
+	}
+
+	public static final Logger s_logger = Logger.getLogger(UploadListener.class.getName());
+	public static final int SMALL_DELAY = 100;
+	public static final long STATUS_POLL_INTERVAL = 10000L;
+	
+	public static final String UPLOADED=Status.UPLOADED.toString();
+	public static final String NOT_UPLOADED=Status.NOT_UPLOADED.toString();
+	public static final String UPLOAD_ERROR=Status.UPLOAD_ERROR.toString();
+	public static final String UPLOAD_IN_PROGRESS=Status.UPLOAD_IN_PROGRESS.toString();
+	public static final String UPLOAD_ABANDONED=Status.ABANDONED.toString();
+
+
+	private HostVO sserver;
+	private VMTemplateVO template;
+	
+	private boolean uploadActive = true;
+
+	private VMTemplateHostDao vmTemplateHostDao;
+
+	private final UploadMonitorImpl uploadMonitor;
+	
+	private UploadState currState;
+	
+	private UploadCommand cmd;
+
+	private Timer timer;
+
+	private StatusTask statusTask;
+	private TimeoutTask timeoutTask;
+	private Date lastUpdated = new Date();
+	private String jobId;
+	
+	private final Map<String,  UploadState> stateMap = new HashMap<String, UploadState>();
+	private Long templateHostId;
+	
+	public UploadListener(HostVO host, VMTemplateVO template, Timer _timer, VMTemplateHostDao dao, Long templHostId, UploadMonitorImpl uploadMonitor, UploadCommand cmd) {
+		this.sserver = host;
+		this.template = template;
+		this.vmTemplateHostDao = dao;
+		this.uploadMonitor = uploadMonitor;
+		this.cmd = cmd;
+		this.templateHostId = templHostId;
+		initStateMachine();
+		this.currState = getState(Status.NOT_UPLOADED.toString());
+		this.timer = _timer;
+		this.timeoutTask = new TimeoutTask(this);
+		this.timer.schedule(timeoutTask, 3*STATUS_POLL_INTERVAL);
+		updateDatabase(Status.NOT_UPLOADED, cmd.getUrl(),"");
+	}
+	
+	public UploadListener(UploadMonitorImpl monitor) {
+	    uploadMonitor = monitor;
+	}	
+	
+	public void checkProgress() {
+		transition(UploadEvent.TIMEOUT_CHECK, null);
+	}
+
+	@Override
+	public int getTimeout() {
+		return -1;
+	}
+
+	@Override
+	public boolean isRecurring() {
+		return false;
+	}
+
+	public void setCommand(UploadCommand _cmd) {
+		this.cmd = _cmd;
+	}
+	
+	public void setJobId(String _jobId) {
+		this.jobId = _jobId;
+	}
+	
+	public String getJobId() {
+		return jobId;
+	}
+	
+	@Override
+	public boolean processAnswer(long agentId, long seq, Answer[] answers) {
+		boolean processed = false;
+    	if(answers != null & answers.length > 0) {
+    		if(answers[0] instanceof UploadAnswer) {
+    			final UploadAnswer answer = (UploadAnswer)answers[0];
+    			if (getJobId() == null) {
+    				setJobId(answer.getJobId());
+    			} else if (!getJobId().equalsIgnoreCase(answer.getJobId())){
+    				return false;//TODO
+    			}
+    			transition(UploadEvent.UPLOAD_ANSWER, answer);
+    			processed = true;
+    		}
+    	}
+        return processed;
+	}
+	
+
+	@Override
+	public boolean processCommand(long agentId, long seq, Command[] commands) {
+		return false;
+	}
+
+	@Override
+	public boolean processConnect(HostVO agent, StartupCommand cmd) {		
+		if (!(cmd instanceof StartupStorageCommand)) {
+	        return true;
+	    }
+	   /* if (cmd.getGuid().startsWith("iso:")) {
+	        //FIXME: do not download template for ISO secondary
+	        return true;
+	    }*/
+	    
+	    long agentId = agent.getId();
+	    
+	    StartupStorageCommand storage = (StartupStorageCommand)cmd;
+	    if (storage.getResourceType() == Storage.StorageResourceType.STORAGE_HOST ||
+	    storage.getResourceType() == Storage.StorageResourceType.SECONDARY_STORAGE )
+	    {
+	    	uploadMonitor.handleUploadTemplateSync(agentId, storage.getTemplateInfo());
+	    } else {
+	    	//downloadMonitor.handlePoolTemplateSync(storage.getPoolInfo(), storage.getTemplateInfo());
+	    	//no need to do anything. The storagepoolmonitor will initiate template sync.
+	    }
+		return true;
+	}
+
+	@Override
+	public AgentControlAnswer processControlCommand(long agentId,
+			AgentControlCommand cmd) {
+		return null;
+	}
+	
+	public void setUploadInactive(Status reason) {
+		uploadActive=false;
+		uploadMonitor.handleUploadEvent(sserver, template, reason);
+	}
+	
+	public void logUploadStart() {
+		uploadMonitor.logEvent(template.getAccountId(), EventTypes.EVENT_TEMPLATE_UPLOAD_START, "Storage server " + sserver.getName() + " started upload of template " + template.getName(), EventVO.LEVEL_INFO);
+	}
+	
+	public void cancelTimeoutTask() {
+		if (timeoutTask != null) timeoutTask.cancel();
+	}
+	
+	public void cancelStatusTask() {
+		if (statusTask != null) statusTask.cancel();
+	}
+
+	@Override
+	public boolean processDisconnect(long agentId, com.cloud.host.Status state) {	
+		setDisconnected();
+		return true;
+	}
+
+	@Override
+	public boolean processTimeout(long agentId, long seq) {		
+		return true;
+	}
+	
+	private void initStateMachine() {
+		stateMap.put(Status.NOT_UPLOADED.toString(), new NotUploadedState(this));
+		stateMap.put(Status.UPLOADED.toString(), new UploadCompleteState(this));
+		stateMap.put(Status.UPLOAD_ERROR.toString(), new UploadErrorState(this));
+		stateMap.put(Status.UPLOAD_IN_PROGRESS.toString(), new UploadInProgressState(this));
+		stateMap.put(Status.ABANDONED.toString(), new UploadAbandonedState(this));
+	}
+	
+	private UploadState getState(String stateName) {
+		return stateMap.get(stateName);
+	}
+
+	private synchronized void transition(UploadEvent event, Object evtObj) {
+	    if (currState == null) {
+	        return;
+	    }
+		String prevName = currState.getName();
+		String nextState = currState.handleEvent(event, evtObj);
+		if (nextState != null) {
+			currState = getState(nextState);
+			if (currState != null) {
+				currState.onEntry(prevName, event, evtObj);
+			} else {
+				throw new CloudRuntimeException("Invalid next state: currState="+prevName+", evt="+event + ", next=" + nextState);
+			}
+		} else {
+			throw new CloudRuntimeException("Unhandled event transition: currState="+prevName+", evt="+event);
+		}
+	}
+	
+	public Date getLastUpdated() {
+		return lastUpdated;
+	}
+	
+	public void setLastUpdated() {
+		lastUpdated  = new Date();
+	}
+	
+	public void log(String message, Level level) {
+		s_logger.log(level, message + ", template=" + template.getName() + " at host " + sserver.getName());
+	}
+
+	public void setDisconnected() {
+		transition(UploadEvent.DISCONNECT, null);
+	}
+	
+	public void scheduleStatusCheck(com.cloud.agent.api.storage.UploadProgressCommand.RequestType getStatus) {
+		if (statusTask != null) statusTask.cancel();
+
+		statusTask = new StatusTask(this, getStatus);
+		timer.schedule(statusTask, STATUS_POLL_INTERVAL);
+	}
+
+	public void scheduleTimeoutTask(long delay) {
+		if (timeoutTask != null) timeoutTask.cancel();
+
+		timeoutTask = new TimeoutTask(this);
+		timer.schedule(timeoutTask, delay);
+		if (s_logger.isDebugEnabled()) {
+			log("Scheduling timeout at " + delay + " ms", Level.DEBUG);
+		}
+	}
+	
+	public void updateDatabase(Status state, String uploadErrorString) {
+		
+		VMTemplateHostVO vo = vmTemplateHostDao.createForUpdate();
+		vo.setUploadState(state);
+		vo.setLastUpdated(new Date());
+		vo.setUpload_errorString(uploadErrorString);
+		vmTemplateHostDao.update(getTemplateHostId(), vo);
+	}
+	
+	public void updateDatabase(Status state, String uploadUrl,String uploadErrorString) {
+		
+		VMTemplateHostVO vo = vmTemplateHostDao.createForUpdate();
+		vo.setUploadState(state);
+		vo.setLastUpdated(new Date());
+		vo.setUploadUrl(uploadUrl);
+		vo.setUploadJobId(null);
+		vo.setUploadPercent(0);
+		vo.setUpload_errorString(uploadErrorString);
+		
+		vmTemplateHostDao.update(getTemplateHostId(), vo);
+	}
+	
+	private Long getTemplateHostId() {
+		if (templateHostId == null){
+			VMTemplateHostVO templHost = vmTemplateHostDao.findByHostTemplate(sserver.getId(), template.getId());
+			templateHostId = templHost.getId();
+		}
+		return templateHostId;
+	}
+
+	public synchronized void updateDatabase(UploadAnswer answer) {		
+		
+        VMTemplateHostVO updateBuilder = vmTemplateHostDao.createForUpdate();
+		updateBuilder.setUploadPercent(answer.getUploadPct());
+		updateBuilder.setUploadState(answer.getUploadStatus());
+		updateBuilder.setLastUpdated(new Date());
+		updateBuilder.setUpload_errorString(answer.getErrorString());
+		updateBuilder.setUploadJobId(answer.getJobId());
+		
+		vmTemplateHostDao.update(getTemplateHostId(), updateBuilder);
+	}
+
+	public void sendCommand(RequestType reqType) {
+		if (getJobId() != null) {
+			if (s_logger.isTraceEnabled()) {
+				log("Sending progress command ", Level.TRACE);
+			}
+			long sent = uploadMonitor.send(sserver.getId(), new UploadProgressCommand(getCommand(), getJobId(), reqType), this);
+			if (sent == -1) {
+				setDisconnected();
+			}
+		}
+		
+	}
+	
+	private UploadCommand getCommand() {
+		return cmd;
+	}
+
+	public void logDisconnect() {
+		s_logger.warn("Unable to monitor upload progress of " + template.getName() + " at host " + sserver.getName());
+		uploadMonitor.logEvent(template.getAccountId(), EventTypes.EVENT_TEMPLATE_UPLOAD_FAILED, "Storage server " + sserver.getName() + " disconnected during upload of template " + template.getName(), EventVO.LEVEL_WARN);
+	}
+	
+	public void scheduleImmediateStatusCheck(RequestType request) {
+		if (statusTask != null) statusTask.cancel();
+		statusTask = new StatusTask(this, request);
+		timer.schedule(statusTask, SMALL_DELAY);
+	}
+
+	public void setCurrState(Status uploadState) {
+		this.currState = getState(currState.toString());		
+	}
+	
+}

server/src/com/cloud/storage/upload/UploadMonitor.java

@@ -0,0 +1,43 @@
+/**
+ *  Copyright (C) 2010 Cloud.com, Inc.  All rights reserved.
+ * 
+ * This software is licensed under the GNU General Public License v3 or later.
+ * 
+ * It is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ */
+
+package com.cloud.storage.upload;
+
+import java.util.Map;
+
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.template.TemplateInfo;
+import com.cloud.utils.component.Manager;
+
+/**
+ * Monitor upload progress of all templates.
+ * @author nitin
+ *
+ */
+public interface UploadMonitor extends Manager{		
+	
+	public void cancelAllUploads(Long templateId);
+
+	public void extractTemplate(VMTemplateVO template, String url,
+			VMTemplateHostVO tmpltHostRef,Long dataCenterId);
+
+	void handleUploadTemplateSync(long sserverId,
+			Map<String, TemplateInfo> templateInfo);
+
+}

server/src/com/cloud/storage/upload/UploadMonitorImpl.java

@@ -0,0 +1,314 @@
+package com.cloud.storage.upload;
+
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Timer;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.ejb.Local;
+import javax.naming.ConfigurationException;
+
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.AgentManager;
+import com.cloud.agent.Listener;
+import com.cloud.agent.api.Command;
+import com.cloud.agent.api.storage.UploadCommand;
+import com.cloud.agent.api.storage.UploadProgressCommand;
+import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
+import com.cloud.configuration.dao.ConfigurationDao;
+import com.cloud.dc.dao.DataCenterDao;
+import com.cloud.event.EventTypes;
+import com.cloud.event.EventVO;
+import com.cloud.event.dao.EventDao;
+import com.cloud.host.Host;
+import com.cloud.host.HostVO;
+import com.cloud.host.dao.HostDao;
+import com.cloud.storage.StoragePoolHostVO;
+import com.cloud.storage.VMTemplateHostVO;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
+import com.cloud.storage.dao.StoragePoolHostDao;
+import com.cloud.storage.dao.VMTemplateDao;
+import com.cloud.storage.dao.VMTemplateHostDao;
+import com.cloud.storage.dao.VMTemplatePoolDao;
+import com.cloud.storage.template.TemplateInfo;
+import com.cloud.utils.component.Inject;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.vm.dao.SecondaryStorageVmDao;
+
+/**
+ * @author nitin
+ *
+ */
+@Local(value={UploadMonitor.class})
+public class UploadMonitorImpl implements UploadMonitor {
+
+	static final Logger s_logger = Logger.getLogger(UploadMonitorImpl.class);
+	
+	private String _hyperVisorType;
+    @Inject 
+    VMTemplateHostDao _vmTemplateHostDao;
+    @Inject
+	VMTemplatePoolDao _vmTemplatePoolDao;
+    @Inject
+    StoragePoolHostDao _poolHostDao;
+    @Inject
+    SecondaryStorageVmDao _secStorageVmDao;
+
+    
+    @Inject
+    HostDao _serverDao = null;
+    @Inject
+    private final DataCenterDao _dcDao = null;
+    @Inject
+    VMTemplateDao _templateDao =  null;
+    @Inject
+	private final EventDao _eventDao = null;
+    @Inject
+	private AgentManager _agentMgr;
+    @Inject
+    ConfigurationDao _configDao;
+
+	private String _name;
+	private Boolean _sslCopy = new Boolean(false);
+	private String _copyAuthPasswd;
+
+
+	Timer _timer;
+
+	final Map<VMTemplateHostVO, UploadListener> _listenerMap = new ConcurrentHashMap<VMTemplateHostVO, UploadListener>();
+
+	
+	@Override
+	public void cancelAllUploads(Long templateId) {
+		// TODO Auto-generated method stub
+
+	}
+	public boolean isTemplateUploadInProgress(Long templateId) {
+		List<VMTemplateHostVO> uploadsInProgress =
+			_vmTemplateHostDao.listByTemplateStatus(templateId, VMTemplateHostVO.Status.UPLOAD_IN_PROGRESS);
+		return (uploadsInProgress.size() != 0);
+	}
+
+	@Override
+	public void extractTemplate( VMTemplateVO template, String url,
+			VMTemplateHostVO vmTemplateHost,Long dataCenterId){
+
+		if (isTemplateUploadInProgress(template.getId()) ){
+			return;
+		}		
+		
+		List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
+		HostVO sserver = storageServers.get(0);			
+		
+		_vmTemplateHostDao.updateUploadStatus(sserver.getId(), template.getId(), 0, VMTemplateStorageResourceAssoc.Status.NOT_UPLOADED, "jobid0000", url);                
+        		
+		if(vmTemplateHost != null) {
+		    start();
+			UploadCommand ucmd = new UploadCommand(template, url, vmTemplateHost);	
+			UploadListener ul = new UploadListener(sserver, template, _timer, _vmTemplateHostDao, vmTemplateHost.getId(), this, ucmd);
+			_listenerMap.put(vmTemplateHost, ul);
+
+			long result = send(sserver.getId(), ucmd, ul);	
+			if (result == -1) {
+				s_logger.warn("Unable to start upload of template " + template.getUniqueName() + " from " + sserver.getName() + " to " +url);
+				ul.setDisconnected();
+				ul.scheduleStatusCheck(RequestType.GET_OR_RESTART);
+			}
+		}
+		
+	}
+
+
+	public long send(Long hostId, Command cmd, Listener listener) {
+		return _agentMgr.gatherStats(hostId, cmd, listener);
+	}
+
+	@Override
+	public boolean configure(String name, Map<String, Object> params)
+			throws ConfigurationException {
+		_name = name;
+        final Map<String, String> configs = _configDao.getConfiguration("ManagementServer", params);
+        _sslCopy = Boolean.parseBoolean(configs.get("secstorage.encrypt.copy"));
+        
+        String cert = configs.get("secstorage.secure.copy.cert");
+        if ("realhostip.com".equalsIgnoreCase(cert)) {
+        	s_logger.warn("Only realhostip.com ssl cert is supported, ignoring self-signed and other certs");
+        }
+        
+        _hyperVisorType = _configDao.getValue("hypervisor.type");
+        
+        _copyAuthPasswd = configs.get("secstorage.copy.password");
+        
+        _agentMgr.registerForHostEvents(new UploadListener(this), true, false, false);
+		return true;
+	}
+
+	@Override
+	public String getName() {
+		// TODO Auto-generated method stub
+		return _name;
+	}
+
+	@Override
+	public boolean start() {
+		_timer = new Timer();
+		return true;
+	}
+
+	@Override
+	public boolean stop() {		
+		return true;
+	}
+	
+	public void handleUploadEvent(HostVO host, VMTemplateVO template, Status upldStatus) {
+		
+		if ((upldStatus == VMTemplateStorageResourceAssoc.Status.UPLOADED) || (upldStatus==Status.ABANDONED)){
+			VMTemplateHostVO vmTemplateHost = new VMTemplateHostVO(host.getId(), template.getId());
+			UploadListener oldListener = _listenerMap.get(vmTemplateHost);
+			if (oldListener != null) {
+				_listenerMap.remove(vmTemplateHost);
+			}
+		}
+		if (upldStatus == VMTemplateStorageResourceAssoc.Status.UPLOADED) {
+			logEvent(template.getAccountId(), EventTypes.EVENT_TEMPLATE_UPLOAD_SUCCESS, template.getName() + " successfully uploaded from storage server " + host.getName(), EventVO.LEVEL_INFO);
+		}
+		if (upldStatus == Status.UPLOAD_ERROR) {
+			logEvent(template.getAccountId(), EventTypes.EVENT_TEMPLATE_UPLOAD_FAILED, template.getName() + " failed to upload from storage server " + host.getName(), EventVO.LEVEL_ERROR);
+		}
+		if (upldStatus == Status.ABANDONED) {
+			logEvent(template.getAccountId(), EventTypes.EVENT_TEMPLATE_UPLOAD_FAILED, template.getName() + " :aborted upload from storage server " + host.getName(), EventVO.LEVEL_WARN);
+		}
+		
+		/*VMTemplateHostVO vmTemplateHost = _vmTemplateHostDao.findByHostTemplate(host.getId(), template.getId());
+		
+        if (upldStatus == Status.UPLOADED) {
+            long size = -1;
+            if(vmTemplateHost!=null){
+            	size = vmTemplateHost.getSize();
+            }
+            else{
+            	s_logger.warn("Failed to get size for template" + template.getName());
+            }
+			String eventParams = "id=" + template.getId() + "\ndcId="+host.getDataCenterId()+"\nsize="+size;
+            EventVO event = new EventVO();
+            event.setUserId(1L);
+            event.setAccountId(template.getAccountId());
+            if((template.getFormat()).equals(ImageFormat.ISO)){
+            	event.setType(EventTypes.EVENT_ISO_CREATE);
+            	event.setDescription("Successfully uploaded ISO " + template.getName());
+            }
+            else{
+            	event.setType(EventTypes.EVENT_TEMPLATE_);
+            	event.setDescription("Successfully uploaded template " + template.getName());
+            }
+            event.setParameters(eventParams);
+            event.setLevel(EventVO.LEVEL_INFO);
+            _eventDao.persist(event);
+        } 
+        
+		if (vmTemplateHost != null) {
+			Long poolId = vmTemplateHost.getPoolId();
+			if (poolId != null) {
+				VMTemplateStoragePoolVO vmTemplatePool = _vmTemplatePoolDao.findByPoolTemplate(poolId, template.getId());
+				StoragePoolHostVO poolHost = _poolHostDao.findByPoolHost(poolId, host.getId());
+				if (vmTemplatePool != null && poolHost != null) {
+					vmTemplatePool.setDownloadPercent(vmTemplateHost.getUploadPercent());
+					vmTemplatePool.setDownloadState(vmTemplateHost.getUploadState());
+					vmTemplatePool.setErrorString(vmTemplateHost.getUpload_errorString());
+					String localPath = poolHost.getLocalPath();
+					String installPath = vmTemplateHost.getInstallPath();
+					if (installPath != null) {
+						if (!installPath.startsWith("/")) {
+							installPath = "/" + installPath;
+						}
+						if (!(localPath == null) && !installPath.startsWith(localPath)) {
+							localPath = localPath.replaceAll("/\\p{Alnum}+/*$", ""); //remove instance if necessary
+						}
+						if (!(localPath == null) && installPath.startsWith(localPath)) {
+							installPath = installPath.substring(localPath.length());
+						}
+					}
+					vmTemplatePool.setInstallPath(installPath);
+					vmTemplatePool.setLastUpdated(vmTemplateHost.getLastUpdated());
+					vmTemplatePool.setJobId(vmTemplateHost.getJobId());
+					vmTemplatePool.setLocalDownloadPath(vmTemplateHost.getLocalDownloadPath());
+					_vmTemplatePoolDao.update(vmTemplatePool.getId(),vmTemplatePool);
+				}
+			}
+		}*/
+
+	}
+	
+	public void logEvent(long accountId, String evtType, String description, String level) {
+		EventVO event = new EventVO();
+		event.setUserId(1);
+		event.setAccountId(accountId);
+		event.setType(evtType);
+		event.setDescription(description);
+		event.setLevel(level);
+		_eventDao.persist(event);
+		
+	}
+
+	@Override
+	public void handleUploadTemplateSync(long sserverId, Map<String, TemplateInfo> templateInfo) {
+		HostVO storageHost = _serverDao.findById(sserverId);
+		if (storageHost == null) {
+			s_logger.warn("Huh? Agent id " + sserverId + " does not correspond to a row in hosts table?");
+			return;
+		}		
+		
+		List<VMTemplateVO> allTemplates = _templateDao.listAllInZone(storageHost.getDataCenterId());
+		VMTemplateVO rtngTmplt = _templateDao.findRoutingTemplate();
+		VMTemplateVO defaultBuiltin = _templateDao.findDefaultBuiltinTemplate();
+
+		if (rtngTmplt != null && !allTemplates.contains(rtngTmplt))
+			allTemplates.add(rtngTmplt);
+
+		if (defaultBuiltin != null && !allTemplates.contains(defaultBuiltin)) {
+			allTemplates.add(defaultBuiltin);
+		}			
+		        
+        
+		for (VMTemplateVO tmplt: allTemplates) {
+			String uniqueName = tmplt.getUniqueName();
+			VMTemplateHostVO tmpltHost = _vmTemplateHostDao.findByHostTemplate(sserverId, tmplt.getId());
+			if (templateInfo.containsKey(uniqueName)) {		
+				if (tmpltHost != null) {
+					s_logger.info("Template Sync found " + uniqueName + " already in the template host table");
+                    if (tmpltHost.getUploadState() != Status.UPLOADED) {
+                    	tmpltHost.setUpload_errorString("");
+                    }
+                    tmpltHost.setUploadPercent(100);
+                    tmpltHost.setUploadState(Status.UPLOADED);                    
+                    tmpltHost.setLastUpdated(new Date());
+					_vmTemplateHostDao.update(tmpltHost.getId(), tmpltHost);
+				} else {
+					VMTemplateHostVO templtHost = new VMTemplateHostVO(sserverId, tmplt.getId(), new Date(), 100, Status.UPLOADED, null, null, null, templateInfo.get(uniqueName).getInstallPath(), tmplt.getUrl());
+					templtHost.setSize(templateInfo.get(uniqueName).getSize());
+					_vmTemplateHostDao.persist(templtHost);
+				}
+				templateInfo.remove(uniqueName);
+				continue;
+			}
+			/*if (tmpltHost != null && tmpltHost.getUploadState() != Status.UPLOADED) {
+				s_logger.info("Template Sync did not find " + uniqueName + " ready on server " + sserverId + ", will request upload to start/resume shortly");
+
+			} else if (tmpltHost == null) {
+				s_logger.info("Template Sync did not find " + uniqueName + " on the server " + sserverId + ", will request upload shortly");
+				VMTemplateHostVO templtHost = new VMTemplateHostVO(sserverId, tmplt.getId(), new Date(), 0, Status.NOT_UPLOADED, null, null, null, null, tmplt.getUrl());
+				_vmTemplateHostDao.persist(templtHost);
+			}*/
+
+		}				
+	}	
+}

server/src/com/cloud/storage/upload/UploadState.java

@@ -0,0 +1,70 @@
+package com.cloud.storage.upload;
+
+import java.util.Date;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.cloud.agent.api.storage.UploadAnswer;
+import com.cloud.storage.upload.UploadState.UploadEvent;
+
+public abstract class UploadState {
+
+	public static enum UploadEvent {UPLOAD_ANSWER, ABANDON_UPLOAD, TIMEOUT_CHECK, DISCONNECT};
+	protected static final Logger s_logger = Logger.getLogger(UploadListener.class.getName());
+
+	private UploadListener ul;
+	
+	public UploadState(UploadListener ul) {
+		this.ul = ul;
+	}
+	
+	protected UploadListener getUploadListener() {
+		return ul;
+	}
+	
+	public String handleEvent(UploadEvent event, Object eventObj){
+		if (s_logger.isTraceEnabled()) {
+			getUploadListener().log("handleEvent, event type=" + event + ", curr state=" + getName(), Level.TRACE);
+		}
+		switch (event) {
+		case UPLOAD_ANSWER:
+			UploadAnswer answer=(UploadAnswer)eventObj;
+			return handleAnswer(answer);
+		case ABANDON_UPLOAD:
+			return handleAbort();
+		case TIMEOUT_CHECK:
+			Date now = new Date();
+			long update = now.getTime() - ul.getLastUpdated().getTime();
+			return handleTimeout(update);
+		case DISCONNECT:
+			return handleDisconnect();
+		}
+		return null;
+	}
+	
+	public   void onEntry(String prevState, UploadEvent event, Object evtObj){
+		if (s_logger.isTraceEnabled()) {
+			getUploadListener().log("onEntry, event type=" + event + ", curr state=" + getName(), Level.TRACE);
+		}
+		if (event == UploadEvent.UPLOAD_ANSWER) {
+			getUploadListener().updateDatabase((UploadAnswer)evtObj);
+		}
+	}
+	
+	public  void onExit() {
+		
+	}
+	
+	public abstract String handleTimeout(long updateMs) ;
+	
+	public abstract String handleAbort();
+	
+	public abstract  String handleDisconnect();
+
+	public abstract String handleAnswer(UploadAnswer answer) ;
+	
+	public abstract String getName();
+
+
+}

server/src/com/cloud/template/TemplateManager.java

@@ -127,5 +127,7 @@ public interface TemplateManager extends Manager {
     void evictTemplateFromStoragePool(VMTemplateStoragePoolVO templatePoolVO);
     
     boolean templateIsDeleteable(VMTemplateHostVO templateHostRef);
+
+	void extract(VMTemplateVO template, String url, VMTemplateHostVO tmpltHostRef, Long zoneId);
     
 }

server/src/com/cloud/template/TemplateManagerImpl.java

@@ -66,6 +66,7 @@ import com.cloud.storage.dao.VMTemplatePoolDao;
 import com.cloud.storage.dao.VMTemplateZoneDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.storage.download.DownloadMonitor;
+import com.cloud.storage.upload.UploadMonitor;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
 import com.cloud.user.AccountVO;
@@ -98,6 +99,7 @@ public class TemplateManagerImpl implements TemplateManager {
     @Inject StoragePoolHostDao _poolHostDao;
     @Inject EventDao _eventDao;
     @Inject DownloadMonitor _downloadMonitor;
+    @Inject UploadMonitor _uploadMonitor;
     @Inject UserAccountDao _userAccountDao;
     @Inject AccountDao _accountDao;
     @Inject UserDao _userDao;
@@ -109,7 +111,7 @@ public class TemplateManagerImpl implements TemplateManager {
     @Inject SnapshotDao _snapshotDao;
     long _routerTemplateId = -1;
     @Inject StorageManager _storageMgr;
-    protected SearchBuilder<VMTemplateHostVO> HostTemplateStatesSearch;
+    protected SearchBuilder<VMTemplateHostVO> HostTemplateStatesSearch;	
     
 
     @Override
@@ -145,6 +147,11 @@ public class TemplateManagerImpl implements TemplateManager {
         return id;
     }
     
+    @Override
+    public void extract(VMTemplateVO template, String url, VMTemplateHostVO tmpltHostRef, Long zoneId){
+    	_uploadMonitor.extractTemplate(template, url, tmpltHostRef, zoneId);
+    }
+    
     @Override @DB
     public VMTemplateStoragePoolVO prepareTemplateForCreate(VMTemplateVO template, StoragePoolVO pool) {
     	template = _tmpltDao.findById(template.getId(), true);

server/test/async-job-component.xml

@@ -150,6 +150,8 @@
         </manager>
         <manager name="download manager" class="com.vmops.storage.download.DownloadMonitorImpl">
         </manager>
+        <manager name="upload manager" class="com.cloud.storage.upload.UploadMonitorImpl">
+        </manager>
         <manager name="console proxy manager" class="com.vmops.consoleproxy.ConsoleProxyManagerImpl">
         </manager>
         <manager name="vm manager" class="com.vmops.vm.UserVmManagerImpl"/>

setup/db/create-schema.sql

@@ -623,13 +623,18 @@ CREATE TABLE  `cloud`.`template_host_ref` (
   `created` DATETIME NOT NULL,
   `last_updated` DATETIME,
   `job_id` varchar(255),
+  `upload_job_id` varchar(255),
   `download_pct` int(10) unsigned,
+  `upload_pct` int(10) unsigned,
   `size` bigint unsigned,
   `download_state` varchar(255),
+  `upload_state` varchar(255),
   `error_str` varchar(255),
+  `upload_error_str` varchar(255),
   `local_path` varchar(255),
   `install_path` varchar(255),
   `url` varchar(255),
+  `upload_url` varchar(255),
   `destroyed` tinyint(1) COMMENT 'indicates whether the template_host entry was destroyed by the user or not',
   `is_copy` tinyint(1) NOT NULL DEFAULT 0 COMMENT 'indicates whether this was copied ',
   PRIMARY KEY  (`id`)

setup/db/server-setup.xml

@@ -12,14 +12,14 @@
   <zones>
     <zone>
       <id>1</id>
-      <name>ZONE1</name>
-      <dns1>72.52.126.11</dns1>
-      <dns2>72.52.126.12</dns2>
-      <internalDns1>192.168.10.253</internalDns1>
-      <internalDns2>192.168.10.254</internalDns2>
-      <gateway>172.16.0.1</gateway>
-      <netmask>255.255.0.0</netmask>
-      <ipAddressRange>172.16.1.1-172.16.255.253</ipAddressRange>
+      <name>NM</name>
+      <dns1>4.2.2.2</dns1>
+      <dns2>10.10.10.14</dns2>
+      <internalDns1>4.2.2.2</internalDns1>
+      <internalDns2>4.2.2.2</internalDns2>
+	  <gateway>10.91.28.1</gateway>
+	  <netmask>255.255.255.0</netmask>
+	  <vnet>560-579</vnet>      
       <guestNetworkCidr>10.1.1.0/24</guestNetworkCidr>
     </zone>
   </zones>
@@ -35,24 +35,45 @@
   <pods>
     <pod>
       <id>1</id>
-      <name>POD1</name>
+      <name>NM</name>
       <zoneId>1</zoneId> 
-      <gateway>192.168.2.1</gateway>
-      <cidr>192.168.2.0/24</cidr>
-      <ipAddressRange>192.168.2.20-192.168.2.170</ipAddressRange>
+      <gateway>10.91.28.1</gateway>
+      <cidr>10.91.28.0/24</cidr>
+      <ipAddressRange>10.91.28.160-10.91.28.179</ipAddressRange>
     </pod>
   </pods>
 
-<!--
+<storagePools>
+<storagePool>
+<zoneId>1</zoneId>
+<podId>1</podId>
+<name>idc-ss</name>
+<hostAddress>10.91.28.6</hostAddress>
+<hostPath>/export/home/nitin/primary</hostPath>
+</storagePool>
+</storagePools>
+
+<!--  
+<secondaryStorages>
+<secondaryStorage>
+<zoneId>1</zoneId>
+<podId>1</podId>
+<url>nfs://10.91.28.6/export/home/nitin/secondary</url>
+</secondaryStorage>
+</secondaryStorages>
+-->
+
   <vlans>
     <vlan>
       <zoneId>1</zoneId>
       <vlanId>30</vlanId>
       <vlanType>VirtualNetwork</vlanType>
-      <gateway>192.168.30.1</gateway>
+      <gateway>10.91.30.1</gateway>
       <netmask>255.255.255.0</netmask>
-      <ipAddressRange>192.168.30.10-192.168.30.19</ipAddressRange>
+      <ipAddressRange>10.91.30.160-10.91.30.179</ipAddressRange>
     </vlan>
+  </vlans>
+<!--    
     <vlan>
       <zoneId>1</zoneId>
       <vlanId>31</vlanId>
@@ -61,7 +82,6 @@
       <netmask>255.255.255.0</netmask>
       <ipAddressRange>192.168.31.10-192.168.31.19</ipAddressRange>
     </vlan>
-  </vlans>
   -->
   <!--
     * id is the unique id of the service offering