From 6723549fc7b1dc6b3b84dc5e5e87f026f3e121b3 Mon Sep 17 00:00:00 2001 From: nit Date: Wed, 20 Oct 2010 11:40:03 +0530 Subject: [PATCH] bug 6399: extending extract functionlity for templates and ISO's to allow download. The extraction can have two modes FTP_UPLOAD and HTTP_DOWNLOAD. In the former one the user would provide the ftp url where the entity needs to be uploaded and in the later the user would be provided a HTTP URL where from he/she can download the entity. This url would be exposed for a specific time limit and would not function after the time limit. --- .../CreateEntityDownloadURLAnswer.java | 21 +++ .../CreateEntityDownloadURLCommand.java | 30 ++++ .../DeleteEntityDownloadURLAnswer.java | 21 +++ .../DeleteEntityDownloadURLCommand.java | 25 +++ core/src/com/cloud/storage/Upload.java | 5 +- core/src/com/cloud/storage/UploadVO.java | 27 +++- core/src/com/cloud/storage/dao/UploadDao.java | 3 + .../com/cloud/storage/dao/UploadDaoImpl.java | 16 ++ .../resource/NfsSecondaryStorageResource.java | 9 +- .../cloud/storage/template/UploadManager.java | 9 ++ .../storage/template/UploadManagerImpl.java | 88 +++++++--- server/src/com/cloud/api/ApiDBUtils.java | 10 +- server/src/com/cloud/api/BaseCmd.java | 2 +- .../com/cloud/api/commands/ExtractIsoCmd.java | 28 +++- .../api/commands/ExtractTemplateCmd.java | 37 +++-- .../cloud/api/response/ExtractResponse.java | 59 +++++-- .../executor/ExtractJobResultObject.java | 84 ++++++---- .../executor/ExtractTemplateExecutor.java | 3 +- .../async/executor/ExtractTemplateParam.java | 25 ++- .../cloud/server/ManagementServerImpl.java | 7 +- .../cloud/storage/upload/UploadListener.java | 4 +- .../cloud/storage/upload/UploadMonitor.java | 4 + .../storage/upload/UploadMonitorImpl.java | 152 +++++++++++++++++- .../com/cloud/template/TemplateManager.java | 4 +- .../cloud/template/TemplateManagerImpl.java | 126 +++++++++------ setup/db/create-schema.sql | 1 + 26 files changed, 651 insertions(+), 149 deletions(-) create mode 100644 core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java create mode 100644 core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java create mode 100644 core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLAnswer.java create mode 100644 core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLCommand.java mode change 100644 => 100755 core/src/com/cloud/storage/template/UploadManager.java mode change 100644 => 100755 server/src/com/cloud/api/ApiDBUtils.java mode change 100644 => 100755 server/src/com/cloud/api/response/ExtractResponse.java mode change 100644 => 100755 server/src/com/cloud/async/executor/ExtractTemplateExecutor.java diff --git a/core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java b/core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java new file mode 100644 index 00000000000..e2b22065121 --- /dev/null +++ b/core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java @@ -0,0 +1,21 @@ +package com.cloud.agent.api.storage; + +import com.cloud.agent.api.Answer; + +public class CreateEntityDownloadURLAnswer extends Answer{ + + String resultString; + short resultCode; + public static final short RESULT_SUCCESS = 1; + public static final short RESULT_FAILURE = 0; + + public CreateEntityDownloadURLAnswer(String resultString, short resultCode) { + super(); + this.resultString = resultString; + this.resultCode = resultCode; + } + + public CreateEntityDownloadURLAnswer(){ + } + +} diff --git a/core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java b/core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java new file mode 100644 index 00000000000..da77cea9c4e --- /dev/null +++ b/core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java @@ -0,0 +1,30 @@ +package com.cloud.agent.api.storage; + +import com.cloud.agent.api.Command; + +public class CreateEntityDownloadURLCommand extends AbstractDownloadCommand { + + public CreateEntityDownloadURLCommand(String installPath) { + super(); + this.installPath = installPath; + } + + public CreateEntityDownloadURLCommand() { + } + + private String installPath; + + @Override + public boolean executeInSequence() { + return false; + } + + public String getInstallPath() { + return installPath; + } + + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + +} diff --git a/core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLAnswer.java b/core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLAnswer.java new file mode 100644 index 00000000000..d3022c85426 --- /dev/null +++ b/core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLAnswer.java @@ -0,0 +1,21 @@ +package com.cloud.agent.api.storage; + +import com.cloud.agent.api.Answer; + +public class DeleteEntityDownloadURLAnswer extends Answer{ + + String resultString; + short resultCode; + public static final short RESULT_SUCCESS = 1; + public static final short RESULT_FAILURE = 0; + + public DeleteEntityDownloadURLAnswer(String resultString, short resultCode) { + super(); + this.resultString = resultString; + this.resultCode = resultCode; + } + + public DeleteEntityDownloadURLAnswer(){ + } + +} diff --git a/core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLCommand.java b/core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLCommand.java new file mode 100644 index 00000000000..19f916278c7 --- /dev/null +++ b/core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLCommand.java @@ -0,0 +1,25 @@ +package com.cloud.agent.api.storage; + +public class DeleteEntityDownloadURLCommand extends AbstractDownloadCommand { + + String path; + + public DeleteEntityDownloadURLCommand(String path) { + super(); + this.path = path; + } + + public DeleteEntityDownloadURLCommand() { + super(); + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + +} diff --git a/core/src/com/cloud/storage/Upload.java b/core/src/com/cloud/storage/Upload.java index b6b54f7ce90..8b902db7a65 100755 --- a/core/src/com/cloud/storage/Upload.java +++ b/core/src/com/cloud/storage/Upload.java @@ -2,6 +2,7 @@ package com.cloud.storage; public interface Upload { - public static enum Status {UNKNOWN, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, NOT_COPIED, COPY_IN_PROGRESS, COPY_ERROR, COPY_COMPLETE} - public static enum Type {VOLUME, TEMPLATE, ISO} + public static enum Status {UNKNOWN, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, NOT_COPIED, COPY_IN_PROGRESS, COPY_ERROR, COPY_COMPLETE, DOWNLOAD_URL_CREATED, DOWNLOAD_URL_NOT_CREATED, ERROR} + public static enum Type {VOLUME, TEMPLATE, ISO} + public static enum Mode {FTP_UPLOAD, HTTP_DOWNLOAD} } diff --git a/core/src/com/cloud/storage/UploadVO.java b/core/src/com/cloud/storage/UploadVO.java index 5f1e6adf3d6..80164708cc7 100755 --- a/core/src/com/cloud/storage/UploadVO.java +++ b/core/src/com/cloud/storage/UploadVO.java @@ -65,6 +65,10 @@ public class UploadVO implements Upload { @Enumerated(EnumType.STRING) private Type type; + @Column (name="mode") + @Enumerated(EnumType.STRING) + private Mode mode = Mode.FTP_UPLOAD; + @Column (name="upload_state") @Enumerated(EnumType.STRING) private Status uploadState; @@ -122,6 +126,19 @@ public class UploadVO implements Upload { this.jobId = jobId; this.uploadUrl = uploadUrl; } + + public UploadVO(long hostId, long typeId, Date lastUpdated, + Status uploadState, int uploadPercent, Type type, + Mode mode) { + super(); + this.hostId = hostId; + this.typeId = typeId; + this.lastUpdated = lastUpdated; + this.uploadState = uploadState; + this.uploadPercent = uploadPercent; + this.type = type; + this.mode = mode; + } protected UploadVO() { } @@ -200,7 +217,15 @@ public class UploadVO implements Upload { this.type = type; } - public String getUploadUrl() { + public Mode getMode() { + return mode; + } + + public void setMode(Mode mode) { + this.mode = mode; + } + + public String getUploadUrl() { return uploadUrl; } diff --git a/core/src/com/cloud/storage/dao/UploadDao.java b/core/src/com/cloud/storage/dao/UploadDao.java index 95637029bd2..34885715e0d 100755 --- a/core/src/com/cloud/storage/dao/UploadDao.java +++ b/core/src/com/cloud/storage/dao/UploadDao.java @@ -5,6 +5,7 @@ import java.util.List; import com.cloud.storage.UploadVO; import com.cloud.storage.Upload.Status; import com.cloud.storage.Upload.Type; +import com.cloud.storage.Upload.Mode; import com.cloud.utils.db.GenericDao; public interface UploadDao extends GenericDao { @@ -13,5 +14,7 @@ public interface UploadDao extends GenericDao { Status uploadState); List listByHostAndUploadStatus(long sserverId, Status uploadInProgress); + + List listByModeAndStatus(Mode mode, Status uploadState); } diff --git a/core/src/com/cloud/storage/dao/UploadDaoImpl.java b/core/src/com/cloud/storage/dao/UploadDaoImpl.java index 9d4da07c223..e3938bb9da5 100755 --- a/core/src/com/cloud/storage/dao/UploadDaoImpl.java +++ b/core/src/com/cloud/storage/dao/UploadDaoImpl.java @@ -6,6 +6,7 @@ import org.apache.log4j.Logger; import com.cloud.storage.UploadVO; import com.cloud.storage.Upload.Status; +import com.cloud.storage.Upload.Mode; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -15,6 +16,7 @@ public class UploadDaoImpl extends GenericDaoBase implements Upl public static final Logger s_logger = Logger.getLogger(UploadDaoImpl.class.getName()); protected final SearchBuilder typeUploadStatusSearch; protected final SearchBuilder typeHostAndUploadStatusSearch; + protected final SearchBuilder typeModeAndStatusSearch; protected static final String UPDATE_UPLOAD_INFO = "UPDATE upload SET upload_state = ?, upload_pct= ?, last_updated = ? " @@ -37,6 +39,12 @@ public class UploadDaoImpl extends GenericDaoBase implements Upl typeHostAndUploadStatusSearch.and("host_id", typeHostAndUploadStatusSearch.entity().getHostId(), SearchCriteria.Op.EQ); typeHostAndUploadStatusSearch.and("upload_state", typeHostAndUploadStatusSearch.entity().getUploadState(), SearchCriteria.Op.EQ); typeHostAndUploadStatusSearch.done(); + + typeModeAndStatusSearch = createSearchBuilder(); + typeModeAndStatusSearch.and("mode", typeModeAndStatusSearch.entity().getMode(), SearchCriteria.Op.EQ); + typeModeAndStatusSearch.and("upload_state", typeModeAndStatusSearch.entity().getUploadState(), SearchCriteria.Op.EQ); + typeModeAndStatusSearch.done(); + } @Override @@ -55,4 +63,12 @@ public class UploadDaoImpl extends GenericDaoBase implements Upl sc.setParameters("upload_state", uploadState.toString()); return listBy(sc); } + + @Override + public List listByModeAndStatus(Mode mode, Status uploadState){ + SearchCriteria sc = typeModeAndStatusSearch.create(); + sc.setParameters("mode", mode.toString()); + sc.setParameters("upload_state", uploadState.toString()); + return listBy(sc); + } } \ No newline at end of file diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index a1f4f03bdb0..df1683b11b1 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -45,6 +45,9 @@ import com.cloud.agent.api.SecStorageSetupCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupStorageCommand; import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig; +import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer; +import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand; +import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand; import com.cloud.agent.api.storage.DeleteTemplateCommand; import com.cloud.agent.api.storage.DownloadCommand; import com.cloud.agent.api.storage.DownloadProgressCommand; @@ -117,7 +120,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return _dlMgr.handleDownloadCommand((DownloadCommand)cmd); }else if (cmd instanceof UploadCommand) { return _upldMgr.handleUploadCommand((UploadCommand)cmd); - } else if (cmd instanceof GetStorageStatsCommand) { + }else if (cmd instanceof CreateEntityDownloadURLCommand){ + return _upldMgr.handleCreateEntityURLCommand((CreateEntityDownloadURLCommand)cmd); + }else if(cmd instanceof DeleteEntityDownloadURLCommand){ + return _upldMgr.handleDeleteEntityDownloadURLCommand((DeleteEntityDownloadURLCommand)cmd); + }else if (cmd instanceof GetStorageStatsCommand) { return execute((GetStorageStatsCommand)cmd); } else if (cmd instanceof CheckHealthCommand) { return new CheckHealthAnswer((CheckHealthCommand)cmd, true); diff --git a/core/src/com/cloud/storage/template/UploadManager.java b/core/src/com/cloud/storage/template/UploadManager.java old mode 100644 new mode 100755 index df096235c85..e0c14efddc9 --- a/core/src/com/cloud/storage/template/UploadManager.java +++ b/core/src/com/cloud/storage/template/UploadManager.java @@ -1,5 +1,9 @@ package com.cloud.storage.template; +import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer; +import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand; +import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer; +import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand; import com.cloud.agent.api.storage.UploadAnswer; import com.cloud.agent.api.storage.UploadCommand; import com.cloud.storage.StorageResource; @@ -61,4 +65,9 @@ public interface UploadManager extends Manager { String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes); + + CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd); + + DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd); + } diff --git a/core/src/com/cloud/storage/template/UploadManagerImpl.java b/core/src/com/cloud/storage/template/UploadManagerImpl.java index b6c8ffd128d..87b51c8577a 100755 --- a/core/src/com/cloud/storage/template/UploadManagerImpl.java +++ b/core/src/com/cloud/storage/template/UploadManagerImpl.java @@ -1,7 +1,6 @@ package com.cloud.storage.template; import java.io.File; -import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.text.SimpleDateFormat; @@ -12,20 +11,22 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import javax.naming.ConfigurationException; import org.apache.log4j.Logger; +import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer; +import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand; +import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer; +import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand; import com.cloud.agent.api.storage.UploadAnswer; import com.cloud.agent.api.storage.UploadProgressCommand; import com.cloud.agent.api.storage.UploadCommand; -import com.cloud.agent.api.storage.UploadAnswer; -import com.cloud.agent.api.storage.UploadCommand; import com.cloud.storage.StorageLayer; import com.cloud.storage.StorageResource; import com.cloud.storage.UploadVO; -import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback; import com.cloud.storage.template.TemplateUploader.Status; @@ -33,6 +34,7 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.UUID; import com.cloud.utils.component.Adapters; import com.cloud.utils.component.ComponentLocator; +import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @@ -152,18 +154,19 @@ public class UploadManagerImpl implements UploadManager { return templatesize; } } - public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class); - private ExecutorService threadPool; - private final Map jobs = new ConcurrentHashMap(); - private String parentDir; - private Adapters _processors; - private String publicTemplateRepo; - private StorageLayer _storage; - private int installTimeoutPerGig; - private boolean _sslCopy; - private String _name; - private boolean hvm; - + public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class); + private ExecutorService threadPool; + private final Map jobs = new ConcurrentHashMap(); + private String parentDir; + private Adapters _processors; + private String publicTemplateRepo; + private StorageLayer _storage; + private int installTimeoutPerGig; + private boolean _sslCopy; + private String _name; + private boolean hvm; + + @Override public String uploadPublicTemplate(long id, String url, String name, ImageFormat format, Long accountId, String descr, @@ -320,6 +323,52 @@ public class UploadManagerImpl implements UploadManager { return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId), getUploadTemplateSize(jobId)); } + + @Override + public CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd){ + + // Create the directory structure so that its visible under apache server root + Script command = new Script("mkdir", s_logger); + command.add("-p"); + command.add("/var/www/html/"); + String result = command.execute(); + if (result != null) { + String errorString = "Error in creating directory =" + result; + s_logger.warn(errorString); + return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE); + } + + // Create a symbolic link from the actual directory to the template location + cmd.getInstallPath(); + command = new Script("/bin/bash", s_logger); + command.add("-c"); + command.add("ln -sf " + publicTemplateRepo + cmd.getInstallPath() + " /var/www/html/"); + result = command.execute(); + if (result != null) { + String errorString = "Error in linking err=" + result; + s_logger.warn(errorString); + return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE); + } + + return new CreateEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS); + + } + + @Override + public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd){ + + s_logger.debug("handleDeleteEntityDownloadURLCommand "+cmd.getPath()); + Script command = new Script("/bin/bash", s_logger); + command.add("-c"); + command.add("unlink /var/www/html/"+cmd.getPath()); + String result = command.execute(); + if (result != null) { + String errorString = "Error in deleting =" + result; + s_logger.warn(errorString); + return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE); + } + return new DeleteEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS); + } private String getInstallPath(String jobId) { // TODO Auto-generated method stub @@ -375,9 +424,9 @@ public class UploadManagerImpl implements UploadManager { configureFolders(name, params); String inSystemVM = (String)params.get("secondary.storage.vm"); if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) { - s_logger.info("UploadManager: starting additional services since we are inside system vm"); - startAdditionalServices(); - blockOutgoingOnPrivate(); + //s_logger.info("UploadManager: starting additional services since we are inside system vm"); + //startAdditionalServices(); + //blockOutgoingOnPrivate(); } value = (String) params.get("install.timeout.pergig"); @@ -407,6 +456,7 @@ public class UploadManagerImpl implements UploadManager { processors.add(processor); // Add more processors here. threadPool = Executors.newFixedThreadPool(numInstallThreads); + return true; } diff --git a/server/src/com/cloud/api/ApiDBUtils.java b/server/src/com/cloud/api/ApiDBUtils.java old mode 100644 new mode 100755 index 660727f1e48..175a9e3083b --- a/server/src/com/cloud/api/ApiDBUtils.java +++ b/server/src/com/cloud/api/ApiDBUtils.java @@ -47,6 +47,7 @@ import com.cloud.storage.SnapshotVO; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePoolVO; import com.cloud.storage.StorageStats; +import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; @@ -55,6 +56,7 @@ import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.StoragePoolDao; +import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VolumeDao; @@ -109,6 +111,7 @@ public class ApiDBUtils { private static StoragePoolDao _storagePoolDao; private static VMTemplateDao _templateDao; private static VMTemplateHostDao _templateHostDao; + private static UploadDao _uploadDao; private static UserDao _userDao; private static UserStatisticsDao _userStatsDao; private static UserVmDao _userVmDao; @@ -147,6 +150,7 @@ public class ApiDBUtils { _storagePoolDao = locator.getDao(StoragePoolDao.class); _templateDao = locator.getDao(VMTemplateDao.class); _templateHostDao = locator.getDao(VMTemplateHostDao.class); + _uploadDao = locator.getDao(UploadDao.class); _userDao = locator.getDao(UserDao.class); _userStatsDao = locator.getDao(UserStatisticsDao.class); _userVmDao = locator.getDao(UserVmDao.class); @@ -368,7 +372,11 @@ public class ApiDBUtils { return _templateHostDao.findByHostTemplate(secondaryStorageHost.getId(), templateId); } } - + + public static UploadVO findUploadById(Long id){ + return _uploadDao.findById(id); + } + public static User findUserById(Long userId) { return _userDao.findById(userId); } diff --git a/server/src/com/cloud/api/BaseCmd.java b/server/src/com/cloud/api/BaseCmd.java index fc771f9c0c1..39ece3c7e82 100755 --- a/server/src/com/cloud/api/BaseCmd.java +++ b/server/src/com/cloud/api/BaseCmd.java @@ -94,7 +94,7 @@ public abstract class BaseCmd { @Parameter(name="response", type=CommandType.STRING) private String responseType; - + public String getResponseType() { if (responseType == null) { return RESPONSE_TYPE_XML; diff --git a/server/src/com/cloud/api/commands/ExtractIsoCmd.java b/server/src/com/cloud/api/commands/ExtractIsoCmd.java index 791a1c45f3d..27eff84cbfc 100755 --- a/server/src/com/cloud/api/commands/ExtractIsoCmd.java +++ b/server/src/com/cloud/api/commands/ExtractIsoCmd.java @@ -18,6 +18,7 @@ package com.cloud.api.commands; import org.apache.log4j.Logger; +import org.junit.runners.Parameterized.Parameters; import com.cloud.api.ApiDBUtils; import com.cloud.api.BaseAsyncCmd; @@ -26,6 +27,7 @@ import com.cloud.api.Implementation; import com.cloud.api.Parameter; import com.cloud.api.response.ExtractResponse; import com.cloud.event.EventTypes; +import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; @@ -42,12 +44,14 @@ public class ExtractIsoCmd extends BaseAsyncCmd { @Parameter(name="id", type=CommandType.LONG, required=true, description="the ID of the ISO file") private Long id; - //FIXME - add description - @Parameter(name="url", type=CommandType.STRING, required=true) + @Parameter(name="url", type=CommandType.STRING, required=false, description="the url to which the ISO would be extracted") private String url; @Parameter(name="zoneid", type=CommandType.LONG, required=true, description="the ID of the zone where the ISO is originally located") private Long zoneId; + + @Parameter(name="mode", type=CommandType.STRING, required=true, description="the mode of extraction - HTTP_DOWNLOAD or FTP_UPLOAD") + private String mode; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -65,6 +69,9 @@ public class ExtractIsoCmd extends BaseAsyncCmd { return zoneId; } + public String getMode() { + return mode; + } ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -83,7 +90,7 @@ public class ExtractIsoCmd extends BaseAsyncCmd { public long getAccountId() { VMTemplateVO iso = ApiDBUtils.findTemplateById(getId()); if (iso != null) { - return iso.getId(); + return iso.getAccountId(); } // invalid id, parent this command to SYSTEM so ERROR events are tracked @@ -97,8 +104,21 @@ public class ExtractIsoCmd extends BaseAsyncCmd { @Override @SuppressWarnings("unchecked") public ExtractResponse getResponse() { - ExtractResponse response = (ExtractResponse)getResponseObject(); + Long uploadId = (Long)getResponseObject(); + UploadVO uploadInfo = ApiDBUtils.findUploadById(uploadId); + + ExtractResponse response = new ExtractResponse(); response.setResponseName(getName()); + response.setId(id); + response.setName(ApiDBUtils.findTemplateById(id).getName()); + response.setZoneId(zoneId); + response.setZoneName(ApiDBUtils.findZoneById(zoneId).getName()); + response.setMode(mode); + response.setUploadId(uploadId); + response.setState(uploadInfo.getUploadState().toString()); + response.setAccountId(getAccountId()); + //FIX ME - Need to set the url once the gson jar is upgraded since it is throwing an error right now due to a bug. + //response.setUrl(uploadInfo.getUploadUrl()); return response; } diff --git a/server/src/com/cloud/api/commands/ExtractTemplateCmd.java b/server/src/com/cloud/api/commands/ExtractTemplateCmd.java index 3bb7aca38ca..506cb3e9894 100755 --- a/server/src/com/cloud/api/commands/ExtractTemplateCmd.java +++ b/server/src/com/cloud/api/commands/ExtractTemplateCmd.java @@ -20,12 +20,14 @@ package com.cloud.api.commands; import org.apache.log4j.Logger; import com.cloud.api.BaseAsyncCmd; +import com.cloud.api.BaseCmd.CommandType; import com.cloud.api.BaseCmd.Manager; import com.cloud.api.ApiDBUtils; import com.cloud.api.Implementation; import com.cloud.api.Parameter; import com.cloud.api.response.ExtractResponse; import com.cloud.event.EventTypes; +import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateVO; import com.cloud.user.Account; @@ -39,18 +41,18 @@ public class ExtractTemplateCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - //FIXME - add description - @Parameter(name="id", type=CommandType.LONG, required=true) + @Parameter(name="id", type=CommandType.LONG, required=true, description="the ID of the template") private Long id; - //FIXME - add description - @Parameter(name="url", type=CommandType.STRING, required=true) + @Parameter(name="url", type=CommandType.STRING, required=false, description="the url to which the ISO would be extracted") private String url; - - //FIXME - add description - @Parameter(name="zoneid", type=CommandType.LONG, required=true) + + @Parameter(name="zoneid", type=CommandType.LONG, required=true, description="the ID of the zone where the ISO is originally located" ) private Long zoneId; + @Parameter(name="mode", type=CommandType.STRING, required=true, description="the mode of extraction - HTTP_DOWNLOAD or FTP_UPLOAD") + private String mode; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -67,6 +69,10 @@ public class ExtractTemplateCmd extends BaseAsyncCmd { return zoneId; } + public String getMode() { + return mode; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -84,7 +90,7 @@ public class ExtractTemplateCmd extends BaseAsyncCmd { public long getAccountId() { VMTemplateVO template = ApiDBUtils.findTemplateById(getId()); if (template != null) { - return template.getId(); + return template.getAccountId(); } // invalid id, parent this command to SYSTEM so ERROR events are tracked @@ -103,8 +109,21 @@ public class ExtractTemplateCmd extends BaseAsyncCmd { @Override @SuppressWarnings("unchecked") public ExtractResponse getResponse() { - ExtractResponse response = (ExtractResponse)getResponseObject(); + Long uploadId = (Long)getResponseObject(); + UploadVO uploadInfo = ApiDBUtils.findUploadById(uploadId); + + ExtractResponse response = new ExtractResponse(); response.setResponseName(getName()); + response.setId(id); + response.setName(ApiDBUtils.findTemplateById(id).getName()); + response.setZoneId(zoneId); + response.setZoneName(ApiDBUtils.findZoneById(zoneId).getName()); + response.setMode(mode); + response.setUploadId(uploadId); + response.setState(uploadInfo.getUploadState().toString()); + response.setAccountId(getAccountId()); + //FIX ME - Need to set the url once the gson jar is upgraded since it is throwing an error right now. + //response.setUrl(uploadInfo.getUploadUrl()); return response; } } diff --git a/server/src/com/cloud/api/response/ExtractResponse.java b/server/src/com/cloud/api/response/ExtractResponse.java old mode 100644 new mode 100755 index 602ac495025..d694931036c --- a/server/src/com/cloud/api/response/ExtractResponse.java +++ b/server/src/com/cloud/api/response/ExtractResponse.java @@ -24,21 +24,22 @@ import com.google.gson.annotations.SerializedName; public class ExtractResponse extends BaseResponse { @SerializedName("id") @Param(description="the id of extracted object") - private long id; + private Long id; @SerializedName("name") @Param(description="the name of the extracted object") private String name; - //FIXME - add description - @SerializedName("uploadpercentage") - private int uploadPercent; + @SerializedName("uploadId") @Param(description="the upload id of extracted object") + private Long uploadId; - //FIXME - add description - @SerializedName("uploadstatus") - private String uploadStatus; + @SerializedName("uploadpercentage") @Param(description="the percentage of the entity uploaded to the specified location") + private Integer uploadPercent; + + @SerializedName("status") @Param(description="the status of the ") + private String status; @SerializedName("accountid") @Param(description="the account id to which the extracted object belongs") - private long accountId; + private Long accountId; @SerializedName("resultstring") @Param(description="") private String resultString; @@ -63,7 +64,13 @@ public class ExtractResponse extends BaseResponse { @SerializedName("zonename") @Param(description="zone name the object was extracted from") private String zoneName; - public long getId() { + @SerializedName("extractMode") @Param(description="the mode of extraction - upload or download") + private String mode; + + @SerializedName("url") @Param(description="if mode = upload then url of the uploaded entity. if mode = download the url from which the entity can be downloaded") + private String url; + + public Long getId() { return id; } @@ -79,7 +86,15 @@ public class ExtractResponse extends BaseResponse { this.name = name; } - public int getUploadPercent() { + public Long getUploadId() { + return uploadId; + } + + public void setUploadId(Long uploadId) { + this.uploadId = uploadId; + } + + public Integer getUploadPercent() { return uploadPercent; } @@ -88,14 +103,14 @@ public class ExtractResponse extends BaseResponse { } public String getUploadStatus() { - return uploadStatus; + return status; } - public void setUploadStatus(String uploadStatus) { - this.uploadStatus = uploadStatus; + public void setUploadStatus(String Status) { + this.status = status; } - public long getAccountId() { + public Long getAccountId() { return accountId; } @@ -158,4 +173,20 @@ public class ExtractResponse extends BaseResponse { public void setZoneName(String zoneName) { this.zoneName = zoneName; } + + public String getMode() { + return mode; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } } diff --git a/server/src/com/cloud/async/executor/ExtractJobResultObject.java b/server/src/com/cloud/async/executor/ExtractJobResultObject.java index 61474ea8962..aa68e8ad475 100644 --- a/server/src/com/cloud/async/executor/ExtractJobResultObject.java +++ b/server/src/com/cloud/async/executor/ExtractJobResultObject.java @@ -9,14 +9,22 @@ import com.cloud.storage.upload.UploadState; public class ExtractJobResultObject { - public ExtractJobResultObject(Long accountId, String typeName, String currState, int i, Long uploadId){ + public ExtractJobResultObject(Long accountId, String typeName, String currState, int uploadPercent, Long uploadId){ this.accountId = accountId; this.name = typeName; this.state = currState; this.id = uploadId; - this.uploadPercent = i; + this.uploadPercent = uploadPercent; } + public ExtractJobResultObject(Long accountId, String typeName, String currState, Long uploadId, String url){ + this.accountId = accountId; + this.name = typeName; + this.state = currState; + this.id = uploadId; + this.url = url; + } + public ExtractJobResultObject(){ } @@ -38,30 +46,6 @@ public class ExtractJobResultObject { @Param(name="result_string") String result_string; - public int getUploadPercent() { - return uploadPercent; - } - - public void setUploadPercent(int i) { - this.uploadPercent = i; - } - - public String getUploadStatus() { - return uploadStatus; - } - - public void setUploadStatus(String uploadStatus) { - this.uploadStatus = uploadStatus; - } - - public String getResult_string() { - return result_string; - } - - public void setResult_string(String resultString) { - result_string = resultString; - } - @Param(name="created") private Date createdDate; @@ -77,6 +61,45 @@ public class ExtractJobResultObject { @Param(name="zoneid") private Long zoneId; + @Param(name="zonename") + private String zoneName; + + @Param(name="url") + private String url; + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public int getUploadPercent() { + return uploadPercent; + } + + public void setUploadPercent(int i) { + this.uploadPercent = i; + } + + public String getUploadStatus() { + return uploadStatus; + } + + public void setUploadStatus(String uploadStatus) { + this.uploadStatus = uploadStatus; + } + + public String getResult_string() { + return result_string; + } + + public void setResult_string(String resultString) { + result_string = resultString; + } + + public Long getZoneId() { return zoneId; } @@ -93,11 +116,6 @@ public class ExtractJobResultObject { this.zoneName = zoneName; } - @Param(name="zonename") - private String zoneName; - - private long size; - public String getStorage() { return storage; } @@ -121,10 +139,6 @@ public class ExtractJobResultObject { public String getName() { return name; } - - public void setSize(long size) { - this.size = size; - } public void setCreatedDate(Date createdDate) { this.createdDate = createdDate; diff --git a/server/src/com/cloud/async/executor/ExtractTemplateExecutor.java b/server/src/com/cloud/async/executor/ExtractTemplateExecutor.java old mode 100644 new mode 100755 index 141f905649f..6229002ac7f --- a/server/src/com/cloud/async/executor/ExtractTemplateExecutor.java +++ b/server/src/com/cloud/async/executor/ExtractTemplateExecutor.java @@ -25,8 +25,7 @@ public class ExtractTemplateExecutor extends BaseAsyncJobExecutor { /* try { - managementServer.extractTemplate(param.getUrl(), param.getTemplateId(), param.getZoneId(), param.getEventId(), getJob().getId()); - + managementServer.extractTemplate(param.getUrl(), param.getTemplateId(), param.getZoneId(), param.getEventId(), getJob().getId(), param.getExtractMode()); } catch (Exception e) { s_logger.warn("Unable to extract template: " + e.getMessage(), e); asyncMgr.completeAsyncJob(getJob().getId(), AsyncJobResult.STATUS_FAILED, BaseCmd.INTERNAL_ERROR, e.getMessage()); diff --git a/server/src/com/cloud/async/executor/ExtractTemplateParam.java b/server/src/com/cloud/async/executor/ExtractTemplateParam.java index b42242e39c8..8fb2dbf2284 100644 --- a/server/src/com/cloud/async/executor/ExtractTemplateParam.java +++ b/server/src/com/cloud/async/executor/ExtractTemplateParam.java @@ -1,5 +1,7 @@ package com.cloud.async.executor; +import com.cloud.storage.Upload; + public class ExtractTemplateParam { private long userId; @@ -7,16 +9,27 @@ public class ExtractTemplateParam { private Long zoneId; private long eventId; private String url; + private Upload.Mode extractMode; public ExtractTemplateParam() { } - - public ExtractTemplateParam(long userId, long templateId, Long zoneId, long eventId, String url) { + + public ExtractTemplateParam(long userId, long templateId, Long zoneId, long eventId, String url) { + this.userId = userId; + this.templateId = templateId; + this.zoneId = zoneId; + this.eventId = eventId; + this.url = url; + this.extractMode = Upload.Mode.FTP_UPLOAD; + } + + public ExtractTemplateParam(long userId, long templateId, Long zoneId, long eventId, String url, Upload.Mode mode) { this.userId = userId; this.templateId = templateId; this.zoneId = zoneId; this.eventId = eventId; this.url = url; + this.extractMode = mode; } public String getUrl() { @@ -47,4 +60,12 @@ public class ExtractTemplateParam { return eventId; } + public Upload.Mode getExtractMode() { + return extractMode; + } + + public void setExtractMode(Upload.Mode extractMode) { + this.extractMode = extractMode; + } + } diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index 871dc64c616..9ab36f851b9 100755 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -219,6 +219,8 @@ import com.cloud.storage.GuestOSCategoryVO; import com.cloud.storage.GuestOSVO; import com.cloud.storage.LaunchPermissionVO; import com.cloud.storage.Snapshot; +import com.cloud.storage.Upload; +import com.cloud.storage.Volume; import com.cloud.storage.Snapshot.SnapshotType; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotVO; @@ -231,7 +233,6 @@ import com.cloud.storage.StorageStats; import com.cloud.storage.Upload.Type; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.Volume; import com.cloud.storage.Volume.VolumeType; import com.cloud.storage.VolumeStats; import com.cloud.storage.VolumeVO; @@ -248,6 +249,7 @@ import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDao.TemplateFilter; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.download.DownloadMonitor; import com.cloud.storage.preallocatedlun.PreallocatedLunVO; import com.cloud.storage.preallocatedlun.dao.PreallocatedLunDao; import com.cloud.storage.secondary.SecondaryStorageVmManager; @@ -464,7 +466,7 @@ public class ManagementServerImpl implements ManagementServer { _tmpltMgr = locator.getManager(TemplateManager.class); _snapMgr = locator.getManager(SnapshotManager.class); _networkGroupMgr = locator.getManager(NetworkGroupManager.class); - _uploadMonitor = locator.getManager(UploadMonitor.class); + _uploadMonitor = locator.getManager(UploadMonitor.class); _userAuthenticators = locator.getAdapters(UserAuthenticator.class); if (_userAuthenticators == null || !_userAuthenticators.isSet()) { @@ -3536,6 +3538,7 @@ public class ManagementServerImpl implements ManagementServer { } else { domainId = ((account == null) ? DomainVO.ROOT_DOMAIN : account.getDomainId()); } + } else { accountName = account.getAccountName(); accountId = account.getId(); diff --git a/server/src/com/cloud/storage/upload/UploadListener.java b/server/src/com/cloud/storage/upload/UploadListener.java index 69140b6ce69..4ed3cd6278d 100755 --- a/server/src/com/cloud/storage/upload/UploadListener.java +++ b/server/src/com/cloud/storage/upload/UploadListener.java @@ -372,9 +372,9 @@ public class UploadListener implements Listener { asyncMgr.updateAsyncJobAttachment(asyncJobId, type.toString(), 1L); asyncMgr.updateAsyncJobStatus(asyncJobId, AsyncJobResult.STATUS_IN_PROGRESS, resultObj); }else if(answer.getUploadStatus() == Status.UPLOADED){ - asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_SUCCEEDED, 0, resultObj); + asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_SUCCEEDED, 1, resultObj); }else{ - asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_FAILED, 0, resultObj); + asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_FAILED, 2, resultObj); } UploadVO updateBuilder = uploadDao.createForUpdate(); updateBuilder.setUploadPercent(answer.getUploadPct()); diff --git a/server/src/com/cloud/storage/upload/UploadMonitor.java b/server/src/com/cloud/storage/upload/UploadMonitor.java index be5d8c7769d..d62c3969817 100755 --- a/server/src/com/cloud/storage/upload/UploadMonitor.java +++ b/server/src/com/cloud/storage/upload/UploadMonitor.java @@ -21,6 +21,7 @@ package com.cloud.storage.upload; import java.util.Map; import com.cloud.async.AsyncJobManager; +import com.cloud.exception.InternalErrorException; import com.cloud.host.HostVO; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; @@ -55,4 +56,7 @@ public interface UploadMonitor extends Manager{ Long dataCenterId, String installPath, long eventId, long asyncJobId, AsyncJobManager asyncMgr); + UploadVO createEntityDownloadURL(VMTemplateVO template, + VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId) throws InternalErrorException; + } \ No newline at end of file diff --git a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java index 084b58a658d..109e526b32b 100755 --- a/server/src/com/cloud/storage/upload/UploadMonitorImpl.java +++ b/server/src/com/cloud/storage/upload/UploadMonitorImpl.java @@ -1,11 +1,16 @@ package com.cloud.storage.upload; +import java.io.File; import java.util.ArrayList; +import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Timer; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.ejb.Local; import javax.naming.ConfigurationException; @@ -15,6 +20,8 @@ import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; import com.cloud.agent.api.Command; +import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand; +import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand; import com.cloud.agent.api.storage.UploadCommand; import com.cloud.agent.api.storage.UploadProgressCommand.RequestType; import com.cloud.async.AsyncJobManager; @@ -22,20 +29,28 @@ import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.event.EventTypes; import com.cloud.event.EventVO; import com.cloud.event.dao.EventDao; +import com.cloud.exception.InternalErrorException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; -import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Upload; -import com.cloud.storage.Upload.Type; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Upload.Mode; +import com.cloud.storage.Upload.Status; +import com.cloud.storage.Upload.Type; import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; +import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.Inject; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.GlobalLock; +import com.cloud.vm.SecondaryStorageVmVO; +import com.cloud.vm.State; import com.cloud.vm.dao.SecondaryStorageVmDao; /** @@ -70,7 +85,7 @@ public class UploadMonitorImpl implements UploadMonitor { private String _name; private Boolean _sslCopy = new Boolean(false); private String _copyAuthPasswd; - + private ScheduledExecutorService _executor = null; Timer _timer; @@ -159,6 +174,72 @@ public class UploadMonitorImpl implements UploadMonitor { } } + + @Override + public UploadVO createEntityDownloadURL(VMTemplateVO template, VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId) throws InternalErrorException{ + + List storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId); + if(storageServers == null ) + throw new InternalErrorException("No Storage Server found at the datacenter - " +dataCenterId); + + Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE ; + + //Check if it already exists. + List extractURLList = _uploadDao.listByTypeUploadStatus(template.getId(), type, UploadVO.Status.DOWNLOAD_URL_CREATED); + if (extractURLList.size() > 0) + return extractURLList.get(0); + + // It doesn't exist so create a DB entry. + HostVO sserver = storageServers.get(0); + UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(), + Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD); + + _uploadDao.persist(uploadTemplateObj); + + // Create Symlink at ssvm + CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(vmTemplateHost.getInstallPath()); + long result = send(sserver.getId(), cmd, null); + if (result == -1){ + s_logger.warn("Unable to create a link for the template/iso "); + throw new InternalErrorException("Unable to create a link at the SSVM"); + } + + //Construct actual URL locally now that the symlink exists at SSVM + List ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running); + if (ssVms.size() > 0) { + SecondaryStorageVmVO ssVm = ssVms.get(0); + if (ssVm.getPublicIpAddress() == null) { + s_logger.warn("A running secondary storage vm has a null public ip?"); + throw new InternalErrorException("SSVM has null public IP - couldnt create the URL"); + } + String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), vmTemplateHost.getInstallPath()); + UploadVO vo = _uploadDao.createForUpdate(); + vo.setLastUpdated(new Date()); + vo.setUploadUrl(extractURL); + vo.setUploadState(Status.DOWNLOAD_URL_CREATED); + + if(extractURL == null){ + vo.setUploadState(Status.ERROR); + vo.setErrorString("Could not create the download URL"); + } + _uploadDao.update(uploadTemplateObj.getId(), vo); + return _uploadDao.findById(uploadTemplateObj.getId(), true); + } + throw new InternalErrorException("Couldnt find a running SSVM in the zone" + dataCenterId+ ".couldnt create the extraction URL."); + + } + + private String generateCopyUrl(String ipAddress, String path){ + String hostname = ipAddress; + String scheme = "http"; + if (_sslCopy) { + hostname = ipAddress.replace(".", "-"); + hostname = hostname + ".realhostip.com"; + scheme = "https"; + } + return scheme + "://" + hostname + path.substring(path.lastIndexOf(File.separator)); + } + public long send(Long hostId, Command cmd, Listener listener) { @@ -182,6 +263,10 @@ public class UploadMonitorImpl implements UploadMonitor { _copyAuthPasswd = configs.get("secstorage.copy.password"); _agentMgr.registerForHostEvents(new UploadListener(this), true, false, false); + + String workers = (String)params.get("expunge.workers"); + int wrks = NumbersUtil.parseInt(workers, 1); + _executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("UploadMonitor-Scavenger")); return true; } @@ -192,6 +277,8 @@ public class UploadMonitorImpl implements UploadMonitor { @Override public boolean start() { + //FIX ME - Make the timings configurable. + _executor.scheduleWithFixedDelay(new StorageGarbageCollector(), 86400, 86400, TimeUnit.SECONDS); _timer = new Timer(); return true; } @@ -267,5 +354,62 @@ public class UploadMonitorImpl implements UploadMonitor { } - + + protected class StorageGarbageCollector implements Runnable { + + public StorageGarbageCollector() { + } + + @Override + public void run() { + try { + s_logger.info("Extract Monitor Garbage Collection Thread is running."); + + GlobalLock scanLock = GlobalLock.getInternLock(this.getClass().getName()); + try { + if (scanLock.lock(3)) { + try { + cleanupStorage(); + } finally { + scanLock.unlock(); + } + } + } finally { + scanLock.releaseRef(); + } + + } catch (Exception e) { + s_logger.error("Caught the following Exception", e); + } + } + } + + + private long getTimeDiff(Date date){ + Calendar currentCalendar = Calendar.getInstance(); + Calendar givenDateCalendar = Calendar.getInstance(); + givenDateCalendar.setTime(date); + + return (currentCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000; + } + + public void cleanupStorage() { + + final int EXTRACT_URL_TIME_LIMIT = 40; + List extractURLs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED); + + for (UploadVO extractURL : extractURLs){ + if( getTimeDiff(extractURL.getLastUpdated()) < EXTRACT_URL_TIME_LIMIT) continue; + String path = extractURL.getUploadUrl().substring( (extractURL.getUploadUrl().lastIndexOf("/")) +1 ); + DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path); + long result = send(extractURL.getHostId(), cmd, null); + if (result == -1){ + s_logger.warn("Unable to delete the link for " +extractURL.getType()+ " id=" +extractURL.getTypeId()+ " url="+extractURL.getUploadUrl()); + }else{ + _uploadDao.remove(extractURL.getId()); + } + } + + } + } diff --git a/server/src/com/cloud/template/TemplateManager.java b/server/src/com/cloud/template/TemplateManager.java index 78b14e38d08..4e45de20d58 100755 --- a/server/src/com/cloud/template/TemplateManager.java +++ b/server/src/com/cloud/template/TemplateManager.java @@ -192,6 +192,6 @@ public interface TemplateManager extends Manager { boolean deleteIso(DeleteIsoCmd cmd) throws InvalidParameterValueException, InternalErrorException, PermissionDeniedException; void extract(VMTemplateVO template, String url, VMTemplateHostVO tmpltHostRef, Long zoneId, long eventId, long asyncJobId, AsyncJobManager asyncMgr); - void extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException; - void extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException; + Long extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException; + Long extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException; } diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 197502a4cea..917bb7fcf94 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -49,7 +49,9 @@ import com.cloud.api.commands.ExtractTemplateCmd; import com.cloud.api.commands.RegisterIsoCmd; import com.cloud.api.commands.RegisterTemplateCmd; import com.cloud.async.AsyncJobManager; +import com.cloud.async.AsyncJobResult; import com.cloud.async.AsyncJobVO; +import com.cloud.async.executor.ExtractJobResultObject; import com.cloud.configuration.ResourceCount.ResourceType; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.dc.DataCenterVO; @@ -76,6 +78,8 @@ import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolVO; +import com.cloud.storage.Upload; +import com.cloud.storage.UploadVO; import com.cloud.storage.Upload.Type; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStoragePoolVO; @@ -87,6 +91,7 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplatePoolDao; @@ -146,6 +151,7 @@ public class TemplateManagerImpl implements TemplateManager { @Inject VolumeDao _volumeDao; @Inject SnapshotDao _snapshotDao; @Inject DomainDao _domainDao; + @Inject UploadDao _uploadDao; long _routerTemplateId = -1; @Inject StorageManager _storageMgr; @Inject UserVmManager _vmMgr; @@ -405,26 +411,30 @@ public class TemplateManagerImpl implements TemplateManager { } @Override - public void extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException { + public Long extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException { Account account = (Account)UserContext.current().getAccountObject(); Long templateId = cmd.getId(); Long zoneId = cmd.getZoneId(); String url = cmd.getUrl(); - - extract(account, templateId, url, zoneId, true, cmd.getJob(), cmd.getAsyncJobManager()); + String mode = cmd.getMode(); + Long eventId = cmd.getStartEventId(); + + return extract(account, templateId, url, zoneId, mode, eventId, true, cmd.getJob(), cmd.getAsyncJobManager()); } @Override - public void extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException { + public Long extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException { Account account = (Account)UserContext.current().getAccountObject(); Long templateId = cmd.getId(); Long zoneId = cmd.getZoneId(); String url = cmd.getUrl(); + String mode = cmd.getMode(); + Long eventId = cmd.getStartEventId(); - extract(account, templateId, url, zoneId, true, cmd.getJob(), cmd.getAsyncJobManager()); + return extract(account, templateId, url, zoneId, mode, eventId, false, cmd.getJob(), cmd.getAsyncJobManager()); } - private void extract(Account account, Long templateId, String url, Long zoneId, boolean isISO, AsyncJobVO job, AsyncJobManager mgr) throws InvalidParameterValueException, PermissionDeniedException { + private Long extract(Account account, Long templateId, String url, Long zoneId, String mode, Long eventId, boolean isISO, AsyncJobVO job, AsyncJobManager mgr) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException { String desc = "template"; if (isISO) { desc = "ISO"; @@ -432,10 +442,10 @@ public class TemplateManagerImpl implements TemplateManager { VMTemplateVO template = _tmpltDao.findById(templateId); if (template == null) { - throw new InvalidParameterValueException("Unable to find template with id " + templateId); + throw new InvalidParameterValueException("Unable to find " +desc+ " with id " + templateId); } if (template.getName().startsWith("xs-tools") ){ - throw new InvalidParameterValueException("Unable to extract the " + desc + " " + template.getName() + " It is not allowed"); + throw new InvalidParameterValueException("Unable to extract the ISO " + template.getName() + " It is not allowed"); } if (isISO) { if (template.getFormat() != ImageFormat.ISO ){ @@ -451,33 +461,6 @@ public class TemplateManagerImpl implements TemplateManager { throw new IllegalArgumentException("Please specify a valid zone."); } - if (url.toLowerCase().contains("file://")){ - throw new InvalidParameterValueException("file:// type urls are currently unsupported"); - } - - URI uri = null; - try { - uri = new URI(url); - if ((uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp") )) { - throw new InvalidParameterValueException("Unsupported scheme for url: " + url); - } - } catch (URISyntaxException ex) { - throw new InvalidParameterValueException("Invalid url given: " + url); - } - - String host = uri.getHost(); - try { - InetAddress hostAddr = InetAddress.getByName(host); - if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress() ) { - throw new InvalidParameterValueException("Illegal host specified in url"); - } - if (hostAddr instanceof Inet6Address) { - throw new InvalidParameterValueException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")"); - } - } catch (UnknownHostException uhe) { - throw new InvalidParameterValueException("Unable to resolve " + host); - } - if (account != null) { if(!isAdmin(account.getType())){ if (template.getAccountId() != account.getId()){ @@ -486,7 +469,7 @@ public class TemplateManagerImpl implements TemplateManager { } else { Account userAccount = _accountDao.findById(template.getAccountId()); if((userAccount == null) || !_domainDao.isChildDomain(account.getDomainId(), userAccount.getDomainId())) { - throw new PermissionDeniedException("Unable to extract " + desc + " " + templateId + " to " + url + ", permission denied."); + throw new PermissionDeniedException("Unable to extract " + desc + "=" + templateId + " - permission denied."); } } } @@ -499,21 +482,68 @@ public class TemplateManagerImpl implements TemplateManager { throw new InvalidParameterValueException("The " + desc + " has not been downloaded "); } } - - if ( _uploadMonitor.isTypeUploadInProgress(templateId, isISO ? Type.ISO : Type.TEMPLATE) ){ - throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same"); + + Upload.Mode extractMode; + if( mode == null || (!mode.equals(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equals(Upload.Mode.HTTP_DOWNLOAD.toString())) ){ + throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid extract Mode "); + }else{ + extractMode = mode.equals(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD; } - + long userId = UserContext.current().getUserId(); long accountId = template.getAccountId(); - String event = isISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD; - long eventId = EventUtils.saveScheduledEvent(userId, accountId, event, "Extraction job"); - -// FIXME: scheduled event should've already been saved, we should be saving this started event here... -// String event = template.getFormat() == ImageFormat.ISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD; -// EventUtils.saveStartedEvent(template.getAccountId(), template.getAccountId(), event, "Starting upload of " +template.getName()+ " to " +url, cmd.getStartEventId()); - - extract(template, url, tmpltHostRef, zoneId, eventId, job.getId(), mgr); + String event = isISO ? EventTypes.EVENT_ISO_EXTRACT : EventTypes.EVENT_TEMPLATE_EXTRACT; + if (extractMode == Upload.Mode.FTP_UPLOAD){ + URI uri = null; + try { + uri = new URI(url); + if ((uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp") )) { + throw new InvalidParameterValueException("Unsupported scheme for url: " + url); + } + } catch (URISyntaxException ex) { + throw new InvalidParameterValueException("Invalid url given: " + url); + } + + String host = uri.getHost(); + try { + InetAddress hostAddr = InetAddress.getByName(host); + if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress() ) { + throw new InvalidParameterValueException("Illegal host specified in url"); + } + if (hostAddr instanceof Inet6Address) { + throw new InvalidParameterValueException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")"); + } + } catch (UnknownHostException uhe) { + throw new InvalidParameterValueException("Unable to resolve " + host); + } + + if ( _uploadMonitor.isTypeUploadInProgress(templateId, isISO ? Type.ISO : Type.TEMPLATE) ){ + throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same"); + } + + //long eventId = EventUtils.saveScheduledEvent(userId, accountId, event, "Extraction job"); + + // FIXME: scheduled event should've already been saved, we should be saving this started event here... + // String event = template.getFormat() == ImageFormat.ISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD; + // EventUtils.saveStartedEvent(template.getAccountId(), template.getAccountId(), event, "Starting upload of " +template.getName()+ " to " +url, cmd.getStartEventId()); + + EventUtils.saveStartedEvent(userId, accountId, event, "Starting extraction of " +template.getName()+ " mode= " +extractMode.toString(), eventId); + extract(template, url, tmpltHostRef, zoneId, eventId, job.getId(), mgr); + return 1L; //FIX ME + } + + EventUtils.saveStartedEvent(userId, accountId, event, "Starting extraction of " +template.getName()+ " in mode:" +extractMode.toString(), eventId); + UploadVO vo = _uploadMonitor.createEntityDownloadURL(template, tmpltHostRef, zoneId, eventId); + if (vo!=null){ + ExtractJobResultObject resultObject = new ExtractJobResultObject(template.getAccountId(), template.getName(), Upload.Status.DOWNLOAD_URL_CREATED.toString(), vo.getId(), url); + mgr.completeAsyncJob(job.getId(), AsyncJobResult.STATUS_SUCCEEDED, 1, resultObject); + EventUtils.saveEvent(userId, accountId, EventVO.LEVEL_INFO, event, "Completed extraction of "+template.getName()+ " in mode:" +extractMode.toString(), null, eventId); + return vo.getId(); + }else{ + EventUtils.saveEvent(userId, accountId, EventVO.LEVEL_ERROR, event, "Failed extraction of "+template.getName()+ " in mode:" +extractMode.toString(), null, eventId); + mgr.completeAsyncJob(job.getId(), AsyncJobResult.STATUS_FAILED, 2, null); + return null; + } } @Override diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index b45ef5b9412..2c93a92c832 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -633,6 +633,7 @@ CREATE TABLE `cloud`.`upload` ( `host_id` bigint unsigned NOT NULL, `type_id` bigint unsigned NOT NULL, `type` varchar(255), + `mode` varchar(255), `created` DATETIME NOT NULL, `last_updated` DATETIME, `job_id` varchar(255),