etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug 6399: extending extract functionlity for templates and ISO's to allow download. The extraction can have two modes FTP_UPLOAD and HTTP_DOWNLOAD. In the former one the user would provide the ftp url where the entity needs to be uploaded and in the later the user would be provided a HTTP URL where from he/she can download the entity. This url would be exposed for a specific time limit and would not function after the time limit.

This commit is contained in:
nit 2010-10-20 11:40:03 +05:30
parent 6721c4423e
commit 6723549fc7
26 changed files with 651 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java Normal file
View File

@ -0,0 +1,21 @@
package com.cloud.agent.api.storage;
import com.cloud.agent.api.Answer;
public class CreateEntityDownloadURLAnswer extends Answer{
String resultString;
short resultCode;
public static final short RESULT_SUCCESS = 1;
public static final short RESULT_FAILURE = 0;
public CreateEntityDownloadURLAnswer(String resultString, short resultCode) {
super();
this.resultString = resultString;
this.resultCode = resultCode;
}
public CreateEntityDownloadURLAnswer(){
}
}

30
core/src/com/cloud/agent/api/storage/CreateEntityDownloadURLCommand.java Normal file
View File

@ -0,0 +1,30 @@
package com.cloud.agent.api.storage;
import com.cloud.agent.api.Command;
public class CreateEntityDownloadURLCommand extends AbstractDownloadCommand {
public CreateEntityDownloadURLCommand(String installPath) {
super();
this.installPath = installPath;
}
public CreateEntityDownloadURLCommand() {
}
private String installPath;
@Override
public boolean executeInSequence() {
return false;
}
public String getInstallPath() {
return installPath;
}
public void setInstallPath(String installPath) {
this.installPath = installPath;
}
}

21
core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLAnswer.java Normal file
View File

@ -0,0 +1,21 @@
package com.cloud.agent.api.storage;
import com.cloud.agent.api.Answer;
public class DeleteEntityDownloadURLAnswer extends Answer{
String resultString;
short resultCode;
public static final short RESULT_SUCCESS = 1;
public static final short RESULT_FAILURE = 0;
public DeleteEntityDownloadURLAnswer(String resultString, short resultCode) {
super();
this.resultString = resultString;
this.resultCode = resultCode;
}
public DeleteEntityDownloadURLAnswer(){
}
}

25
core/src/com/cloud/agent/api/storage/DeleteEntityDownloadURLCommand.java Normal file
View File

@ -0,0 +1,25 @@
package com.cloud.agent.api.storage;
public class DeleteEntityDownloadURLCommand extends AbstractDownloadCommand {
String path;
public DeleteEntityDownloadURLCommand(String path) {
super();
this.path = path;
}
public DeleteEntityDownloadURLCommand() {
super();
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
}

5
core/src/com/cloud/storage/Upload.java
View File

@ -2,6 +2,7 @@ package com.cloud.storage;
public interface Upload {
public static enum Status {UNKNOWN, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, NOT_COPIED, COPY_IN_PROGRESS, COPY_ERROR, COPY_COMPLETE}
public static enum Type {VOLUME, TEMPLATE, ISO}
public static enum Status {UNKNOWN, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, NOT_COPIED, COPY_IN_PROGRESS, COPY_ERROR, COPY_COMPLETE, DOWNLOAD_URL_CREATED, DOWNLOAD_URL_NOT_CREATED, ERROR}
public static enum Type {VOLUME, TEMPLATE, ISO}
public static enum Mode {FTP_UPLOAD, HTTP_DOWNLOAD}
}

27
core/src/com/cloud/storage/UploadVO.java
View File

@ -65,6 +65,10 @@ public class UploadVO implements Upload {
@Enumerated(EnumType.STRING)
private Type type;
@Column (name="mode")
@Enumerated(EnumType.STRING)
private Mode mode = Mode.FTP_UPLOAD;
@Column (name="upload_state")
@Enumerated(EnumType.STRING)
private Status uploadState;
@ -122,6 +126,19 @@ public class UploadVO implements Upload {
this.jobId = jobId;
this.uploadUrl = uploadUrl;
}
public UploadVO(long hostId, long typeId, Date lastUpdated,
Status uploadState, int uploadPercent, Type type,
Mode mode) {
super();
this.hostId = hostId;
this.typeId = typeId;
this.lastUpdated = lastUpdated;
this.uploadState = uploadState;
this.uploadPercent = uploadPercent;
this.type = type;
this.mode = mode;
}
protected UploadVO() {
}
@ -200,7 +217,15 @@ public class UploadVO implements Upload {
this.type = type;
}
public String getUploadUrl() {
public Mode getMode() {
return mode;
}
public void setMode(Mode mode) {
this.mode = mode;
}
public String getUploadUrl() {
return uploadUrl;
}

3
core/src/com/cloud/storage/dao/UploadDao.java
View File

@ -5,6 +5,7 @@ import java.util.List;
import com.cloud.storage.UploadVO;
import com.cloud.storage.Upload.Status;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.Upload.Mode;
import com.cloud.utils.db.GenericDao;
public interface UploadDao extends GenericDao<UploadVO, Long> {
@ -13,5 +14,7 @@ public interface UploadDao extends GenericDao<UploadVO, Long> {
Status uploadState);
List<UploadVO> listByHostAndUploadStatus(long sserverId, Status uploadInProgress);
List<UploadVO> listByModeAndStatus(Mode mode, Status uploadState);
}

16
core/src/com/cloud/storage/dao/UploadDaoImpl.java
View File

@ -6,6 +6,7 @@ import org.apache.log4j.Logger;
import com.cloud.storage.UploadVO;
import com.cloud.storage.Upload.Status;
import com.cloud.storage.Upload.Mode;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
@ -15,6 +16,7 @@ public class UploadDaoImpl extends GenericDaoBase<UploadVO, Long> implements Upl
public static final Logger s_logger = Logger.getLogger(UploadDaoImpl.class.getName());
protected final SearchBuilder<UploadVO> typeUploadStatusSearch;
protected final SearchBuilder<UploadVO> typeHostAndUploadStatusSearch;
protected final SearchBuilder<UploadVO> typeModeAndStatusSearch;
protected static final String UPDATE_UPLOAD_INFO =
"UPDATE upload SET upload_state = ?, upload_pct= ?, last_updated = ? "
@ -37,6 +39,12 @@ public class UploadDaoImpl extends GenericDaoBase<UploadVO, Long> implements Upl
typeHostAndUploadStatusSearch.and("host_id", typeHostAndUploadStatusSearch.entity().getHostId(), SearchCriteria.Op.EQ);
typeHostAndUploadStatusSearch.and("upload_state", typeHostAndUploadStatusSearch.entity().getUploadState(), SearchCriteria.Op.EQ);
typeHostAndUploadStatusSearch.done();
typeModeAndStatusSearch = createSearchBuilder();
typeModeAndStatusSearch.and("mode", typeModeAndStatusSearch.entity().getMode(), SearchCriteria.Op.EQ);
typeModeAndStatusSearch.and("upload_state", typeModeAndStatusSearch.entity().getUploadState(), SearchCriteria.Op.EQ);
typeModeAndStatusSearch.done();
}
@Override
@ -55,4 +63,12 @@ public class UploadDaoImpl extends GenericDaoBase<UploadVO, Long> implements Upl
sc.setParameters("upload_state", uploadState.toString());
return listBy(sc);
}
@Override
public List<UploadVO> listByModeAndStatus(Mode mode, Status uploadState){
SearchCriteria<UploadVO> sc = typeModeAndStatusSearch.create();
sc.setParameters("mode", mode.toString());
sc.setParameters("upload_state", uploadState.toString());
return listBy(sc);
}
}

9
core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
View File

@ -45,6 +45,9 @@ import com.cloud.agent.api.SecStorageSetupCommand;
import com.cloud.agent.api.StartupCommand;
import com.cloud.agent.api.StartupStorageCommand;
import com.cloud.agent.api.SecStorageFirewallCfgCommand.PortConfig;
import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
import com.cloud.agent.api.storage.DeleteTemplateCommand;
import com.cloud.agent.api.storage.DownloadCommand;
import com.cloud.agent.api.storage.DownloadProgressCommand;
@ -117,7 +120,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return _dlMgr.handleDownloadCommand((DownloadCommand)cmd);
}else if (cmd instanceof UploadCommand) {
return _upldMgr.handleUploadCommand((UploadCommand)cmd);
} else if (cmd instanceof GetStorageStatsCommand) {
}else if (cmd instanceof CreateEntityDownloadURLCommand){
return _upldMgr.handleCreateEntityURLCommand((CreateEntityDownloadURLCommand)cmd);
}else if(cmd instanceof DeleteEntityDownloadURLCommand){
return _upldMgr.handleDeleteEntityDownloadURLCommand((DeleteEntityDownloadURLCommand)cmd);
}else if (cmd instanceof GetStorageStatsCommand) {
return execute((GetStorageStatsCommand)cmd);
} else if (cmd instanceof CheckHealthCommand) {
return new CheckHealthAnswer((CheckHealthCommand)cmd, true);

9
core/src/com/cloud/storage/template/UploadManager.java Normal file → Executable file
View File

@ -1,5 +1,9 @@
package com.cloud.storage.template;
import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
import com.cloud.agent.api.storage.UploadAnswer;
import com.cloud.agent.api.storage.UploadCommand;
import com.cloud.storage.StorageResource;
@ -61,4 +65,9 @@ public interface UploadManager extends Manager {
String cksum, String installPathPrefix, String user,
String password, long maxTemplateSizeInBytes);
CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd);
DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd);
}

88
core/src/com/cloud/storage/template/UploadManagerImpl.java
View File

@ -1,7 +1,6 @@
package com.cloud.storage.template;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.text.SimpleDateFormat;
@ -12,20 +11,22 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import com.cloud.agent.api.storage.CreateEntityDownloadURLAnswer;
import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLAnswer;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
import com.cloud.agent.api.storage.UploadAnswer;
import com.cloud.agent.api.storage.UploadProgressCommand;
import com.cloud.agent.api.storage.UploadCommand;
import com.cloud.agent.api.storage.UploadAnswer;
import com.cloud.agent.api.storage.UploadCommand;
import com.cloud.storage.StorageLayer;
import com.cloud.storage.StorageResource;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.template.TemplateUploader.UploadCompleteCallback;
import com.cloud.storage.template.TemplateUploader.Status;
@ -33,6 +34,7 @@ import com.cloud.utils.NumbersUtil;
import com.cloud.utils.UUID;
import com.cloud.utils.component.Adapters;
import com.cloud.utils.component.ComponentLocator;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
@ -152,18 +154,19 @@ public class UploadManagerImpl implements UploadManager {
return templatesize;
}
}
public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class);
private ExecutorService threadPool;
private final Map<String, UploadJob> jobs = new ConcurrentHashMap<String, UploadJob>();
private String parentDir;
private Adapters<Processor> _processors;
private String publicTemplateRepo;
private StorageLayer _storage;
private int installTimeoutPerGig;
private boolean _sslCopy;
private String _name;
private boolean hvm;
public static final Logger s_logger = Logger.getLogger(UploadManagerImpl.class);
private ExecutorService threadPool;
private final Map<String, UploadJob> jobs = new ConcurrentHashMap<String, UploadJob>();
private String parentDir;
private Adapters<Processor> _processors;
private String publicTemplateRepo;
private StorageLayer _storage;
private int installTimeoutPerGig;
private boolean _sslCopy;
private String _name;
private boolean hvm;
@Override
public String uploadPublicTemplate(long id, String url, String name,
ImageFormat format, Long accountId, String descr,
@ -320,6 +323,52 @@ public class UploadManagerImpl implements UploadManager {
return new UploadAnswer(jobId, getUploadPct(jobId), getUploadError(jobId), getUploadStatus2(jobId), getUploadLocalPath(jobId), getInstallPath(jobId),
getUploadTemplateSize(jobId));
}
@Override
public CreateEntityDownloadURLAnswer handleCreateEntityURLCommand(CreateEntityDownloadURLCommand cmd){
// Create the directory structure so that its visible under apache server root
Script command = new Script("mkdir", s_logger);
command.add("-p");
command.add("/var/www/html/");
String result = command.execute();
if (result != null) {
String errorString = "Error in creating directory =" + result;
s_logger.warn(errorString);
return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
}
// Create a symbolic link from the actual directory to the template location
cmd.getInstallPath();
command = new Script("/bin/bash", s_logger);
command.add("-c");
command.add("ln -sf " + publicTemplateRepo + cmd.getInstallPath() + " /var/www/html/");
result = command.execute();
if (result != null) {
String errorString = "Error in linking err=" + result;
s_logger.warn(errorString);
return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
}
return new CreateEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS);
}
@Override
public DeleteEntityDownloadURLAnswer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd){
s_logger.debug("handleDeleteEntityDownloadURLCommand "+cmd.getPath());
Script command = new Script("/bin/bash", s_logger);
command.add("-c");
command.add("unlink /var/www/html/"+cmd.getPath());
String result = command.execute();
if (result != null) {
String errorString = "Error in deleting =" + result;
s_logger.warn(errorString);
return new DeleteEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE);
}
return new DeleteEntityDownloadURLAnswer("", CreateEntityDownloadURLAnswer.RESULT_SUCCESS);
}
private String getInstallPath(String jobId) {
// TODO Auto-generated method stub
@ -375,9 +424,9 @@ public class UploadManagerImpl implements UploadManager {
configureFolders(name, params);
String inSystemVM = (String)params.get("secondary.storage.vm");
if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
s_logger.info("UploadManager: starting additional services since we are inside system vm");
startAdditionalServices();
blockOutgoingOnPrivate();
//s_logger.info("UploadManager: starting additional services since we are inside system vm");
//startAdditionalServices();
//blockOutgoingOnPrivate();
}
value = (String) params.get("install.timeout.pergig");
@ -407,6 +456,7 @@ public class UploadManagerImpl implements UploadManager {
processors.add(processor);
// Add more processors here.
threadPool = Executors.newFixedThreadPool(numInstallThreads);
return true;
}

10
server/src/com/cloud/api/ApiDBUtils.java Normal file → Executable file
View File

@ -47,6 +47,7 @@ import com.cloud.storage.SnapshotVO;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePoolVO;
import com.cloud.storage.StorageStats;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
@ -55,6 +56,7 @@ import com.cloud.storage.dao.GuestOSCategoryDao;
import com.cloud.storage.dao.GuestOSDao;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.StoragePoolDao;
import com.cloud.storage.dao.UploadDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateHostDao;
import com.cloud.storage.dao.VolumeDao;
@ -109,6 +111,7 @@ public class ApiDBUtils {
private static StoragePoolDao _storagePoolDao;
private static VMTemplateDao _templateDao;
private static VMTemplateHostDao _templateHostDao;
private static UploadDao _uploadDao;
private static UserDao _userDao;
private static UserStatisticsDao _userStatsDao;
private static UserVmDao _userVmDao;
@ -147,6 +150,7 @@ public class ApiDBUtils {
_storagePoolDao = locator.getDao(StoragePoolDao.class);
_templateDao = locator.getDao(VMTemplateDao.class);
_templateHostDao = locator.getDao(VMTemplateHostDao.class);
_uploadDao = locator.getDao(UploadDao.class);
_userDao = locator.getDao(UserDao.class);
_userStatsDao = locator.getDao(UserStatisticsDao.class);
_userVmDao = locator.getDao(UserVmDao.class);
@ -368,7 +372,11 @@ public class ApiDBUtils {
return _templateHostDao.findByHostTemplate(secondaryStorageHost.getId(), templateId);
}
}
public static UploadVO findUploadById(Long id){
return _uploadDao.findById(id);
}
public static User findUserById(Long userId) {
return _userDao.findById(userId);
}

2
server/src/com/cloud/api/BaseCmd.java
View File

@ -94,7 +94,7 @@ public abstract class BaseCmd {
@Parameter(name="response", type=CommandType.STRING)
private String responseType;
public String getResponseType() {
if (responseType == null) {
return RESPONSE_TYPE_XML;

28
server/src/com/cloud/api/commands/ExtractIsoCmd.java
View File

@ -18,6 +18,7 @@
package com.cloud.api.commands;
import org.apache.log4j.Logger;
import org.junit.runners.Parameterized.Parameters;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.BaseAsyncCmd;
@ -26,6 +27,7 @@ import com.cloud.api.Implementation;
import com.cloud.api.Parameter;
import com.cloud.api.response.ExtractResponse;
import com.cloud.event.EventTypes;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.user.Account;
@ -42,12 +44,14 @@ public class ExtractIsoCmd extends BaseAsyncCmd {
@Parameter(name="id", type=CommandType.LONG, required=true, description="the ID of the ISO file")
private Long id;
//FIXME - add description
@Parameter(name="url", type=CommandType.STRING, required=true)
@Parameter(name="url", type=CommandType.STRING, required=false, description="the url to which the ISO would be extracted")
private String url;
@Parameter(name="zoneid", type=CommandType.LONG, required=true, description="the ID of the zone where the ISO is originally located")
private Long zoneId;
@Parameter(name="mode", type=CommandType.STRING, required=true, description="the mode of extraction - HTTP_DOWNLOAD or FTP_UPLOAD")
private String mode;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
@ -65,6 +69,9 @@ public class ExtractIsoCmd extends BaseAsyncCmd {
return zoneId;
}
public String getMode() {
return mode;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -83,7 +90,7 @@ public class ExtractIsoCmd extends BaseAsyncCmd {
public long getAccountId() {
VMTemplateVO iso = ApiDBUtils.findTemplateById(getId());
if (iso != null) {
return iso.getId();
return iso.getAccountId();
}
// invalid id, parent this command to SYSTEM so ERROR events are tracked
@ -97,8 +104,21 @@ public class ExtractIsoCmd extends BaseAsyncCmd {
@Override @SuppressWarnings("unchecked")
public ExtractResponse getResponse() {
ExtractResponse response = (ExtractResponse)getResponseObject();
Long uploadId = (Long)getResponseObject();
UploadVO uploadInfo = ApiDBUtils.findUploadById(uploadId);
ExtractResponse response = new ExtractResponse();
response.setResponseName(getName());
response.setId(id);
response.setName(ApiDBUtils.findTemplateById(id).getName());
response.setZoneId(zoneId);
response.setZoneName(ApiDBUtils.findZoneById(zoneId).getName());
response.setMode(mode);
response.setUploadId(uploadId);
response.setState(uploadInfo.getUploadState().toString());
response.setAccountId(getAccountId());
//FIX ME - Need to set the url once the gson jar is upgraded since it is throwing an error right now due to a bug.
//response.setUrl(uploadInfo.getUploadUrl());
return response;
}

37
server/src/com/cloud/api/commands/ExtractTemplateCmd.java
View File

@ -20,12 +20,14 @@ package com.cloud.api.commands;
import org.apache.log4j.Logger;
import com.cloud.api.BaseAsyncCmd;
import com.cloud.api.BaseCmd.CommandType;
import com.cloud.api.BaseCmd.Manager;
import com.cloud.api.ApiDBUtils;
import com.cloud.api.Implementation;
import com.cloud.api.Parameter;
import com.cloud.api.response.ExtractResponse;
import com.cloud.event.EventTypes;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.user.Account;
@ -39,18 +41,18 @@ public class ExtractTemplateCmd extends BaseAsyncCmd {
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
//FIXME - add description
@Parameter(name="id", type=CommandType.LONG, required=true)
@Parameter(name="id", type=CommandType.LONG, required=true, description="the ID of the template")
private Long id;
//FIXME - add description
@Parameter(name="url", type=CommandType.STRING, required=true)
@Parameter(name="url", type=CommandType.STRING, required=false, description="the url to which the ISO would be extracted")
private String url;
//FIXME - add description
@Parameter(name="zoneid", type=CommandType.LONG, required=true)
@Parameter(name="zoneid", type=CommandType.LONG, required=true, description="the ID of the zone where the ISO is originally located" )
private Long zoneId;
@Parameter(name="mode", type=CommandType.STRING, required=true, description="the mode of extraction - HTTP_DOWNLOAD or FTP_UPLOAD")
private String mode;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -67,6 +69,10 @@ public class ExtractTemplateCmd extends BaseAsyncCmd {
return zoneId;
}
public String getMode() {
return mode;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -84,7 +90,7 @@ public class ExtractTemplateCmd extends BaseAsyncCmd {
public long getAccountId() {
VMTemplateVO template = ApiDBUtils.findTemplateById(getId());
if (template != null) {
return template.getId();
return template.getAccountId();
}
// invalid id, parent this command to SYSTEM so ERROR events are tracked
@ -103,8 +109,21 @@ public class ExtractTemplateCmd extends BaseAsyncCmd {
@Override @SuppressWarnings("unchecked")
public ExtractResponse getResponse() {
ExtractResponse response = (ExtractResponse)getResponseObject();
Long uploadId = (Long)getResponseObject();
UploadVO uploadInfo = ApiDBUtils.findUploadById(uploadId);
ExtractResponse response = new ExtractResponse();
response.setResponseName(getName());
response.setId(id);
response.setName(ApiDBUtils.findTemplateById(id).getName());
response.setZoneId(zoneId);
response.setZoneName(ApiDBUtils.findZoneById(zoneId).getName());
response.setMode(mode);
response.setUploadId(uploadId);
response.setState(uploadInfo.getUploadState().toString());
response.setAccountId(getAccountId());
//FIX ME - Need to set the url once the gson jar is upgraded since it is throwing an error right now.
//response.setUrl(uploadInfo.getUploadUrl());
return response;
}
}

59
server/src/com/cloud/api/response/ExtractResponse.java Normal file → Executable file
View File

@ -24,21 +24,22 @@ import com.google.gson.annotations.SerializedName;
public class ExtractResponse extends BaseResponse {
@SerializedName("id") @Param(description="the id of extracted object")
private long id;
private Long id;
@SerializedName("name") @Param(description="the name of the extracted object")
private String name;
//FIXME - add description
@SerializedName("uploadpercentage")
private int uploadPercent;
@SerializedName("uploadId") @Param(description="the upload id of extracted object")
private Long uploadId;
//FIXME - add description
@SerializedName("uploadstatus")
private String uploadStatus;
@SerializedName("uploadpercentage") @Param(description="the percentage of the entity uploaded to the specified location")
private Integer uploadPercent;
@SerializedName("status") @Param(description="the status of the ")
private String status;
@SerializedName("accountid") @Param(description="the account id to which the extracted object belongs")
private long accountId;
private Long accountId;
@SerializedName("resultstring") @Param(description="")
private String resultString;
@ -63,7 +64,13 @@ public class ExtractResponse extends BaseResponse {
@SerializedName("zonename") @Param(description="zone name the object was extracted from")
private String zoneName;
public long getId() {
@SerializedName("extractMode") @Param(description="the mode of extraction - upload or download")
private String mode;
@SerializedName("url") @Param(description="if mode = upload then url of the uploaded entity. if mode = download the url from which the entity can be downloaded")
private String url;
public Long getId() {
return id;
}
@ -79,7 +86,15 @@ public class ExtractResponse extends BaseResponse {
this.name = name;
}
public int getUploadPercent() {
public Long getUploadId() {
return uploadId;
}
public void setUploadId(Long uploadId) {
this.uploadId = uploadId;
}
public Integer getUploadPercent() {
return uploadPercent;
}
@ -88,14 +103,14 @@ public class ExtractResponse extends BaseResponse {
}
public String getUploadStatus() {
return uploadStatus;
return status;
}
public void setUploadStatus(String uploadStatus) {
this.uploadStatus = uploadStatus;
public void setUploadStatus(String Status) {
this.status = status;
}
public long getAccountId() {
public Long getAccountId() {
return accountId;
}
@ -158,4 +173,20 @@ public class ExtractResponse extends BaseResponse {
public void setZoneName(String zoneName) {
this.zoneName = zoneName;
}
public String getMode() {
return mode;
}
public void setMode(String mode) {
this.mode = mode;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
}

84
server/src/com/cloud/async/executor/ExtractJobResultObject.java
View File

@ -9,14 +9,22 @@ import com.cloud.storage.upload.UploadState;
public class ExtractJobResultObject {
public ExtractJobResultObject(Long accountId, String typeName, String currState, int i, Long uploadId){
public ExtractJobResultObject(Long accountId, String typeName, String currState, int uploadPercent, Long uploadId){
this.accountId = accountId;
this.name = typeName;
this.state = currState;
this.id = uploadId;
this.uploadPercent = i;
this.uploadPercent = uploadPercent;
}
public ExtractJobResultObject(Long accountId, String typeName, String currState, Long uploadId, String url){
this.accountId = accountId;
this.name = typeName;
this.state = currState;
this.id = uploadId;
this.url = url;
}
public ExtractJobResultObject(){
}
@ -38,30 +46,6 @@ public class ExtractJobResultObject {
@Param(name="result_string")
String result_string;
public int getUploadPercent() {
return uploadPercent;
}
public void setUploadPercent(int i) {
this.uploadPercent = i;
}
public String getUploadStatus() {
return uploadStatus;
}
public void setUploadStatus(String uploadStatus) {
this.uploadStatus = uploadStatus;
}
public String getResult_string() {
return result_string;
}
public void setResult_string(String resultString) {
result_string = resultString;
}
@Param(name="created")
private Date createdDate;
@ -77,6 +61,45 @@ public class ExtractJobResultObject {
@Param(name="zoneid")
private Long zoneId;
@Param(name="zonename")
private String zoneName;
@Param(name="url")
private String url;
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public int getUploadPercent() {
return uploadPercent;
}
public void setUploadPercent(int i) {
this.uploadPercent = i;
}
public String getUploadStatus() {
return uploadStatus;
}
public void setUploadStatus(String uploadStatus) {
this.uploadStatus = uploadStatus;
}
public String getResult_string() {
return result_string;
}
public void setResult_string(String resultString) {
result_string = resultString;
}
public Long getZoneId() {
return zoneId;
}
@ -93,11 +116,6 @@ public class ExtractJobResultObject {
this.zoneName = zoneName;
}
@Param(name="zonename")
private String zoneName;
private long size;
public String getStorage() {
return storage;
}
@ -121,10 +139,6 @@ public class ExtractJobResultObject {
public String getName() {
return name;
}
public void setSize(long size) {
this.size = size;
}
public void setCreatedDate(Date createdDate) {
this.createdDate = createdDate;

3
server/src/com/cloud/async/executor/ExtractTemplateExecutor.java Normal file → Executable file
View File

@ -25,8 +25,7 @@ public class ExtractTemplateExecutor extends BaseAsyncJobExecutor {
/*
try {
managementServer.extractTemplate(param.getUrl(), param.getTemplateId(), param.getZoneId(), param.getEventId(), getJob().getId());
managementServer.extractTemplate(param.getUrl(), param.getTemplateId(), param.getZoneId(), param.getEventId(), getJob().getId(), param.getExtractMode());
} catch (Exception e) {
s_logger.warn("Unable to extract template: " + e.getMessage(), e);
asyncMgr.completeAsyncJob(getJob().getId(), AsyncJobResult.STATUS_FAILED, BaseCmd.INTERNAL_ERROR, e.getMessage());

25
server/src/com/cloud/async/executor/ExtractTemplateParam.java
View File

@ -1,5 +1,7 @@
package com.cloud.async.executor;
import com.cloud.storage.Upload;
public class ExtractTemplateParam {
private long userId;
@ -7,16 +9,27 @@ public class ExtractTemplateParam {
private Long zoneId;
private long eventId;
private String url;
private Upload.Mode extractMode;
public ExtractTemplateParam() {
}
public ExtractTemplateParam(long userId, long templateId, Long zoneId, long eventId, String url) {
public ExtractTemplateParam(long userId, long templateId, Long zoneId, long eventId, String url) {
this.userId = userId;
this.templateId = templateId;
this.zoneId = zoneId;
this.eventId = eventId;
this.url = url;
this.extractMode = Upload.Mode.FTP_UPLOAD;
}
public ExtractTemplateParam(long userId, long templateId, Long zoneId, long eventId, String url, Upload.Mode mode) {
this.userId = userId;
this.templateId = templateId;
this.zoneId = zoneId;
this.eventId = eventId;
this.url = url;
this.extractMode = mode;
}
public String getUrl() {
@ -47,4 +60,12 @@ public class ExtractTemplateParam {
return eventId;
}
public Upload.Mode getExtractMode() {
return extractMode;
}
public void setExtractMode(Upload.Mode extractMode) {
this.extractMode = extractMode;
}
}

7
server/src/com/cloud/server/ManagementServerImpl.java
View File

@ -219,6 +219,8 @@ import com.cloud.storage.GuestOSCategoryVO;
import com.cloud.storage.GuestOSVO;
import com.cloud.storage.LaunchPermissionVO;
import com.cloud.storage.Snapshot;
import com.cloud.storage.Upload;
import com.cloud.storage.Volume;
import com.cloud.storage.Snapshot.SnapshotType;
import com.cloud.storage.SnapshotPolicyVO;
import com.cloud.storage.SnapshotVO;
@ -231,7 +233,6 @@ import com.cloud.storage.StorageStats;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.Volume;
import com.cloud.storage.Volume.VolumeType;
import com.cloud.storage.VolumeStats;
import com.cloud.storage.VolumeVO;
@ -248,6 +249,7 @@ import com.cloud.storage.dao.UploadDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateDao.TemplateFilter;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.download.DownloadMonitor;
import com.cloud.storage.preallocatedlun.PreallocatedLunVO;
import com.cloud.storage.preallocatedlun.dao.PreallocatedLunDao;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
@ -464,7 +466,7 @@ public class ManagementServerImpl implements ManagementServer {
_tmpltMgr = locator.getManager(TemplateManager.class);
_snapMgr = locator.getManager(SnapshotManager.class);
_networkGroupMgr = locator.getManager(NetworkGroupManager.class);
_uploadMonitor = locator.getManager(UploadMonitor.class);
_uploadMonitor = locator.getManager(UploadMonitor.class);
_userAuthenticators = locator.getAdapters(UserAuthenticator.class);
if (_userAuthenticators == null || !_userAuthenticators.isSet()) {
@ -3536,6 +3538,7 @@ public class ManagementServerImpl implements ManagementServer {
} else {
domainId = ((account == null) ? DomainVO.ROOT_DOMAIN : account.getDomainId());
}
} else {
accountName = account.getAccountName();
accountId = account.getId();

4
server/src/com/cloud/storage/upload/UploadListener.java
View File

@ -372,9 +372,9 @@ public class UploadListener implements Listener {
asyncMgr.updateAsyncJobAttachment(asyncJobId, type.toString(), 1L);
asyncMgr.updateAsyncJobStatus(asyncJobId, AsyncJobResult.STATUS_IN_PROGRESS, resultObj);
}else if(answer.getUploadStatus() == Status.UPLOADED){
asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_SUCCEEDED, 0, resultObj);
asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_SUCCEEDED, 1, resultObj);
}else{
asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_FAILED, 0, resultObj);
asyncMgr.completeAsyncJob(asyncJobId, AsyncJobResult.STATUS_FAILED, 2, resultObj);
}
UploadVO updateBuilder = uploadDao.createForUpdate();
updateBuilder.setUploadPercent(answer.getUploadPct());

4
server/src/com/cloud/storage/upload/UploadMonitor.java
View File

@ -21,6 +21,7 @@ package com.cloud.storage.upload;
import java.util.Map;
import com.cloud.async.AsyncJobManager;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.HostVO;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateHostVO;
@ -55,4 +56,7 @@ public interface UploadMonitor extends Manager{
Long dataCenterId, String installPath, long eventId,
long asyncJobId, AsyncJobManager asyncMgr);
UploadVO createEntityDownloadURL(VMTemplateVO template,
VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId) throws InternalErrorException;
}

152
server/src/com/cloud/storage/upload/UploadMonitorImpl.java
View File

@ -1,11 +1,16 @@
package com.cloud.storage.upload;
import java.io.File;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.ejb.Local;
import javax.naming.ConfigurationException;
@ -15,6 +20,8 @@ import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
import com.cloud.agent.api.storage.UploadCommand;
import com.cloud.agent.api.storage.UploadProgressCommand.RequestType;
import com.cloud.async.AsyncJobManager;
@ -22,20 +29,28 @@ import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.event.EventTypes;
import com.cloud.event.EventVO;
import com.cloud.event.dao.EventDao;
import com.cloud.exception.InternalErrorException;
import com.cloud.host.Host;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Upload;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Upload.Mode;
import com.cloud.storage.Upload.Status;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.dao.UploadDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateHostDao;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.component.Inject;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.GlobalLock;
import com.cloud.vm.SecondaryStorageVmVO;
import com.cloud.vm.State;
import com.cloud.vm.dao.SecondaryStorageVmDao;
/**
@ -70,7 +85,7 @@ public class UploadMonitorImpl implements UploadMonitor {
private String _name;
private Boolean _sslCopy = new Boolean(false);
private String _copyAuthPasswd;
private ScheduledExecutorService _executor = null;
Timer _timer;
@ -159,6 +174,72 @@ public class UploadMonitorImpl implements UploadMonitor {
}
}
@Override
public UploadVO createEntityDownloadURL(VMTemplateVO template, VMTemplateHostVO vmTemplateHost, Long dataCenterId, long eventId) throws InternalErrorException{
List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
if(storageServers == null )
throw new InternalErrorException("No Storage Server found at the datacenter - " +dataCenterId);
Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE ;
//Check if it already exists.
List<UploadVO> extractURLList = _uploadDao.listByTypeUploadStatus(template.getId(), type, UploadVO.Status.DOWNLOAD_URL_CREATED);
if (extractURLList.size() > 0)
return extractURLList.get(0);
// It doesn't exist so create a DB entry.
HostVO sserver = storageServers.get(0);
UploadVO uploadTemplateObj = new UploadVO(sserver.getId(), template.getId(), new Date(),
Status.DOWNLOAD_URL_NOT_CREATED, 0, type, Mode.HTTP_DOWNLOAD);
_uploadDao.persist(uploadTemplateObj);
// Create Symlink at ssvm
CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(vmTemplateHost.getInstallPath());
long result = send(sserver.getId(), cmd, null);
if (result == -1){
s_logger.warn("Unable to create a link for the template/iso ");
throw new InternalErrorException("Unable to create a link at the SSVM");
}
//Construct actual URL locally now that the symlink exists at SSVM
List<SecondaryStorageVmVO> ssVms = _secStorageVmDao.getSecStorageVmListInStates(dataCenterId, State.Running);
if (ssVms.size() > 0) {
SecondaryStorageVmVO ssVm = ssVms.get(0);
if (ssVm.getPublicIpAddress() == null) {
s_logger.warn("A running secondary storage vm has a null public ip?");
throw new InternalErrorException("SSVM has null public IP - couldnt create the URL");
}
String extractURL = generateCopyUrl(ssVm.getPublicIpAddress(), vmTemplateHost.getInstallPath());
UploadVO vo = _uploadDao.createForUpdate();
vo.setLastUpdated(new Date());
vo.setUploadUrl(extractURL);
vo.setUploadState(Status.DOWNLOAD_URL_CREATED);
if(extractURL == null){
vo.setUploadState(Status.ERROR);
vo.setErrorString("Could not create the download URL");
}
_uploadDao.update(uploadTemplateObj.getId(), vo);
return _uploadDao.findById(uploadTemplateObj.getId(), true);
}
throw new InternalErrorException("Couldnt find a running SSVM in the zone" + dataCenterId+ ".couldnt create the extraction URL.");
}
private String generateCopyUrl(String ipAddress, String path){
String hostname = ipAddress;
String scheme = "http";
if (_sslCopy) {
hostname = ipAddress.replace(".", "-");
hostname = hostname + ".realhostip.com";
scheme = "https";
}
return scheme + "://" + hostname + path.substring(path.lastIndexOf(File.separator));
}
public long send(Long hostId, Command cmd, Listener listener) {
@ -182,6 +263,10 @@ public class UploadMonitorImpl implements UploadMonitor {
_copyAuthPasswd = configs.get("secstorage.copy.password");
_agentMgr.registerForHostEvents(new UploadListener(this), true, false, false);
String workers = (String)params.get("expunge.workers");
int wrks = NumbersUtil.parseInt(workers, 1);
_executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("UploadMonitor-Scavenger"));
return true;
}
@ -192,6 +277,8 @@ public class UploadMonitorImpl implements UploadMonitor {
@Override
public boolean start() {
//FIX ME - Make the timings configurable.
_executor.scheduleWithFixedDelay(new StorageGarbageCollector(), 86400, 86400, TimeUnit.SECONDS);
_timer = new Timer();
return true;
}
@ -267,5 +354,62 @@ public class UploadMonitorImpl implements UploadMonitor {
}
protected class StorageGarbageCollector implements Runnable {
public StorageGarbageCollector() {
}
@Override
public void run() {
try {
s_logger.info("Extract Monitor Garbage Collection Thread is running.");
GlobalLock scanLock = GlobalLock.getInternLock(this.getClass().getName());
try {
if (scanLock.lock(3)) {
try {
cleanupStorage();
} finally {
scanLock.unlock();
}
}
} finally {
scanLock.releaseRef();
}
} catch (Exception e) {
s_logger.error("Caught the following Exception", e);
}
}
}
private long getTimeDiff(Date date){
Calendar currentCalendar = Calendar.getInstance();
Calendar givenDateCalendar = Calendar.getInstance();
givenDateCalendar.setTime(date);
return (currentCalendar.getTimeInMillis() - givenDateCalendar.getTimeInMillis() )/1000;
}
public void cleanupStorage() {
final int EXTRACT_URL_TIME_LIMIT = 40;
List<UploadVO> extractURLs= _uploadDao.listByModeAndStatus(Mode.HTTP_DOWNLOAD, Status.DOWNLOAD_URL_CREATED);
for (UploadVO extractURL : extractURLs){
if( getTimeDiff(extractURL.getLastUpdated()) < EXTRACT_URL_TIME_LIMIT) continue;
String path = extractURL.getUploadUrl().substring( (extractURL.getUploadUrl().lastIndexOf("/")) +1 );
DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(path);
long result = send(extractURL.getHostId(), cmd, null);
if (result == -1){
s_logger.warn("Unable to delete the link for " +extractURL.getType()+ " id=" +extractURL.getTypeId()+ " url="+extractURL.getUploadUrl());
}else{
_uploadDao.remove(extractURL.getId());
}
}
}
}

4
server/src/com/cloud/template/TemplateManager.java
View File

@ -192,6 +192,6 @@ public interface TemplateManager extends Manager {
boolean deleteIso(DeleteIsoCmd cmd) throws InvalidParameterValueException, InternalErrorException, PermissionDeniedException;
void extract(VMTemplateVO template, String url, VMTemplateHostVO tmpltHostRef, Long zoneId, long eventId, long asyncJobId, AsyncJobManager asyncMgr);
void extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException;
void extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException;
Long extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException;
Long extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException;
}

126
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -49,7 +49,9 @@ import com.cloud.api.commands.ExtractTemplateCmd;
import com.cloud.api.commands.RegisterIsoCmd;
import com.cloud.api.commands.RegisterTemplateCmd;
import com.cloud.async.AsyncJobManager;
import com.cloud.async.AsyncJobResult;
import com.cloud.async.AsyncJobVO;
import com.cloud.async.executor.ExtractJobResultObject;
import com.cloud.configuration.ResourceCount.ResourceType;
import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.dc.DataCenterVO;
@ -76,6 +78,8 @@ import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.StoragePoolVO;
import com.cloud.storage.Upload;
import com.cloud.storage.UploadVO;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateStoragePoolVO;
@ -87,6 +91,7 @@ import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.StoragePoolDao;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.UploadDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateHostDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
@ -146,6 +151,7 @@ public class TemplateManagerImpl implements TemplateManager {
@Inject VolumeDao _volumeDao;
@Inject SnapshotDao _snapshotDao;
@Inject DomainDao _domainDao;
@Inject UploadDao _uploadDao;
long _routerTemplateId = -1;
@Inject StorageManager _storageMgr;
@Inject UserVmManager _vmMgr;
@ -405,26 +411,30 @@ public class TemplateManagerImpl implements TemplateManager {
}
@Override
public void extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException {
public Long extract(ExtractIsoCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException {
Account account = (Account)UserContext.current().getAccountObject();
Long templateId = cmd.getId();
Long zoneId = cmd.getZoneId();
String url = cmd.getUrl();
extract(account, templateId, url, zoneId, true, cmd.getJob(), cmd.getAsyncJobManager());
String mode = cmd.getMode();
Long eventId = cmd.getStartEventId();
return extract(account, templateId, url, zoneId, mode, eventId, true, cmd.getJob(), cmd.getAsyncJobManager());
}
@Override
public void extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException {
public Long extract(ExtractTemplateCmd cmd) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException {
Account account = (Account)UserContext.current().getAccountObject();
Long templateId = cmd.getId();
Long zoneId = cmd.getZoneId();
String url = cmd.getUrl();
String mode = cmd.getMode();
Long eventId = cmd.getStartEventId();
extract(account, templateId, url, zoneId, true, cmd.getJob(), cmd.getAsyncJobManager());
return extract(account, templateId, url, zoneId, mode, eventId, false, cmd.getJob(), cmd.getAsyncJobManager());
}
private void extract(Account account, Long templateId, String url, Long zoneId, boolean isISO, AsyncJobVO job, AsyncJobManager mgr) throws InvalidParameterValueException, PermissionDeniedException {
private Long extract(Account account, Long templateId, String url, Long zoneId, String mode, Long eventId, boolean isISO, AsyncJobVO job, AsyncJobManager mgr) throws InvalidParameterValueException, PermissionDeniedException, InternalErrorException {
String desc = "template";
if (isISO) {
desc = "ISO";
@ -432,10 +442,10 @@ public class TemplateManagerImpl implements TemplateManager {
VMTemplateVO template = _tmpltDao.findById(templateId);
if (template == null) {
throw new InvalidParameterValueException("Unable to find template with id " + templateId);
throw new InvalidParameterValueException("Unable to find " +desc+ " with id " + templateId);
}
if (template.getName().startsWith("xs-tools") ){
throw new InvalidParameterValueException("Unable to extract the " + desc + " " + template.getName() + " It is not allowed");
throw new InvalidParameterValueException("Unable to extract the ISO " + template.getName() + " It is not allowed");
}
if (isISO) {
if (template.getFormat() != ImageFormat.ISO ){
@ -451,33 +461,6 @@ public class TemplateManagerImpl implements TemplateManager {
throw new IllegalArgumentException("Please specify a valid zone.");
}
if (url.toLowerCase().contains("file://")){
throw new InvalidParameterValueException("file:// type urls are currently unsupported");
}
URI uri = null;
try {
uri = new URI(url);
if ((uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp") )) {
throw new InvalidParameterValueException("Unsupported scheme for url: " + url);
}
} catch (URISyntaxException ex) {
throw new InvalidParameterValueException("Invalid url given: " + url);
}
String host = uri.getHost();
try {
InetAddress hostAddr = InetAddress.getByName(host);
if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress() ) {
throw new InvalidParameterValueException("Illegal host specified in url");
}
if (hostAddr instanceof Inet6Address) {
throw new InvalidParameterValueException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
}
} catch (UnknownHostException uhe) {
throw new InvalidParameterValueException("Unable to resolve " + host);
}
if (account != null) {
if(!isAdmin(account.getType())){
if (template.getAccountId() != account.getId()){
@ -486,7 +469,7 @@ public class TemplateManagerImpl implements TemplateManager {
} else {
Account userAccount = _accountDao.findById(template.getAccountId());
if((userAccount == null) || !_domainDao.isChildDomain(account.getDomainId(), userAccount.getDomainId())) {
throw new PermissionDeniedException("Unable to extract " + desc + " " + templateId + " to " + url + ", permission denied.");
throw new PermissionDeniedException("Unable to extract " + desc + "=" + templateId + " - permission denied.");
}
}
}
@ -499,21 +482,68 @@ public class TemplateManagerImpl implements TemplateManager {
throw new InvalidParameterValueException("The " + desc + " has not been downloaded ");
}
}
if ( _uploadMonitor.isTypeUploadInProgress(templateId, isISO ? Type.ISO : Type.TEMPLATE) ){
throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same");
Upload.Mode extractMode;
if( mode == null || (!mode.equals(Upload.Mode.FTP_UPLOAD.toString()) && !mode.equals(Upload.Mode.HTTP_DOWNLOAD.toString())) ){
throw new ServerApiException(BaseCmd.PARAM_ERROR, "Please specify a valid extract Mode ");
}else{
extractMode = mode.equals(Upload.Mode.FTP_UPLOAD.toString()) ? Upload.Mode.FTP_UPLOAD : Upload.Mode.HTTP_DOWNLOAD;
}
long userId = UserContext.current().getUserId();
long accountId = template.getAccountId();
String event = isISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD;
long eventId = EventUtils.saveScheduledEvent(userId, accountId, event, "Extraction job");
// FIXME: scheduled event should've already been saved, we should be saving this started event here...
// String event = template.getFormat() == ImageFormat.ISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD;
// EventUtils.saveStartedEvent(template.getAccountId(), template.getAccountId(), event, "Starting upload of " +template.getName()+ " to " +url, cmd.getStartEventId());
extract(template, url, tmpltHostRef, zoneId, eventId, job.getId(), mgr);
String event = isISO ? EventTypes.EVENT_ISO_EXTRACT : EventTypes.EVENT_TEMPLATE_EXTRACT;
if (extractMode == Upload.Mode.FTP_UPLOAD){
URI uri = null;
try {
uri = new URI(url);
if ((uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp") )) {
throw new InvalidParameterValueException("Unsupported scheme for url: " + url);
}
} catch (URISyntaxException ex) {
throw new InvalidParameterValueException("Invalid url given: " + url);
}
String host = uri.getHost();
try {
InetAddress hostAddr = InetAddress.getByName(host);
if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress() ) {
throw new InvalidParameterValueException("Illegal host specified in url");
}
if (hostAddr instanceof Inet6Address) {
throw new InvalidParameterValueException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
}
} catch (UnknownHostException uhe) {
throw new InvalidParameterValueException("Unable to resolve " + host);
}
if ( _uploadMonitor.isTypeUploadInProgress(templateId, isISO ? Type.ISO : Type.TEMPLATE) ){
throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same");
}
//long eventId = EventUtils.saveScheduledEvent(userId, accountId, event, "Extraction job");
// FIXME: scheduled event should've already been saved, we should be saving this started event here...
// String event = template.getFormat() == ImageFormat.ISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD;
// EventUtils.saveStartedEvent(template.getAccountId(), template.getAccountId(), event, "Starting upload of " +template.getName()+ " to " +url, cmd.getStartEventId());
EventUtils.saveStartedEvent(userId, accountId, event, "Starting extraction of " +template.getName()+ " mode= " +extractMode.toString(), eventId);
extract(template, url, tmpltHostRef, zoneId, eventId, job.getId(), mgr);
return 1L; //FIX ME
}
EventUtils.saveStartedEvent(userId, accountId, event, "Starting extraction of " +template.getName()+ " in mode:" +extractMode.toString(), eventId);
UploadVO vo = _uploadMonitor.createEntityDownloadURL(template, tmpltHostRef, zoneId, eventId);
if (vo!=null){
ExtractJobResultObject resultObject = new ExtractJobResultObject(template.getAccountId(), template.getName(), Upload.Status.DOWNLOAD_URL_CREATED.toString(), vo.getId(), url);
mgr.completeAsyncJob(job.getId(), AsyncJobResult.STATUS_SUCCEEDED, 1, resultObject);
EventUtils.saveEvent(userId, accountId, EventVO.LEVEL_INFO, event, "Completed extraction of "+template.getName()+ " in mode:" +extractMode.toString(), null, eventId);
return vo.getId();
}else{
EventUtils.saveEvent(userId, accountId, EventVO.LEVEL_ERROR, event, "Failed extraction of "+template.getName()+ " in mode:" +extractMode.toString(), null, eventId);
mgr.completeAsyncJob(job.getId(), AsyncJobResult.STATUS_FAILED, 2, null);
return null;
}
}
@Override

1
setup/db/create-schema.sql
View File

@ -633,6 +633,7 @@ CREATE TABLE `cloud`.`upload` (
`host_id` bigint unsigned NOT NULL,
`type_id` bigint unsigned NOT NULL,
`type` varchar(255),
`mode` varchar(255),
`created` DATETIME NOT NULL,
`last_updated` DATETIME,
`job_id` varchar(255),