etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bug 4286: Making ExtractVolumeCommand async and some code cleanup

This commit is contained in:
nit 2010-09-19 18:34:12 +05:30
parent 3b8c811143
commit 33e683e2c7
11 changed files with 113 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/src/com/cloud/server/ManagementServer.java
View File

@ -2242,14 +2242,16 @@ public interface ManagementServer {
* Extracts the volume to a particular location.
* @param url - the url where the volume needs to be extracted to
* @param zoneId - zone id of the volume
* @param asyncJobId
* @param eventId
* @param volume id - the id of the volume
* @throws URISyntaxException
* @throws InternalErrorException
*
*/
void extractVolume(String url, Long volumeId, Long zoneId) throws
void extractVolume(String url, Long volumeId, Long zoneId, long eventId, Long asyncJobId) throws
URISyntaxException, InternalErrorException;
long extractTemplateAsync(String url, Long templateId, Long zoneId) throws URISyntaxException;
long extractVolumeAsync(String url, Long volumeId, Long zoneId) throws URISyntaxException;
}

33
core/src/com/cloud/storage/template/FtpTemplateUploader.java
View File

@ -19,7 +19,7 @@ public class FtpTemplateUploader implements TemplateUploader {
public TemplateUploader.Status status = TemplateUploader.Status.NOT_STARTED;
public String errorString = "";
public long totalBytes = 0;
public long templateSizeinBytes;
public long entitySizeinBytes;
private String sourcePath;
private String ftpUrl;
private UploadCompleteCallback completionCallback;
@ -28,12 +28,12 @@ public class FtpTemplateUploader implements TemplateUploader {
private BufferedOutputStream outputStream = null;
private static final int CHUNK_SIZE = 1024*1024; //1M
public FtpTemplateUploader(String sourcePath, String url, UploadCompleteCallback callback, long templateSizeinBytes){
public FtpTemplateUploader(String sourcePath, String url, UploadCompleteCallback callback, long entitySizeinBytes){
this.sourcePath = sourcePath;
this.ftpUrl = url;
this.completionCallback = callback;
this.templateSizeinBytes = templateSizeinBytes;
this.entitySizeinBytes = entitySizeinBytes;
}
@ -51,7 +51,7 @@ public class FtpTemplateUploader implements TemplateUploader {
Date start = new Date();
StringBuffer sb = new StringBuffer();
StringBuffer sb = new StringBuffer(ftpUrl);
// check for authentication else assume its anonymous access.
/* if (user != null && password != null)
{
@ -59,11 +59,8 @@ public class FtpTemplateUploader implements TemplateUploader {
sb.append( ':' );
sb.append( password );
sb.append( '@' );
}*/
sb.append( ftpUrl );
/*sb.append( '/' );
sb.append( fileName ); filename where u want to dld it */
/*ftp://10.91.18.14/
}*/
/*
* type ==> a=ASCII mode, i=image (binary) mode, d= file directory
* listing
*/
@ -74,7 +71,7 @@ public class FtpTemplateUploader implements TemplateUploader {
URL url = new URL( sb.toString() );
URLConnection urlc = url.openConnection();
File sourceFile = new File(sourcePath);
templateSizeinBytes = sourceFile.length();
entitySizeinBytes = sourceFile.length();
outputStream = new BufferedOutputStream( urlc.getOutputStream() );
inputStream = new BufferedInputStream( new FileInputStream(sourceFile) );
@ -148,20 +145,20 @@ public class FtpTemplateUploader implements TemplateUploader {
@Override
public String getUploadLocalPath() {
return null;
return sourcePath;
}
@Override
public int getUploadPercent() {
if (templateSizeinBytes == 0) {
if (entitySizeinBytes == 0) {
return 0;
}
return (int)(100.0*totalBytes/templateSizeinBytes);
return (int)(100.0*totalBytes/entitySizeinBytes);
}
@Override
public long getUploadTime() {
// TODO Auto-generated method stub
// TODO
return 0;
}
@ -170,11 +167,6 @@ public class FtpTemplateUploader implements TemplateUploader {
return totalBytes;
}
@Override
public boolean isInited() {
return false;
}
@Override
public void setResume(boolean resume) {
this.resume = resume;
@ -219,7 +211,6 @@ public class FtpTemplateUploader implements TemplateUploader {
default:
return true;
}
}
}
}

5
core/src/com/cloud/storage/template/TemplateUploader.java Normal file → Executable file
View File

@ -69,9 +69,6 @@ public interface TemplateUploader extends Runnable{
public void setUploadError(String string);
public void setResume(boolean resume);
public boolean isInited();
public void setResume(boolean resume);
}

27
core/src/com/cloud/storage/template/UploadManagerImpl.java Normal file → Executable file
View File

@ -53,9 +53,9 @@ public class UploadManagerImpl implements UploadManager {
}
private static class UploadJob {
private final TemplateUploader td;
private final TemplateUploader tu;
private final String jobId;
private final String tmpltName;
private final String name;
private final ImageFormat format;
private String tmpltPath;
private String description;
@ -65,11 +65,11 @@ public class UploadManagerImpl implements UploadManager {
private long templatesize;
private long id;
public UploadJob(TemplateUploader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix) {
public UploadJob(TemplateUploader tu, String jobId, long id, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix) {
super();
this.td = td;
this.tu = tu;
this.jobId = jobId;
this.tmpltName = tmpltName;
this.name = name;
this.format = format;
this.accountId = accountId;
this.description = descr;
@ -80,7 +80,7 @@ public class UploadManagerImpl implements UploadManager {
}
public TemplateUploader getTd() {
return td;
return tu;
}
public String getDescription() {
@ -92,14 +92,14 @@ public class UploadManagerImpl implements UploadManager {
}
public UploadJob(TemplateUploader td, String jobId, UploadCommand cmd) {
this.td = td;
this.tu = td;
this.jobId = jobId;
this.tmpltName = cmd.getName();
this.name = cmd.getName();
this.format = cmd.getFormat();
}
public TemplateUploader getTemplateUploader() {
return td;
return tu;
}
public String getJobId() {
@ -107,7 +107,7 @@ public class UploadManagerImpl implements UploadManager {
}
public String getTmpltName() {
return tmpltName;
return name;
}
public ImageFormat getFormat() {
@ -135,6 +135,13 @@ public class UploadManagerImpl implements UploadManager {
}
public void cleanup() {
if (tu != null) {
String upldPath = tu.getUploadLocalPath();
if (upldPath != null) {
File f = new File(upldPath);
f.delete();
}
}
}
public void setTemplatesize(long templatesize) {

3
server/src/com/cloud/api/commands/CreateDiskOfferingCmd.java Normal file → Executable file
View File

@ -27,6 +27,7 @@ import org.apache.log4j.Logger;
import com.cloud.api.BaseCmd;
import com.cloud.api.ServerApiException;
import com.cloud.domain.DomainVO;
import com.cloud.exception.InternalErrorException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.storage.DiskOfferingVO;
import com.cloud.user.User;
@ -76,6 +77,8 @@ public class CreateDiskOfferingCmd extends BaseCmd {
diskOffering = getManagementServer().createDiskOffering(userId, DomainVO.ROOT_DOMAIN, name, displayText, numGB.intValue(),tags);
} catch (InvalidParameterValueException ex) {
throw new ServerApiException (BaseCmd.VM_INVALID_PARAM_ERROR, ex.getMessage());
} catch (InternalErrorException ex) {
throw new ServerApiException (BaseCmd.VM_INVALID_PARAM_ERROR, ex.getMessage());
}
if (diskOffering == null) {

4
server/src/com/cloud/api/commands/ExtractIsoCmd.java
View File

@ -89,4 +89,8 @@ public class ExtractIsoCmd extends BaseCmd {
public List<Pair<Enum, Boolean>> getProperties() {
return s_properties;
}
public static String getStaticName() {
return "ExtractIso";
}
}

10
server/src/com/cloud/api/commands/ExtractVolumeCmd.java Normal file → Executable file
View File

@ -55,9 +55,9 @@ public class ExtractVolumeCmd extends BaseCmd {
throw new ServerApiException(BaseCmd.PARAM_ERROR, "Unable to extract volume " + volumeId + " to " + url + ", permission denied.");
}
}
long jobId;
try {
managementServer.extractVolume(url, volumeId, zoneId);
jobId = managementServer.extractVolumeAsync(url, volumeId, zoneId);
} catch (Exception e) {
s_logger.error(e.getMessage(), e);
throw new ServerApiException(BaseCmd.INTERNAL_ERROR, "Internal Error Extracting the volume " + e.getMessage());
@ -69,7 +69,7 @@ public class ExtractVolumeCmd extends BaseCmd {
response.add(new Pair<String, Object>(BaseCmd.Properties.URL.getName(), url));
response.add(new Pair<String, Object>(BaseCmd.Properties.ZONE_ID.getName(), zoneId));
response.add(new Pair<String, Object>(BaseCmd.Properties.ZONE_NAME.getName(), zone.getName()));
response.add(new Pair<String, Object>(BaseCmd.Properties.STATUS.getName(), "Processing"));
response.add(new Pair<String, Object>(BaseCmd.Properties.JOB_ID.getName(), jobId));
return response;
}
@ -83,4 +83,8 @@ public class ExtractVolumeCmd extends BaseCmd {
return s_properties;
}
public static String getStaticName() {
return "ExtractVolume";
}
}

6
server/src/com/cloud/async/executor/ExtractTemplateExecutor.java
View File

@ -23,10 +23,8 @@ public class ExtractTemplateExecutor extends BaseAsyncJobExecutor {
ExtractTemplateParam param = gson.fromJson(job.getCmdInfo(), ExtractTemplateParam.class);
ManagementServer managementServer = asyncMgr.getExecutorContext().getManagementServer();
try {
s_logger.warn("nitin extract template");
managementServer.extractTemplate(param.getUrl(), param.getTemplateId(), param.getZoneId(), param.getEventId(), getJob().getId());
//asyncMgr.completeAsyncJob(getJob().getId(), AsyncJobResult.STATUS_SUCCEEDED, 0, null);
try {
managementServer.extractTemplate(param.getUrl(), param.getTemplateId(), param.getZoneId(), param.getEventId(), getJob().getId());
} catch (Exception e) {
s_logger.warn("Unable to extract template: " + e.getMessage(), e);

55
server/src/com/cloud/server/ManagementServerImpl.java
View File

@ -51,6 +51,8 @@ import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import sun.nio.cs.ext.TIS_620;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.GetVncPortAnswer;
import com.cloud.agent.api.GetVncPortCommand;
@ -72,7 +74,9 @@ import com.cloud.api.commands.DeleteIsoCmd;
import com.cloud.api.commands.DeleteTemplateCmd;
import com.cloud.api.commands.DeleteUserCmd;
import com.cloud.api.commands.DeployVMCmd;
import com.cloud.api.commands.ExtractIsoCmd;
import com.cloud.api.commands.ExtractTemplateCmd;
import com.cloud.api.commands.ExtractVolumeCmd;
import com.cloud.api.commands.PrepareForMaintenanceCmd;
import com.cloud.api.commands.PreparePrimaryStorageForMaintenanceCmd;
import com.cloud.api.commands.ReconnectHostCmd;
@ -208,6 +212,7 @@ import com.cloud.storage.VolumeVO;
import com.cloud.storage.Snapshot.SnapshotType;
import com.cloud.storage.Storage.FileSystem;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.Volume.VolumeType;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.DiskTemplateDao;
@ -227,6 +232,7 @@ import com.cloud.storage.preallocatedlun.dao.PreallocatedLunDao;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.storage.snapshot.SnapshotScheduler;
import com.cloud.storage.upload.UploadMonitor;
import com.cloud.template.TemplateManager;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
@ -354,6 +360,7 @@ public class ManagementServerImpl implements ManagementServer {
private final PreallocatedLunDao _lunDao;
private final InstanceGroupDao _vmGroupDao;
private final InstanceGroupVMMapDao _groupVMMapDao;
private final UploadMonitor _uploadMonitor;
private final ScheduledExecutorService _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AccountChecker"));
private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker"));
@ -444,6 +451,7 @@ public class ManagementServerImpl implements ManagementServer {
_snapMgr = locator.getManager(SnapshotManager.class);
_snapshotScheduler = locator.getManager(SnapshotScheduler.class);
_networkGroupMgr = locator.getManager(NetworkGroupManager.class);
_uploadMonitor = locator.getManager(UploadMonitor.class);
_userAuthenticators = locator.getAdapters(UserAuthenticator.class);
if (_userAuthenticators == null || !_userAuthenticators.isSet()) {
@ -4805,9 +4813,9 @@ public class ManagementServerImpl implements ManagementServer {
}
@Override
public void extractVolume(String url, Long volumeId, Long zoneId) throws URISyntaxException, InternalErrorException{
URI uri = new URI(url);
public long extractVolumeAsync(String url, Long volumeId, Long zoneId) throws URISyntaxException{
URI uri = new URI(url);
if ( (uri.getScheme() == null) || (!uri.getScheme().equalsIgnoreCase("ftp") )) {
throw new IllegalArgumentException("Unsupported scheme for url: " + url);
}
@ -4829,6 +4837,31 @@ public class ManagementServerImpl implements ManagementServer {
throw new IllegalArgumentException("Please specify a valid zone.");
}
VolumeVO volume = findVolumeById(volumeId);
if ( _uploadMonitor.isTypeUploadInProgress(volumeId, Type.VOLUME) ){
throw new IllegalArgumentException(volume.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same");
}
long userId = UserContext.current().getUserId();
long accountId = volume.getAccountId();
long eventId = saveScheduledEvent(userId, accountId, EventTypes.EVENT_VOLUME_UPLOAD, "Extraction job");
ExtractTemplateParam param = new ExtractTemplateParam(userId, volumeId, zoneId, eventId , url);
Gson gson = GsonHelper.getBuilder().create();
AsyncJobVO job = new AsyncJobVO();
job.setUserId(userId);
job.setAccountId(accountId);
job.setCmd("ExtractVolume");
job.setCmdInfo(gson.toJson(param));
job.setCmdOriginator(ExtractVolumeCmd.getStaticName());
return _asyncMgr.submitAsyncJob(job);
}
@Override
public void extractVolume(String url, Long volumeId, Long zoneId, long eventId, Long asyncJobId) throws URISyntaxException, InternalErrorException{
VolumeVO volume = findVolumeById(volumeId);
String secondaryStorageURL = _storageMgr.getSecondaryStorageURL(zoneId);
StoragePoolVO srcPool = _poolDao.findById(volume.getPoolId());
@ -4839,10 +4872,10 @@ public class ManagementServerImpl implements ManagementServer {
CopyVolumeAnswer cvAnswer = (CopyVolumeAnswer) _agentMgr.easySend(sourceHostId, cvCmd);
if (cvAnswer == null || !cvAnswer.getResult()) {
throw new InternalErrorException("Failed to copy the volume from the source primary storage pool to secondary storage.");
throw new InternalErrorException("Failed to copy the volume from the source primary storage pool to secondary storage.");
}
_uploadMonitor.extractVolume(volume, url, zoneId, "volumes/"+volume.getId()+"/"+cvAnswer.getVolumePath()+".vhd", eventId, asyncJobId, _asyncMgr);
}
@ -4876,9 +4909,15 @@ public class ManagementServerImpl implements ManagementServer {
if (tmpltHostRef != null && tmpltHostRef.getDownloadState() != com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED){
throw new IllegalArgumentException("The template hasnt been downloaded ");
}
boolean isISO = template.getFormat() == ImageFormat.ISO;
if ( _uploadMonitor.isTypeUploadInProgress(templateId, isISO ? Type.ISO : Type.TEMPLATE) ){
throw new IllegalArgumentException(template.getName() + " upload is in progress. Please wait for some time to schedule another upload for the same");
}
long userId = UserContext.current().getUserId();
long accountId = template.getAccountId();
String event = template.getFormat() == ImageFormat.ISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD;
String event = isISO ? EventTypes.EVENT_ISO_UPLOAD : EventTypes.EVENT_TEMPLATE_UPLOAD;
long eventId = saveScheduledEvent(userId, accountId, event, "Extraction job");
ExtractTemplateParam param = new ExtractTemplateParam(userId, templateId, zoneId, eventId , url);
@ -4889,7 +4928,7 @@ public class ManagementServerImpl implements ManagementServer {
job.setAccountId(accountId);
job.setCmd("ExtractTemplate");
job.setCmdInfo(gson.toJson(param));
job.setCmdOriginator(ExtractTemplateCmd.getStaticName());
job.setCmdOriginator(isISO ? ExtractIsoCmd.getStaticName() : ExtractTemplateCmd.getStaticName());
return _asyncMgr.submitAsyncJob(job);

8
server/src/com/cloud/storage/upload/UploadMonitor.java
View File

@ -23,6 +23,8 @@ import java.util.Map;
import com.cloud.async.AsyncJobManager;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.Upload.Type;
import com.cloud.storage.template.TemplateInfo;
import com.cloud.utils.component.Manager;
@ -41,4 +43,10 @@ public interface UploadMonitor extends Manager{
void handleUploadTemplateSync(long sserverId,
Map<String, TemplateInfo> templateInfo);
void extractVolume(VolumeVO volume, String url, Long dataCenterId,
String installPath, long eventId, long asyncJobId,
AsyncJobManager asyncMgr);
boolean isTypeUploadInProgress(Long typeId, Type type);
}

24
server/src/com/cloud/storage/upload/UploadMonitorImpl.java
View File

@ -95,22 +95,20 @@ public class UploadMonitorImpl implements UploadMonitor {
@Override
public void cancelAllUploads(Long templateId) {
// TODO Auto-generated method stub
// TODO
}
private boolean isTypeUploadInProgress(Long typeId, Type type) {
@Override
public boolean isTypeUploadInProgress(Long typeId, Type type) {
List<UploadVO> uploadsInProgress =
_uploadDao.listByTypeUploadStatus(typeId, type, UploadVO.Status.UPLOAD_IN_PROGRESS);
return (uploadsInProgress.size() != 0);
}
public void extractVolume(VolumeVO volume, String url, Long dataCenterId, String installPath){
if ( isTypeUploadInProgress(volume.getId(), Type.VOLUME) ){
return;
}
@Override
public void extractVolume(VolumeVO volume, String url, Long dataCenterId, String installPath, long eventId, long asyncJobId, AsyncJobManager asyncMgr){
List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
HostVO sserver = storageServers.get(0);
@ -118,13 +116,11 @@ public class UploadMonitorImpl implements UploadMonitor {
UploadVO uploadVolumeObj = new UploadVO(sserver.getId(), volume.getId(), new Date(),
Upload.Status.NOT_UPLOADED, 0, Type.VOLUME,
null, "jobid0000", url);
_uploadDao.persist(uploadVolumeObj);
//_vmTemplateHostDao.updateUploadStatus(sserver.getId(), template.getId(), 0, VMTemplateStorageResourceAssoc.Status.NOT_UPLOADED, "jobid0000", url);
_uploadDao.persist(uploadVolumeObj);
start();
UploadCommand ucmd = new UploadCommand(url, volume.getId(), volume.getSize(), installPath, Type.VOLUME);
UploadListener ul = new UploadListener(sserver, _timer, _uploadDao, uploadVolumeObj.getId(), this, ucmd, volume.getAccountId(), volume.getName(), Type.VOLUME, dataCenterId, dataCenterId, null);
UploadListener ul = new UploadListener(sserver, _timer, _uploadDao, uploadVolumeObj.getId(), this, ucmd, volume.getAccountId(), volume.getName(), Type.VOLUME, eventId, asyncJobId, asyncMgr);
_listenerMap.put(uploadVolumeObj, ul);
long result = send(sserver.getId(), ucmd, ul);
@ -141,11 +137,7 @@ public class UploadMonitorImpl implements UploadMonitor {
VMTemplateHostVO vmTemplateHost,Long dataCenterId, long eventId, long asyncJobId, AsyncJobManager asyncMgr){
Type type = (template.getFormat() == ImageFormat.ISO) ? Type.ISO : Type.TEMPLATE ;
if (isTypeUploadInProgress(template.getId(), type) ){
return; // TO DO raise an exception.
}
List<HostVO> storageServers = _serverDao.listByTypeDataCenter(Host.Type.SecondaryStorage, dataCenterId);
HostVO sserver = storageServers.get(0);