etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

S3 template download test is working except state transition part.

This commit is contained in:
Min Chen 2013-05-03 14:59:13 -07:00
parent df7a56d63f
commit 23db72d8b4
8 changed files with 237 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java
View File

@ -32,10 +32,6 @@ import com.cloud.utils.exception.CloudRuntimeException;
public class LocalNfsSecondaryStorageResource extends
NfsSecondaryStorageResource {
public LocalNfsSecondaryStorageResource(){
_dlMgr = new DownloadManagerImpl();
}
@Override
public Answer executeRequest(Command cmd) {
if (cmd instanceof DownloadSystemTemplateCommand){

4
core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
View File

@ -144,8 +144,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
boolean _inSystemVM = false;
boolean _sslCopy = false;
DownloadManager _dlMgr;
UploadManager _upldMgr;
protected DownloadManager _dlMgr;
protected UploadManager _upldMgr;
private String _configSslScr;
private String _configAuthScr;
private String _configIpFirewallScr;

145
core/src/com/cloud/storage/template/DownloadManagerImpl.java
View File

@ -116,7 +116,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
private final long id;
private final ResourceType resourceType;
public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, ResourceType resourceType) {
public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId,
String descr, String cksum, String installPathPrefix, ResourceType resourceType) {
super();
this.td = td;
this.jobId = jobId;
@ -239,8 +240,14 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
private int installTimeoutPerGig = 180 * 60 * 1000;
private boolean _sslCopy;
public void setThreadPool(ExecutorService threadPool) {
this.threadPool = threadPool;
}
/**
* Get notified of change of job status. Executed in context of downloader thread
* Get notified of change of job status. Executed in context of downloader
* thread
*
* @param jobId
* the id of the job
@ -255,7 +262,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
TemplateDownloader td = dj.getTemplateDownloader();
s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status);
s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + td.getDownloadPercent());
s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct="
+ td.getDownloadPercent());
switch (status) {
case ABORTED:
@ -276,7 +284,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
break;
case DOWNLOAD_FINISHED:
if (!(td instanceof S3TemplateDownloader)) {
// we currently only create template.properties for NFS by running some post download script
// we currently only create template.properties for NFS by
// running some post download script
td.setDownloadError("Download success, starting install ");
String result = postDownload(jobId);
if (result != null) {
@ -304,21 +313,20 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
try {
digest = MessageDigest.getInstance("MD5");
is = new FileInputStream(f);
while( (read = is.read(buffer)) > 0) {
while ((read = is.read(buffer)) > 0) {
digest.update(buffer, 0, read);
}
byte[] md5sum = digest.digest();
BigInteger bigInt = new BigInteger(1, md5sum);
checksum = String.format("%032x",bigInt);
checksum = String.format("%032x", bigInt);
return checksum;
}catch(IOException e) {
} catch (IOException e) {
return null;
}catch (NoSuchAlgorithmException e) {
} catch (NoSuchAlgorithmException e) {
return null;
}
finally {
} finally {
try {
if(is != null)
if (is != null)
is.close();
} catch (IOException e) {
return null;
@ -327,7 +335,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
/**
* Post download activity (install and cleanup). Executed in context of downloader thread
* Post download activity (install and cleanup). Executed in context of
* downloader thread
*
* @throws IOException
*/
@ -337,12 +346,13 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
String resourcePath = null;
ResourceType resourceType = dnld.getResourceType();
// once template path is set, remove the parent dir so that the template is installed with a relative path
// once template path is set, remove the parent dir so that the template
// is installed with a relative path
String finalResourcePath = "";
if (resourceType == ResourceType.TEMPLATE){
if (resourceType == ResourceType.TEMPLATE) {
finalResourcePath += _templateDir + File.separator + dnld.getAccountId() + File.separator + dnld.getId() + File.separator;
resourcePath = dnld.getInstallPathPrefix() + dnld.getAccountId() + File.separator + dnld.getId() + File.separator;// dnld.getTmpltName();
}else {
} else {
finalResourcePath += _volumeDir + File.separator + dnld.getId() + File.separator;
resourcePath = dnld.getInstallPathPrefix() + dnld.getId() + File.separator;// dnld.getTmpltName();
}
@ -375,7 +385,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
// add options common to ISO and template
String extension = dnld.getFormat().getFileExtension();
String templateName = "";
if( extension.equals("iso")) {
if (extension.equals("iso")) {
templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_");
} else {
templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes()).toString();
@ -404,9 +414,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
// Set permissions for template/volume.properties
String propertiesFile = resourcePath;
if (resourceType == ResourceType.TEMPLATE){
if (resourceType == ResourceType.TEMPLATE) {
propertiesFile += "/template.properties";
}else{
} else {
propertiesFile += "/volume.properties";
}
File templateProperties = new File(propertiesFile);
@ -461,7 +471,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
@Override
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr,
String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
@ -489,15 +500,15 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
return jobId;
}
@Override
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr,
String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
UUID uuid = UUID.randomUUID();
String jobId = uuid.toString();
String tmpDir = "";
if(resourceType == ResourceType.TEMPLATE){
if (resourceType == ResourceType.TEMPLATE) {
tmpDir = installPathPrefix + File.separator + accountId + File.separator + id;
}else {
} else {
tmpDir = installPathPrefix + File.separator + id;
}
@ -507,10 +518,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
s_logger.warn("Unable to create " + tmpDir);
return "Unable to create " + tmpDir;
}
// TO DO - define constant for volume properties.
File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) :
_storage.getFile(tmpDir + File.separator + "volume.properties");
if ( file.exists() ) {
// TO DO - define constant for volume properties.
File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage
.getFile(tmpDir + File.separator + "volume.properties");
if (file.exists()) {
file.delete();
}
@ -528,7 +539,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
TemplateDownloader td;
if ((uri != null) && (uri.getScheme() != null)) {
if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy,
resourceType);
} else if (uri.getScheme().equalsIgnoreCase("file")) {
td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
} else if (uri.getScheme().equalsIgnoreCase("scp")) {
@ -633,16 +645,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
return convertStatus(getDownloadStatus(jobId));
}
@Override
public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) {
ResourceType resourceType = cmd.getResourceType();
if (cmd instanceof DownloadProgressCommand) {
return handleDownloadProgressCmd( resource, (DownloadProgressCommand) cmd);
return handleDownloadProgressCmd(resource, (DownloadProgressCommand) cmd);
}
if (cmd.getUrl() == null) {
return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download",
VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
}
if (cmd.getName() == null) {
@ -657,21 +669,23 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
user = cmd.getAuth().getUserName();
password = new String(cmd.getAuth().getPassword());
}
//TO DO - Define Volume max size as well
long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes());
// TO DO - Define Volume max size as well
long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd
.getMaxDownloadSizeInBytes());
String jobId = null;
if (dstore instanceof S3TO){
jobId = downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
}
else{
jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
if (dstore instanceof S3TO) {
jobId = downloadS3Template((S3TO) dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(),
cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
} else {
jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(),
cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType);
}
sleep();
if (jobId == null) {
return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
}
return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId),
getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId));
return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId));
}
private void sleep() {
@ -712,8 +726,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
break;
case PURGE:
td.stopDownload();
answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId),
getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId),
getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId),
getDownloadCheckSum(jobId));
jobs.remove(jobId);
return answer;
default:
@ -741,7 +756,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
}
private List<String> listVolumes(String rootdir) {
List<String> result = new ArrayList<String>();
@ -754,8 +768,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
return result;
}
private List<String> listTemplates(String rootdir) {
List<String> result = new ArrayList<String>();
@ -773,7 +785,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
Map<String, TemplateProp> result = new HashMap<String, TemplateProp>();
String templateDir = rootDir + File.separator + _templateDir;
if (! _storage.exists(templateDir)) {
if (!_storage.exists(templateDir)) {
_storage.mkdirs(templateDir);
}
@ -784,7 +796,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
try {
if (!loc.load()) {
s_logger.warn("Post download installation was not completed for " + path);
//loc.purge();
// loc.purge();
_storage.cleanup(path, templateDir);
continue;
}
@ -798,8 +810,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
if ((tInfo.size == tInfo.physicalSize) && (tInfo.installPath.endsWith(ImageFormat.OVA.getFileExtension()))) {
try {
Processor processor = _processors.get("VMDK Processor");
VmdkProcessor vmdkProcessor = (VmdkProcessor)processor;
long vSize = vmdkProcessor.getTemplateVirtualSize(path, tInfo.installPath.substring(tInfo.installPath.lastIndexOf(File.separator) + 1));
VmdkProcessor vmdkProcessor = (VmdkProcessor) processor;
long vSize = vmdkProcessor.getTemplateVirtualSize(path,
tInfo.installPath.substring(tInfo.installPath.lastIndexOf(File.separator) + 1));
tInfo.size = vSize;
loc.updateVirtualSize(vSize);
loc.save();
@ -830,7 +843,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>();
String volumeDir = rootDir + File.separator + _volumeDir;
if (! _storage.exists(volumeDir)) {
if (!_storage.exists(volumeDir)) {
_storage.mkdirs(volumeDir);
}
@ -841,7 +854,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
try {
if (!loc.load()) {
s_logger.warn("Post download installation was not completed for " + path);
//loc.purge();
// loc.purge();
_storage.cleanup(path, volumeDir);
continue;
}
@ -855,8 +868,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
if ((vInfo.size == vInfo.physicalSize) && (vInfo.installPath.endsWith(ImageFormat.OVA.getFileExtension()))) {
try {
Processor processor = _processors.get("VMDK Processor");
VmdkProcessor vmdkProcessor = (VmdkProcessor)processor;
long vSize = vmdkProcessor.getTemplateVirtualSize(path, vInfo.installPath.substring(vInfo.installPath.lastIndexOf(File.separator) + 1));
VmdkProcessor vmdkProcessor = (VmdkProcessor) processor;
long vSize = vmdkProcessor.getTemplateVirtualSize(path,
vInfo.installPath.substring(vInfo.installPath.lastIndexOf(File.separator) + 1));
vInfo.size = vSize;
loc.updateVirtualSize(vSize);
loc.save();
@ -929,7 +943,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
String value = null;
_storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey);
_storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey);
if (_storage == null) {
value = (String) params.get(StorageLayer.ClassConfigKey);
if (value == null) {
@ -948,12 +962,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
throw new ConfigurationException("Unable to instantiate " + value);
}
}
String useSsl = (String)params.get("sslcopy");
String useSsl = (String) params.get("sslcopy");
if (useSsl != null) {
_sslCopy = Boolean.parseBoolean(useSsl);
}
String inSystemVM = (String)params.get("secondary.storage.vm");
String inSystemVM = (String) params.get("secondary.storage.vm");
if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
s_logger.info("DownloadManager: starting additional services since we are inside system vm");
startAdditionalServices();
@ -1032,12 +1046,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
Script command = new Script("/bin/bash", s_logger);
String intf = "eth1";
command.add("-c");
command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" +
"iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;");
command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;"
+ "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;");
String result = command.execute();
if (result != null) {
s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result );
s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result);
return;
}
}
@ -1064,19 +1078,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
command.add("if [ -d /etc/apache2 ] ; then service apache2 stop; else service httpd stop; fi ");
String result = command.execute();
if (result != null) {
s_logger.warn("Error in stopping httpd service err=" + result );
s_logger.warn("Error in stopping httpd service err=" + result);
}
String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT);
String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF;
command = new Script("/bin/bash", s_logger);
command.add("-c");
command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" +
"iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;");
command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" + "iptables -I INPUT -i "
+ intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;");
result = command.execute();
if (result != null) {
s_logger.warn("Error in opening up httpd port err=" + result );
s_logger.warn("Error in opening up httpd port err=" + result);
return;
}
@ -1085,7 +1099,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd start; fi ");
result = command.execute();
if (result != null) {
s_logger.warn("Error in starting httpd service err=" + result );
s_logger.warn("Error in starting httpd service err=" + result);
return;
}
command = new Script("mkdir", s_logger);
@ -1093,10 +1107,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
command.add("/var/www/html/copy/template");
result = command.execute();
if (result != null) {
s_logger.warn("Error in creating directory =" + result );
s_logger.warn("Error in creating directory =" + result);
return;
}
}
}

37
core/src/com/cloud/storage/template/S3TemplateDownloader.java
View File

@ -40,14 +40,18 @@ import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NoHttpResponseException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.ProgressEvent;
import com.amazonaws.services.s3.model.ProgressListener;
@ -176,8 +180,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
port = 80;
}
URL urlObj = new URL(url);
this.fileName = urlObj.getFile();
this.fileName = StringUtils.substringAfterLast(url, "/");
String host = uri.getHost();
try {
@ -199,10 +202,7 @@ public class S3TemplateDownloader implements TemplateDownloader {
} catch (URISyntaxException use) {
s_logger.warn("Failed uri syntax check: " + use.getMessage());
throw new IllegalArgumentException(use.getMessage());
} catch (MalformedURLException e) {
s_logger.warn("Failed url syntax check: " + e.getMessage());
throw new IllegalArgumentException(e.getMessage());
}
}
}
@Override
@ -215,8 +215,17 @@ public class S3TemplateDownloader implements TemplateDownloader {
default:
}
int bytes=0;
try {
// execute get method
int responseCode = HttpStatus.SC_OK;
if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) ";
return 0; //FIXME: retry?
}
// get the total size of file
Header contentLengthHeader = request.getResponseHeader("Content-Length");
boolean chunked = false;
@ -277,14 +286,23 @@ public class S3TemplateDownloader implements TemplateDownloader {
if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) {
s_logger.info("download completed");
status = TemplateDownloader.Status.DOWNLOAD_FINISHED;
} else {
} else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE){
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR;
} else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE){
status = TemplateDownloader.Status.ABORTED;
} else{
status = TemplateDownloader.Status.IN_PROGRESS;
}
}
});
S3Utils.putObject(s3, putObjectRequest);
while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED &&
status != TemplateDownloader.Status.UNRECOVERABLE_ERROR &&
status != TemplateDownloader.Status.ABORTED ){
// wait for completion
}
// finished or aborted
Date finish = new Date();
String downloaded = "(incomplete download)";
if (totalBytes >= remoteSize) {
@ -300,6 +318,9 @@ public class S3TemplateDownloader implements TemplateDownloader {
} catch (IOException ioe) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error?
errorString = ioe.getMessage();
} catch (AmazonClientException ex) {
status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; // S3 api exception
errorString = ex.getMessage();
} finally {
// close input stream
request.releaseConnection();

103
engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java Normal file
View File

@ -0,0 +1,103 @@
package org.apache.cloudstack.storage;
import static com.cloud.utils.StringUtils.join;
import static java.util.Arrays.asList;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;
import java.util.concurrent.Executors;
import org.apache.cloudstack.storage.command.DownloadSystemTemplateCommand;
import org.springframework.stereotype.Component;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.Command;
import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.NfsTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.resource.NfsSecondaryStorageResource;
import com.cloud.storage.template.DownloadManagerImpl;
import com.cloud.utils.S3Utils;
import com.cloud.utils.UriUtils;
import com.cloud.utils.exception.CloudRuntimeException;
@Component
public class MockLocalNfsSecondaryStorageResource extends
NfsSecondaryStorageResource {
public MockLocalNfsSecondaryStorageResource(){
_dlMgr = new DownloadManagerImpl();
((DownloadManagerImpl)_dlMgr).setThreadPool(Executors.newFixedThreadPool(10));
}
@Override
public Answer executeRequest(Command cmd) {
if (cmd instanceof DownloadSystemTemplateCommand){
return execute((DownloadSystemTemplateCommand)cmd);
} else {
//return Answer.createUnsupportedCommandAnswer(cmd);
return super.executeRequest(cmd);
}
}
private Answer execute(DownloadSystemTemplateCommand cmd){
DataStoreTO dstore = cmd.getDataStore();
if ( dstore instanceof S3TO ){
//TODO: how to handle download progress for S3
S3TO s3 = (S3TO)cmd.getDataStore();
String url = cmd.getUrl();
String user = null;
String password = null;
if (cmd.getAuth() != null) {
user = cmd.getAuth().getUserName();
password = new String(cmd.getAuth().getPassword());
}
// get input stream from the given url
InputStream in = UriUtils.getInputStreamFromUrl(url, user, password);
URI uri;
URL urlObj;
try {
uri = new URI(url);
urlObj = new URL(url);
} catch (URISyntaxException e) {
throw new CloudRuntimeException("URI is incorrect: " + url);
} catch (MalformedURLException e) {
throw new CloudRuntimeException("URL is incorrect: " + url);
}
final String bucket = s3.getBucketName();
// convention is no / in the end for install path based on S3Utils implementation.
String path = determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId(), cmd.getName());
// template key is
// TEMPLATE_ROOT_DIR/account_id/template_id/template_name
String key = join(asList(path, urlObj.getFile()), S3Utils.SEPARATOR);
S3Utils.putObject(s3, in, bucket, key);
List<S3ObjectSummary> s3Obj = S3Utils.getDirectory(s3, bucket, path);
if (s3Obj == null || s3Obj.size() == 0) {
return new Answer(cmd, false, "Failed to download to S3 bucket: " + bucket + " with key: " + key);
} else {
return new DownloadAnswer(null, 100, null, Status.DOWNLOADED, path, path, s3Obj.get(0).getSize(), s3Obj.get(0).getSize(), s3Obj.get(0)
.getETag());
}
}
else if ( dstore instanceof NfsTO ){
return new Answer(cmd, false, "Nfs needs to be pre-installed with system vm templates");
}
else if ( dstore instanceof SwiftTO ){
//TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand) here, but we need to handle
// source is url, most likely we need to modify our existing swiftUpload python script.
return new Answer(cmd, false, "Swift is not currently support DownloadCommand");
}
else{
return new Answer(cmd, false, "Unsupported image data store: " + dstore);
}
}
}

8
engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java
View File

@ -18,6 +18,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.storage.LocalHostEndpoint;
import org.apache.cloudstack.storage.MockLocalNfsSecondaryStorageResource;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailVO;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
@ -108,8 +109,11 @@ public class S3TemplateTest extends CloudStackTestNGBase {
image = templateDao.persist(image);
templateId = image.getId();
Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(new LocalHostEndpoint());
Mockito.when(epSelector.select(Mockito.any(DataStore.class))).thenReturn(new LocalHostEndpoint());
// inject mockito
LocalHostEndpoint ep = new LocalHostEndpoint();
ep.setResource(new MockLocalNfsSecondaryStorageResource());
Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(ep);
Mockito.when(epSelector.select(Mockito.any(DataStore.class))).thenReturn(ep);
}
@Test

12
engine/storage/integration-test/test/resource/testng.xml
View File

@ -29,8 +29,16 @@
<parameter name="devcloud-host-uuid" value="759ee4c9-a15a-297b-67c6-ac267d8aa429"/>
<parameter name="devcloud-secondary-storage" value="nfs://192.168.56.10/opt/storage/secondary"/>
<listeners>
<parameter name="s3-accesskey" value="OYAZXCAFUC1DAFOXNJWI"/>
<parameter name="s3-secretkey" value="YHpRtzNDwl12DtrQmwRvdpnf2xK2AeVM30rXxQ=="/>
<parameter name="s3-endpoint" value="10.223.89.7:8080" />
<parameter name="s3-template-bucket" value="cstemplates"/>
<parameter name="s3-usehttps" value="false" />
<parameter name="image-install-path" value="template/tmpl/1/5/"/>
<listeners>
<listener class-name="org.apache.cloudstack.storage.test.TestNGAop" />
</listeners>

8
engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java
View File

@ -92,5 +92,13 @@ public class LocalHostEndpoint implements EndPoint {
executor.schedule(new CmdRunner2(cmd, listener.getCallback()), 10, TimeUnit.SECONDS);
}
}
public ServerResource getResource() {
return resource;
}
public void setResource(ServerResource resource) {
this.resource = resource;
}
}