From 23db72d8b40903e306a3e5953b7e39155829d81d Mon Sep 17 00:00:00 2001 From: Min Chen Date: Fri, 3 May 2013 14:59:13 -0700 Subject: [PATCH] S3 template download test is working except state transition part. --- .../LocalNfsSecondaryStorageResource.java | 4 - .../resource/NfsSecondaryStorageResource.java | 4 +- .../storage/template/DownloadManagerImpl.java | 145 ++++++++++-------- .../template/S3TemplateDownloader.java | 37 ++++- .../MockLocalNfsSecondaryStorageResource.java | 103 +++++++++++++ .../storage/test/S3TemplateTest.java | 8 +- .../integration-test/test/resource/testng.xml | 12 +- .../cloudstack/storage/LocalHostEndpoint.java | 8 + 8 files changed, 237 insertions(+), 84 deletions(-) create mode 100644 engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java diff --git a/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java index 06933cc4068..a2cf757b432 100644 --- a/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/LocalNfsSecondaryStorageResource.java @@ -32,10 +32,6 @@ import com.cloud.utils.exception.CloudRuntimeException; public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResource { - public LocalNfsSecondaryStorageResource(){ - _dlMgr = new DownloadManagerImpl(); - } - @Override public Answer executeRequest(Command cmd) { if (cmd instanceof DownloadSystemTemplateCommand){ diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index 47d5ce7bcba..0345f6b0e90 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -144,8 +144,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S boolean _inSystemVM = false; boolean _sslCopy = false; - DownloadManager _dlMgr; - UploadManager _upldMgr; + protected DownloadManager _dlMgr; + protected UploadManager _upldMgr; private String _configSslScr; private String _configAuthScr; private String _configIpFirewallScr; diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java index 02a62568856..cd6165d1e3b 100755 --- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java +++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java @@ -116,7 +116,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private final long id; private final ResourceType resourceType; - public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, ResourceType resourceType) { + public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, + String descr, String cksum, String installPathPrefix, ResourceType resourceType) { super(); this.td = td; this.jobId = jobId; @@ -239,8 +240,14 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private int installTimeoutPerGig = 180 * 60 * 1000; private boolean _sslCopy; + + public void setThreadPool(ExecutorService threadPool) { + this.threadPool = threadPool; + } + /** - * Get notified of change of job status. Executed in context of downloader thread + * Get notified of change of job status. Executed in context of downloader + * thread * * @param jobId * the id of the job @@ -255,7 +262,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } TemplateDownloader td = dj.getTemplateDownloader(); s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status); - s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + td.getDownloadPercent()); + s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + + td.getDownloadPercent()); switch (status) { case ABORTED: @@ -276,7 +284,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager break; case DOWNLOAD_FINISHED: if (!(td instanceof S3TemplateDownloader)) { - // we currently only create template.properties for NFS by running some post download script + // we currently only create template.properties for NFS by + // running some post download script td.setDownloadError("Download success, starting install "); String result = postDownload(jobId); if (result != null) { @@ -304,21 +313,20 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager try { digest = MessageDigest.getInstance("MD5"); is = new FileInputStream(f); - while( (read = is.read(buffer)) > 0) { + while ((read = is.read(buffer)) > 0) { digest.update(buffer, 0, read); } byte[] md5sum = digest.digest(); BigInteger bigInt = new BigInteger(1, md5sum); - checksum = String.format("%032x",bigInt); + checksum = String.format("%032x", bigInt); return checksum; - }catch(IOException e) { + } catch (IOException e) { return null; - }catch (NoSuchAlgorithmException e) { + } catch (NoSuchAlgorithmException e) { return null; - } - finally { + } finally { try { - if(is != null) + if (is != null) is.close(); } catch (IOException e) { return null; @@ -327,7 +335,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } /** - * Post download activity (install and cleanup). Executed in context of downloader thread + * Post download activity (install and cleanup). Executed in context of + * downloader thread * * @throws IOException */ @@ -337,12 +346,13 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager String resourcePath = null; ResourceType resourceType = dnld.getResourceType(); - // once template path is set, remove the parent dir so that the template is installed with a relative path + // once template path is set, remove the parent dir so that the template + // is installed with a relative path String finalResourcePath = ""; - if (resourceType == ResourceType.TEMPLATE){ + if (resourceType == ResourceType.TEMPLATE) { finalResourcePath += _templateDir + File.separator + dnld.getAccountId() + File.separator + dnld.getId() + File.separator; resourcePath = dnld.getInstallPathPrefix() + dnld.getAccountId() + File.separator + dnld.getId() + File.separator;// dnld.getTmpltName(); - }else { + } else { finalResourcePath += _volumeDir + File.separator + dnld.getId() + File.separator; resourcePath = dnld.getInstallPathPrefix() + dnld.getId() + File.separator;// dnld.getTmpltName(); } @@ -375,7 +385,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager // add options common to ISO and template String extension = dnld.getFormat().getFileExtension(); String templateName = ""; - if( extension.equals("iso")) { + if (extension.equals("iso")) { templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_"); } else { templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes()).toString(); @@ -404,9 +414,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager // Set permissions for template/volume.properties String propertiesFile = resourcePath; - if (resourceType == ResourceType.TEMPLATE){ + if (resourceType == ResourceType.TEMPLATE) { propertiesFile += "/template.properties"; - }else{ + } else { propertiesFile += "/volume.properties"; } File templateProperties = new File(propertiesFile); @@ -461,7 +471,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } @Override - public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, + String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); @@ -489,15 +500,15 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return jobId; } - @Override - public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, + String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { UUID uuid = UUID.randomUUID(); String jobId = uuid.toString(); String tmpDir = ""; - if(resourceType == ResourceType.TEMPLATE){ + if (resourceType == ResourceType.TEMPLATE) { tmpDir = installPathPrefix + File.separator + accountId + File.separator + id; - }else { + } else { tmpDir = installPathPrefix + File.separator + id; } @@ -507,10 +518,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager s_logger.warn("Unable to create " + tmpDir); return "Unable to create " + tmpDir; } - // TO DO - define constant for volume properties. - File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : - _storage.getFile(tmpDir + File.separator + "volume.properties"); - if ( file.exists() ) { + // TO DO - define constant for volume properties. + File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage + .getFile(tmpDir + File.separator + "volume.properties"); + if (file.exists()) { file.delete(); } @@ -528,7 +539,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager TemplateDownloader td; if ((uri != null) && (uri.getScheme() != null)) { if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { - td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType); + td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, + resourceType); } else if (uri.getScheme().equalsIgnoreCase("file")) { td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId)); } else if (uri.getScheme().equalsIgnoreCase("scp")) { @@ -633,16 +645,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return convertStatus(getDownloadStatus(jobId)); } - @Override public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) { ResourceType resourceType = cmd.getResourceType(); if (cmd instanceof DownloadProgressCommand) { - return handleDownloadProgressCmd( resource, (DownloadProgressCommand) cmd); + return handleDownloadProgressCmd(resource, (DownloadProgressCommand) cmd); } if (cmd.getUrl() == null) { - return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + return new DownloadAnswer(resourceType.toString() + " is corrupted on storage due to an invalid url , cannot download", + VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } if (cmd.getName() == null) { @@ -657,21 +669,23 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager user = cmd.getAuth().getUserName(); password = new String(cmd.getAuth().getPassword()); } - //TO DO - Define Volume max size as well - long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes()); + // TO DO - Define Volume max size as well + long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd + .getMaxDownloadSizeInBytes()); String jobId = null; - if (dstore instanceof S3TO){ - jobId = downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); - } - else{ - jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + if (dstore instanceof S3TO) { + jobId = downloadS3Template((S3TO) dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), + cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + } else { + jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), + cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); } sleep(); if (jobId == null) { return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } - return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId), - getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId)); + return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), + getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplateSize(jobId), getDownloadCheckSum(jobId)); } private void sleep() { @@ -712,8 +726,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager break; case PURGE: td.stopDownload(); - answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), - getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); + answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), + getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), + getDownloadCheckSum(jobId)); jobs.remove(jobId); return answer; default: @@ -741,7 +756,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } - private List listVolumes(String rootdir) { List result = new ArrayList(); @@ -754,8 +768,6 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return result; } - - private List listTemplates(String rootdir) { List result = new ArrayList(); @@ -773,7 +785,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager Map result = new HashMap(); String templateDir = rootDir + File.separator + _templateDir; - if (! _storage.exists(templateDir)) { + if (!_storage.exists(templateDir)) { _storage.mkdirs(templateDir); } @@ -784,7 +796,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager try { if (!loc.load()) { s_logger.warn("Post download installation was not completed for " + path); - //loc.purge(); + // loc.purge(); _storage.cleanup(path, templateDir); continue; } @@ -798,8 +810,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager if ((tInfo.size == tInfo.physicalSize) && (tInfo.installPath.endsWith(ImageFormat.OVA.getFileExtension()))) { try { Processor processor = _processors.get("VMDK Processor"); - VmdkProcessor vmdkProcessor = (VmdkProcessor)processor; - long vSize = vmdkProcessor.getTemplateVirtualSize(path, tInfo.installPath.substring(tInfo.installPath.lastIndexOf(File.separator) + 1)); + VmdkProcessor vmdkProcessor = (VmdkProcessor) processor; + long vSize = vmdkProcessor.getTemplateVirtualSize(path, + tInfo.installPath.substring(tInfo.installPath.lastIndexOf(File.separator) + 1)); tInfo.size = vSize; loc.updateVirtualSize(vSize); loc.save(); @@ -830,7 +843,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager Map result = new HashMap(); String volumeDir = rootDir + File.separator + _volumeDir; - if (! _storage.exists(volumeDir)) { + if (!_storage.exists(volumeDir)) { _storage.mkdirs(volumeDir); } @@ -841,7 +854,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager try { if (!loc.load()) { s_logger.warn("Post download installation was not completed for " + path); - //loc.purge(); + // loc.purge(); _storage.cleanup(path, volumeDir); continue; } @@ -855,8 +868,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager if ((vInfo.size == vInfo.physicalSize) && (vInfo.installPath.endsWith(ImageFormat.OVA.getFileExtension()))) { try { Processor processor = _processors.get("VMDK Processor"); - VmdkProcessor vmdkProcessor = (VmdkProcessor)processor; - long vSize = vmdkProcessor.getTemplateVirtualSize(path, vInfo.installPath.substring(vInfo.installPath.lastIndexOf(File.separator) + 1)); + VmdkProcessor vmdkProcessor = (VmdkProcessor) processor; + long vSize = vmdkProcessor.getTemplateVirtualSize(path, + vInfo.installPath.substring(vInfo.installPath.lastIndexOf(File.separator) + 1)); vInfo.size = vSize; loc.updateVirtualSize(vSize); loc.save(); @@ -929,7 +943,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager String value = null; - _storage = (StorageLayer)params.get(StorageLayer.InstanceConfigKey); + _storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey); if (_storage == null) { value = (String) params.get(StorageLayer.ClassConfigKey); if (value == null) { @@ -948,12 +962,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager throw new ConfigurationException("Unable to instantiate " + value); } } - String useSsl = (String)params.get("sslcopy"); + String useSsl = (String) params.get("sslcopy"); if (useSsl != null) { _sslCopy = Boolean.parseBoolean(useSsl); } - String inSystemVM = (String)params.get("secondary.storage.vm"); + String inSystemVM = (String) params.get("secondary.storage.vm"); if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) { s_logger.info("DownloadManager: starting additional services since we are inside system vm"); startAdditionalServices(); @@ -1032,12 +1046,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager Script command = new Script("/bin/bash", s_logger); String intf = "eth1"; command.add("-c"); - command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" + - "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;"); + command.add("iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "80" + " -j REJECT;" + + "iptables -A OUTPUT -o " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j REJECT;"); String result = command.execute(); if (result != null) { - s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result ); + s_logger.warn("Error in blocking outgoing to port 80/443 err=" + result); return; } } @@ -1064,19 +1078,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager command.add("if [ -d /etc/apache2 ] ; then service apache2 stop; else service httpd stop; fi "); String result = command.execute(); if (result != null) { - s_logger.warn("Error in stopping httpd service err=" + result ); + s_logger.warn("Error in stopping httpd service err=" + result); } String port = Integer.toString(TemplateConstants.DEFAULT_TMPLT_COPY_PORT); String intf = TemplateConstants.DEFAULT_TMPLT_COPY_INTF; command = new Script("/bin/bash", s_logger); command.add("-c"); - command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" + - "iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;"); + command.add("iptables -I INPUT -i " + intf + " -p tcp -m state --state NEW -m tcp --dport " + port + " -j ACCEPT;" + "iptables -I INPUT -i " + + intf + " -p tcp -m state --state NEW -m tcp --dport " + "443" + " -j ACCEPT;"); result = command.execute(); if (result != null) { - s_logger.warn("Error in opening up httpd port err=" + result ); + s_logger.warn("Error in opening up httpd port err=" + result); return; } @@ -1085,7 +1099,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager command.add("if [ -d /etc/apache2 ] ; then service apache2 start; else service httpd start; fi "); result = command.execute(); if (result != null) { - s_logger.warn("Error in starting httpd service err=" + result ); + s_logger.warn("Error in starting httpd service err=" + result); return; } command = new Script("mkdir", s_logger); @@ -1093,10 +1107,9 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager command.add("/var/www/html/copy/template"); result = command.execute(); if (result != null) { - s_logger.warn("Error in creating directory =" + result ); + s_logger.warn("Error in creating directory =" + result); return; } } - } diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java index 84b2b10f847..eac9bbf6922 100644 --- a/core/src/com/cloud/storage/template/S3TemplateDownloader.java +++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java @@ -40,14 +40,18 @@ import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpException; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethodRetryHandler; +import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.NoHttpResponseException; import org.apache.commons.httpclient.UsernamePasswordCredentials; import org.apache.commons.httpclient.auth.AuthScope; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.ProgressEvent; import com.amazonaws.services.s3.model.ProgressListener; @@ -176,8 +180,7 @@ public class S3TemplateDownloader implements TemplateDownloader { port = 80; } - URL urlObj = new URL(url); - this.fileName = urlObj.getFile(); + this.fileName = StringUtils.substringAfterLast(url, "/"); String host = uri.getHost(); try { @@ -199,10 +202,7 @@ public class S3TemplateDownloader implements TemplateDownloader { } catch (URISyntaxException use) { s_logger.warn("Failed uri syntax check: " + use.getMessage()); throw new IllegalArgumentException(use.getMessage()); - } catch (MalformedURLException e) { - s_logger.warn("Failed url syntax check: " + e.getMessage()); - throw new IllegalArgumentException(e.getMessage()); - } + } } @Override @@ -215,8 +215,17 @@ public class S3TemplateDownloader implements TemplateDownloader { default: } + + int bytes=0; try { + // execute get method + int responseCode = HttpStatus.SC_OK; + if ((responseCode = client.executeMethod(request)) != HttpStatus.SC_OK) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString = " HTTP Server returned " + responseCode + " (expected 200 OK) "; + return 0; //FIXME: retry? + } // get the total size of file Header contentLengthHeader = request.getResponseHeader("Content-Length"); boolean chunked = false; @@ -277,14 +286,23 @@ public class S3TemplateDownloader implements TemplateDownloader { if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) { s_logger.info("download completed"); status = TemplateDownloader.Status.DOWNLOAD_FINISHED; - } else { + } else if (progressEvent.getEventCode() == ProgressEvent.FAILED_EVENT_CODE){ + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + } else if (progressEvent.getEventCode() == ProgressEvent.CANCELED_EVENT_CODE){ + status = TemplateDownloader.Status.ABORTED; + } else{ status = TemplateDownloader.Status.IN_PROGRESS; } - } }); S3Utils.putObject(s3, putObjectRequest); + while (status != TemplateDownloader.Status.DOWNLOAD_FINISHED && + status != TemplateDownloader.Status.UNRECOVERABLE_ERROR && + status != TemplateDownloader.Status.ABORTED ){ + // wait for completion + } + // finished or aborted Date finish = new Date(); String downloaded = "(incomplete download)"; if (totalBytes >= remoteSize) { @@ -300,6 +318,9 @@ public class S3TemplateDownloader implements TemplateDownloader { } catch (IOException ioe) { status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? errorString = ioe.getMessage(); + } catch (AmazonClientException ex) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; // S3 api exception + errorString = ex.getMessage(); } finally { // close input stream request.releaseConnection(); diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java new file mode 100644 index 00000000000..a5c0af1ef20 --- /dev/null +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/MockLocalNfsSecondaryStorageResource.java @@ -0,0 +1,103 @@ +package org.apache.cloudstack.storage; + +import static com.cloud.utils.StringUtils.join; +import static java.util.Arrays.asList; + +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.List; +import java.util.concurrent.Executors; + +import org.apache.cloudstack.storage.command.DownloadSystemTemplateCommand; +import org.springframework.stereotype.Component; + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.S3TO; +import com.cloud.agent.api.to.SwiftTO; +import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.resource.NfsSecondaryStorageResource; +import com.cloud.storage.template.DownloadManagerImpl; +import com.cloud.utils.S3Utils; +import com.cloud.utils.UriUtils; +import com.cloud.utils.exception.CloudRuntimeException; + +@Component +public class MockLocalNfsSecondaryStorageResource extends + NfsSecondaryStorageResource { + + public MockLocalNfsSecondaryStorageResource(){ + _dlMgr = new DownloadManagerImpl(); + ((DownloadManagerImpl)_dlMgr).setThreadPool(Executors.newFixedThreadPool(10)); + } + + @Override + public Answer executeRequest(Command cmd) { + if (cmd instanceof DownloadSystemTemplateCommand){ + return execute((DownloadSystemTemplateCommand)cmd); + } else { + //return Answer.createUnsupportedCommandAnswer(cmd); + return super.executeRequest(cmd); + } + } + + private Answer execute(DownloadSystemTemplateCommand cmd){ + DataStoreTO dstore = cmd.getDataStore(); + if ( dstore instanceof S3TO ){ + //TODO: how to handle download progress for S3 + S3TO s3 = (S3TO)cmd.getDataStore(); + String url = cmd.getUrl(); + String user = null; + String password = null; + if (cmd.getAuth() != null) { + user = cmd.getAuth().getUserName(); + password = new String(cmd.getAuth().getPassword()); + } + // get input stream from the given url + InputStream in = UriUtils.getInputStreamFromUrl(url, user, password); + URI uri; + URL urlObj; + try { + uri = new URI(url); + urlObj = new URL(url); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + url); + } catch (MalformedURLException e) { + throw new CloudRuntimeException("URL is incorrect: " + url); + } + + final String bucket = s3.getBucketName(); + // convention is no / in the end for install path based on S3Utils implementation. + String path = determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId(), cmd.getName()); + // template key is + // TEMPLATE_ROOT_DIR/account_id/template_id/template_name + String key = join(asList(path, urlObj.getFile()), S3Utils.SEPARATOR); + S3Utils.putObject(s3, in, bucket, key); + List s3Obj = S3Utils.getDirectory(s3, bucket, path); + if (s3Obj == null || s3Obj.size() == 0) { + return new Answer(cmd, false, "Failed to download to S3 bucket: " + bucket + " with key: " + key); + } else { + return new DownloadAnswer(null, 100, null, Status.DOWNLOADED, path, path, s3Obj.get(0).getSize(), s3Obj.get(0).getSize(), s3Obj.get(0) + .getETag()); + } + } + else if ( dstore instanceof NfsTO ){ + return new Answer(cmd, false, "Nfs needs to be pre-installed with system vm templates"); + } + else if ( dstore instanceof SwiftTO ){ + //TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand) here, but we need to handle + // source is url, most likely we need to modify our existing swiftUpload python script. + return new Answer(cmd, false, "Swift is not currently support DownloadCommand"); + } + else{ + return new Answer(cmd, false, "Unsupported image data store: " + dstore); + } + } +} diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java index 4d4d037de30..9b9f03d2d53 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/S3TemplateTest.java @@ -18,6 +18,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.LocalHostEndpoint; +import org.apache.cloudstack.storage.MockLocalNfsSecondaryStorageResource; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; @@ -108,8 +109,11 @@ public class S3TemplateTest extends CloudStackTestNGBase { image = templateDao.persist(image); templateId = image.getId(); - Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(new LocalHostEndpoint()); - Mockito.when(epSelector.select(Mockito.any(DataStore.class))).thenReturn(new LocalHostEndpoint()); + // inject mockito + LocalHostEndpoint ep = new LocalHostEndpoint(); + ep.setResource(new MockLocalNfsSecondaryStorageResource()); + Mockito.when(epSelector.select(Mockito.any(DataObject.class))).thenReturn(ep); + Mockito.when(epSelector.select(Mockito.any(DataStore.class))).thenReturn(ep); } @Test diff --git a/engine/storage/integration-test/test/resource/testng.xml b/engine/storage/integration-test/test/resource/testng.xml index 55f7edf1e0b..ad4829bc1b0 100644 --- a/engine/storage/integration-test/test/resource/testng.xml +++ b/engine/storage/integration-test/test/resource/testng.xml @@ -29,8 +29,16 @@ - - + + + + + + + + + + diff --git a/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java b/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java index ea93560934b..170772596a0 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java +++ b/engine/storage/src/org/apache/cloudstack/storage/LocalHostEndpoint.java @@ -92,5 +92,13 @@ public class LocalHostEndpoint implements EndPoint { executor.schedule(new CmdRunner2(cmd, listener.getCallback()), 10, TimeUnit.SECONDS); } } + public ServerResource getResource() { + return resource; + } + public void setResource(ServerResource resource) { + this.resource = resource; + } + + }