diff --git a/api/src/com/cloud/agent/api/storage/DownloadCommand.java b/api/src/com/cloud/agent/api/storage/DownloadCommand.java index 8293b445cd7..8abc29a8b6b 100644 --- a/api/src/com/cloud/agent/api/storage/DownloadCommand.java +++ b/api/src/com/cloud/agent/api/storage/DownloadCommand.java @@ -108,7 +108,6 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal private long id; private ResourceType resourceType = ResourceType.TEMPLATE; private DataStoreTO _store; - private Long resourceId; protected DownloadCommand() { } @@ -137,7 +136,6 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal this.setSecUrl(((NfsTO) store).getUrl()); } this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; - this.resourceId = template.getId(); } public DownloadCommand(DataStoreTO store, Volume volume, Long maxDownloadSizeInBytes, String checkSum, String url, ImageFormat format) { @@ -148,7 +146,6 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal this._store = store; this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; this.resourceType = ResourceType.VOLUME; - this.resourceId = volume.getId(); } public DownloadCommand(DataStoreTO store, String url, VirtualMachineTemplate template, String user, String passwd, Long maxDownloadSizeInBytes) { @@ -240,14 +237,6 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal } - public Long getResourceId() { - return resourceId; - } - - - public void setResourceId(Long resourceId) { - this.resourceId = resourceId; - } } diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index 6cb7b9838b5..efbfaf0db7c 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -377,9 +377,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private Answer execute(DownloadCommand cmd) { DataStoreTO dstore = cmd.getDataStore(); - if (dstore instanceof NfsTO) { + if (dstore instanceof NfsTO || dstore instanceof S3TO ) { return _dlMgr.handleDownloadCommand(this, cmd); - } else if (dstore instanceof S3TO) { + } + /* + else if (dstore instanceof S3TO) { // TODO: start download job to handle this // TODO: how to handle download progress for S3 S3TO s3 = (S3TO) cmd.getDataStore(); @@ -427,7 +429,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new DownloadAnswer(null, 100, null, Status.DOWNLOADED, path, path, s3Obj.get(0).getSize(), s3Obj.get(0).getSize(), s3Obj .get(0).getETag()); } - } else if (dstore instanceof SwiftTO) { + } */ + else if (dstore instanceof SwiftTO) { // TODO: need to move code from // execute(uploadTemplateToSwiftFromSecondaryStorageCommand) here, // but we need to handle diff --git a/core/src/com/cloud/storage/template/DownloadManager.java b/core/src/com/cloud/storage/template/DownloadManager.java index 82f4153f6e2..ec424aeb817 100644 --- a/core/src/com/cloud/storage/template/DownloadManager.java +++ b/core/src/com/cloud/storage/template/DownloadManager.java @@ -16,13 +16,13 @@ // under the License. package com.cloud.storage.template; -import java.util.List; import java.util.Map; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.DownloadCommand; import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.agent.api.storage.DownloadCommand.ResourceType; +import com.cloud.agent.api.to.S3TO; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.resource.SecondaryStorageResource; @@ -40,19 +40,21 @@ public interface DownloadManager extends Manager { * @param user username used for authentication to the server * @param password password used for authentication to the server * @param maxDownloadSizeInBytes (optional) max download size for the template, in bytes. - * @param resourceType signifying the type of resource like template, volume etc. + * @param resourceType signifying the type of resource like template, volume etc. * @return job-id that can be used to interrogate the status of the download. */ public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String userName, String passwd, long maxDownloadSizeInBytes, Proxy proxy, ResourceType resourceType); - - + + public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType); + + /** * Get the status of a download job * @param jobId job Id * @return status of the download job */ public TemplateDownloader.Status getDownloadStatus(String jobId); - + /** * Get the status of a download job * @param jobId job Id @@ -80,19 +82,19 @@ public interface DownloadManager extends Manager { * @return public String getDownloadLocalPath(String jobId); */ - + /** Handle download commands from the management server * @param cmd cmd from server * @return answer representing status of download. */ public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd); - + /** /** * @return list of template info for installed templates */ public Map gatherTemplateInfo(String templateDir); - + /** /** * @return list of volume info for installed volumes diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java index 7eb68418b42..c1391a5f7c1 100755 --- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java +++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java @@ -61,6 +61,7 @@ import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.agent.api.storage.DownloadCommand.ResourceType; import com.cloud.agent.api.storage.DownloadProgressCommand; import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType; +import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.S3TO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; @@ -274,15 +275,18 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager threadPool.execute(td); break; case DOWNLOAD_FINISHED: - td.setDownloadError("Download success, starting install "); - String result = postDownload(jobId); - if (result != null) { - s_logger.error("Failed post download script: " + result); - td.setStatus(Status.UNRECOVERABLE_ERROR); - td.setDownloadError("Failed post download script: " + result); - } else { - td.setStatus(Status.POST_DOWNLOAD_FINISHED); - td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); + if (!(td instanceof S3TemplateDownloader)) { + // we currently only create template.properties for NFS by running some post download script + td.setDownloadError("Download success, starting install "); + String result = postDownload(jobId); + if (result != null) { + s_logger.error("Failed post download script: " + result); + td.setStatus(Status.UNRECOVERABLE_ERROR); + td.setDownloadError("Failed post download script: " + result); + } else { + td.setStatus(Status.POST_DOWNLOAD_FINISHED); + td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date())); + } } dj.cleanup(); break; @@ -456,6 +460,36 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return Status.UNKNOWN; } + @Override + public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { + UUID uuid = UUID.randomUUID(); + String jobId = uuid.toString(); + + URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + url); + } + TemplateDownloader td; + if ((uri != null) && (uri.getScheme() != null)) { + if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) { + td = new S3TemplateDownloader(s3, url, installPathPrefix, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, + resourceType); + } else { + throw new CloudRuntimeException("Scheme is not supported " + url); + } + } else { + throw new CloudRuntimeException("Unable to download from URL: " + url); + } + DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType); + jobs.put(jobId, dj); + threadPool.execute(td); + + return jobId; + } + + @Override public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) { UUID uuid = UUID.randomUUID(); @@ -616,10 +650,26 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } String installPathPrefix = null; - if (ResourceType.TEMPLATE == resourceType){ - installPathPrefix = resource.getRootDir(cmd) + File.separator + _templateDir; - }else { - installPathPrefix = resource.getRootDir(cmd) + File.separator + _volumeDir; + DataStoreTO dstore = cmd.getDataStore(); + if (dstore instanceof S3TO) { + if (resourceType == ResourceType.TEMPLATE) { + // convention is no / in the end for install path based on + // S3Utils implementation. + // template key is + // TEMPLATE_ROOT_DIR/account_id/template_id/template_name, by + // adding template_name in the key, I can avoid generating a + // template.properties file + // for listTemplateCommand. + installPathPrefix = join(asList(_templateDir, cmd.getAccountId(), cmd.getId(), cmd.getName()), S3Utils.SEPARATOR); + } else { + installPathPrefix = join(asList(_volumeDir, cmd.getAccountId(), cmd.getId()), S3Utils.SEPARATOR); + } + } else { + if (ResourceType.TEMPLATE == resourceType) { + installPathPrefix = resource.getRootDir(cmd) + File.separator + _templateDir; + } else { + installPathPrefix = resource.getRootDir(cmd) + File.separator + _volumeDir; + } } String user = null; @@ -630,7 +680,13 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } //TO DO - Define Volume max size as well long maxDownloadSizeInBytes = (cmd.getMaxDownloadSizeInBytes() == null) ? TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES : (cmd.getMaxDownloadSizeInBytes()); - String jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + String jobId = null; + if (dstore instanceof S3TO){ + jobId = downloadS3Template((S3TO)dstore, cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + } + else{ + jobId = downloadPublicTemplate(cmd.getId(), cmd.getUrl(), cmd.getName(), cmd.getFormat(), cmd.isHvm(), cmd.getAccountId(), cmd.getDescription(), cmd.getChecksum(), installPathPrefix, user, password, maxDownloadSizeInBytes, cmd.getProxy(), resourceType); + } sleep(); if (jobId == null) { return new DownloadAnswer("Internal Error", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); diff --git a/core/src/com/cloud/storage/template/S3TemplateDownloader.java b/core/src/com/cloud/storage/template/S3TemplateDownloader.java new file mode 100644 index 00000000000..23f11ab2084 --- /dev/null +++ b/core/src/com/cloud/storage/template/S3TemplateDownloader.java @@ -0,0 +1,452 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.template; + + +import static com.cloud.utils.StringUtils.join; +import static java.util.Arrays.asList; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Date; + +import org.apache.commons.httpclient.ChunkedInputStream; +import org.apache.commons.httpclient.Credentials; +import org.apache.commons.httpclient.Header; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpException; +import org.apache.commons.httpclient.HttpMethod; +import org.apache.commons.httpclient.HttpMethodRetryHandler; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.NoHttpResponseException; +import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthScope; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; +import org.apache.log4j.Logger; + +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.ProgressEvent; +import com.amazonaws.services.s3.model.ProgressListener; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.StorageClass; +import com.cloud.agent.api.storage.DownloadCommand.Proxy; +import com.cloud.agent.api.storage.DownloadCommand.ResourceType; +import com.cloud.agent.api.to.S3TO; +import com.cloud.utils.Pair; +import com.cloud.utils.S3Utils; + +/** + * Download a template file using HTTP + * + */ +public class S3TemplateDownloader implements TemplateDownloader { + public static final Logger s_logger = Logger.getLogger(S3TemplateDownloader.class.getName()); + private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); + + private String downloadUrl; + private String installPath; + private String s3Key; + private String fileName; + public TemplateDownloader.Status status= TemplateDownloader.Status.NOT_STARTED; + public String errorString = " "; + private long remoteSize = 0; + public long downloadTime = 0; + public long totalBytes; + private final HttpClient client; + private GetMethod request; + private boolean resume = false; + private DownloadCompleteCallback completionCallback; + S3TO s3; + boolean inited = true; + + private long MAX_TEMPLATE_SIZE_IN_BYTES; + private ResourceType resourceType = ResourceType.TEMPLATE; + private final HttpMethodRetryHandler myretryhandler; + + + + public S3TemplateDownloader (S3TO storageLayer, String downloadUrl, String installPath, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { + this.s3 = storageLayer; + this.downloadUrl = downloadUrl; + this.installPath = installPath; + this.status = TemplateDownloader.Status.NOT_STARTED; + this.resourceType = resourceType; + this.MAX_TEMPLATE_SIZE_IN_BYTES = maxTemplateSizeInBytes; + + this.totalBytes = 0; + this.client = new HttpClient(s_httpClientManager); + + myretryhandler = new HttpMethodRetryHandler() { + @Override + public boolean retryMethod( + final HttpMethod method, + final IOException exception, + int executionCount) { + if (executionCount >= 2) { + // Do not retry if over max retry count + return false; + } + if (exception instanceof NoHttpResponseException) { + // Retry if the server dropped connection on us + return true; + } + if (!method.isRequestSent()) { + // Retry if the request has not been sent fully or + // if it's OK to retry methods that have been sent + return true; + } + // otherwise do not retry + return false; + } + }; + + try { + this.request = new GetMethod(downloadUrl); + this.request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); + this.completionCallback = callback; + + Pair hostAndPort = validateUrl(downloadUrl); + + if (proxy != null) { + client.getHostConfiguration().setProxy(proxy.getHost(), proxy.getPort()); + if (proxy.getUserName() != null) { + Credentials proxyCreds = new UsernamePasswordCredentials(proxy.getUserName(), proxy.getPassword()); + client.getState().setProxyCredentials(AuthScope.ANY, proxyCreds); + } + } + if ((user != null) && (password != null)) { + client.getParams().setAuthenticationPreemptive(true); + Credentials defaultcreds = new UsernamePasswordCredentials(user, password); + client.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); + s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); + } else { + s_logger.info("No credentials configured for host=" + hostAndPort.first() + ":" + hostAndPort.second()); + } + } catch (IllegalArgumentException iae) { + errorString = iae.getMessage(); + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + inited = false; + } catch (Exception ex){ + errorString = "Unable to start download -- check url? "; + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + s_logger.warn("Exception in constructor -- " + ex.toString()); + } catch (Throwable th) { + s_logger.warn("throwable caught ", th); + } + } + + + private Pair validateUrl(String url) throws IllegalArgumentException { + try { + URI uri = new URI(url); + if (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") ) { + throw new IllegalArgumentException("Unsupported scheme for url"); + } + int port = uri.getPort(); + if (!(port == 80 || port == 443 || port == -1)) { + throw new IllegalArgumentException("Only ports 80 and 443 are allowed"); + } + + if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) { + port = 443; + } else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) { + port = 80; + } + + URL urlObj = new URL(url); + this.fileName = urlObj.getFile(); + + String host = uri.getHost(); + try { + InetAddress hostAddr = InetAddress.getByName(host); + if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) { + throw new IllegalArgumentException("Illegal host specified in url"); + } + if (hostAddr instanceof Inet6Address) { + throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")"); + } + return new Pair(host, port); + } catch (UnknownHostException uhe) { + throw new IllegalArgumentException("Unable to resolve " + host); + } + + } catch (IllegalArgumentException iae) { + s_logger.warn("Failed uri validation check: " + iae.getMessage()); + throw iae; + } catch (URISyntaxException use) { + s_logger.warn("Failed uri syntax check: " + use.getMessage()); + throw new IllegalArgumentException(use.getMessage()); + } catch (MalformedURLException e) { + s_logger.warn("Failed url syntax check: " + e.getMessage()); + throw new IllegalArgumentException(e.getMessage()); + } + } + + @Override + public long download(boolean resume, DownloadCompleteCallback callback) { + switch (status) { + case ABORTED: + case UNRECOVERABLE_ERROR: + case DOWNLOAD_FINISHED: + return 0; + default: + + } + int bytes=0; + try { + // get the total size of file + Header contentLengthHeader = request.getResponseHeader("Content-Length"); + boolean chunked = false; + long remoteSize2 = 0; + if (contentLengthHeader == null) { + Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); + if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString=" Failed to receive length of download "; + return 0; //FIXME: what status do we put here? Do we retry? + } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())){ + chunked = true; + } + } else { + remoteSize2 = Long.parseLong(contentLengthHeader.getValue()); + } + + if (remoteSize == 0) { + remoteSize = remoteSize2; + } + + if (remoteSize > MAX_TEMPLATE_SIZE_IN_BYTES) { + s_logger.info("Remote size is too large: " + remoteSize + " , max=" + MAX_TEMPLATE_SIZE_IN_BYTES); + status = Status.UNRECOVERABLE_ERROR; + errorString = "Download file size is too large"; + return 0; + } + + if (remoteSize == 0) { + remoteSize = MAX_TEMPLATE_SIZE_IN_BYTES; + } + + InputStream in = !chunked?new BufferedInputStream(request.getResponseBodyAsStream()) + : new ChunkedInputStream(request.getResponseBodyAsStream()); + + s_logger.info("Starting download from " + getDownloadUrl() + " to s3 bucket " + s3.getBucketName() + " remoteSize=" + remoteSize + " , max size=" + MAX_TEMPLATE_SIZE_IN_BYTES); + + Date start = new Date(); + // compute s3 key + s3Key = join(asList(installPath, fileName), S3Utils.SEPARATOR); + + // download using S3 API + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(remoteSize); + PutObjectRequest putObjectRequest = new PutObjectRequest( + s3.getBucketName(), s3Key, in, metadata) + .withStorageClass(StorageClass.ReducedRedundancy); + // register progress listenser + putObjectRequest + .setProgressListener(new ProgressListener() { + @Override + public void progressChanged( + ProgressEvent progressEvent) { + s_logger.info(progressEvent.getBytesTransfered() + + " number of byte transferd " + + new Date()); + totalBytes += progressEvent.getBytesTransfered(); + if (progressEvent.getEventCode() == ProgressEvent.COMPLETED_EVENT_CODE) { + s_logger.info("download completed"); + status = TemplateDownloader.Status.DOWNLOAD_FINISHED; + } else { + status = TemplateDownloader.Status.IN_PROGRESS; + } + + } + + }); + S3Utils.putObject(s3, putObjectRequest); + Date finish = new Date(); + String downloaded = "(incomplete download)"; + if (totalBytes >= remoteSize) { + status = TemplateDownloader.Status.DOWNLOAD_FINISHED; + downloaded = "(download complete remote=" + remoteSize + "bytes)"; + } + errorString = "Downloaded " + totalBytes + " bytes " + downloaded; + downloadTime += finish.getTime() - start.getTime(); + return totalBytes; + }catch (HttpException hte) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + errorString = hte.getMessage(); + } catch (IOException ioe) { + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; //probably a file write error? + errorString = ioe.getMessage(); + } finally { + // close input stream + request.releaseConnection(); + if (callback != null) { + callback.downloadComplete(status); + } + } + return 0; + } + + public String getDownloadUrl() { + return downloadUrl; + } + + + @Override + public TemplateDownloader.Status getStatus() { + return status; + } + + + @Override + public long getDownloadTime() { + return downloadTime; + } + + + @Override + public long getDownloadedBytes() { + return totalBytes; + } + + @Override + @SuppressWarnings("fallthrough") + public boolean stopDownload() { + switch (getStatus()) { + case IN_PROGRESS: + if (request != null) { + request.abort(); + } + status = TemplateDownloader.Status.ABORTED; + return true; + case UNKNOWN: + case NOT_STARTED: + case RECOVERABLE_ERROR: + case UNRECOVERABLE_ERROR: + case ABORTED: + status = TemplateDownloader.Status.ABORTED; + case DOWNLOAD_FINISHED: + try { + S3Utils.deleteObject(s3, s3.getBucketName(), s3Key); + } catch (Exception ex) { + // ignore delete exception if it is not there + } + return true; + + default: + return true; + } + } + + @Override + public int getDownloadPercent() { + if (remoteSize == 0) { + return 0; + } + + return (int)(100.0*totalBytes/remoteSize); + } + + @Override + public void run() { + try { + download(resume, completionCallback); + } catch (Throwable t) { + s_logger.warn("Caught exception during download "+ t.getMessage(), t); + errorString = "Failed to install: " + t.getMessage(); + status = TemplateDownloader.Status.UNRECOVERABLE_ERROR; + } + + } + + @Override + public void setStatus(TemplateDownloader.Status status) { + this.status = status; + } + + + + public boolean isResume() { + return resume; + } + + @Override + public String getDownloadError() { + return errorString; + } + + @Override + public String getDownloadLocalPath() { + return this.s3Key; + } + + @Override + public void setResume(boolean resume) { + this.resume = resume; + } + + + @Override + public long getMaxTemplateSizeInBytes() { + return this.MAX_TEMPLATE_SIZE_IN_BYTES; + } + + public static void main(String[] args) { + String url ="http://dev.mysql.com/get/Downloads/MySQL-5.0/mysql-noinstall-5.0.77-win32.zip/from/http://mirror.services.wisc.edu/mysql/"; + try { + URI uri = new java.net.URI(url); + } catch (URISyntaxException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + TemplateDownloader td = new S3TemplateDownloader(null, url,"/tmp/mysql", null, TemplateDownloader.DEFAULT_MAX_TEMPLATE_SIZE_IN_BYTES, null, null, null, null); + long bytes = td.download(true, null); + if (bytes > 0) { + System.out.println("Downloaded (" + bytes + " bytes)" + " in " + td.getDownloadTime()/1000 + " secs"); + } else { + System.out.println("Failed download"); + } + + } + + @Override + public void setDownloadError(String error) { + errorString = error; + } + + + + @Override + public boolean isInited() { + return inited; + } + + + public ResourceType getResourceType() { + return resourceType; + } + +} diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java index a0eb7d334ea..8199b084e72 100644 --- a/utils/src/com/cloud/utils/S3Utils.java +++ b/utils/src/com/cloud/utils/S3Utils.java @@ -54,6 +54,7 @@ import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.Bucket; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.cloud.utils.exception.CloudRuntimeException; @@ -156,6 +157,17 @@ public final class S3Utils { } + public static void putObject(final ClientOptions clientOptions, + final PutObjectRequest req) { + + assert clientOptions != null; + assert req != null; + + acquireClient(clientOptions).putObject(req); + + } + + // Note that whenever S3Object is returned, client code needs to close the internal stream to avoid resource leak. public static S3Object getObject(final ClientOptions clientOptions, final String bucketName, final String key) {