mirror of https://github.com/apache/cloudstack.git
Add S3 code to handle DownloadCommand for registerTemplate.
This commit is contained in:
Min Chen
parent
2dd8e2cb97
commit
593337565e
|
|
@ -20,6 +20,7 @@ import java.net.URI;
|
|||
|
||||
import org.apache.cloudstack.api.InternalIdentity;
|
||||
|
||||
import com.cloud.agent.api.to.DataStoreTO;
|
||||
import com.cloud.storage.Storage.ImageFormat;
|
||||
import com.cloud.storage.Volume;
|
||||
import com.cloud.template.VirtualMachineTemplate;
|
||||
|
|
@ -105,6 +106,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
|
|||
private Long maxDownloadSizeInBytes = null;
|
||||
private long id;
|
||||
private ResourceType resourceType = ResourceType.TEMPLATE;
|
||||
private DataStoreTO _store;
|
||||
private Long resourceId;
|
||||
|
||||
protected DownloadCommand() {
|
||||
}
|
||||
|
|
@ -122,14 +125,16 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
|
|||
this.resourceType = that.resourceType;
|
||||
}
|
||||
|
||||
public DownloadCommand(String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes) {
|
||||
public DownloadCommand(DataStoreTO store, String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes) {
|
||||
super(template.getUniqueName(), template.getUrl(), template.getFormat(), template.getAccountId());
|
||||
this._store = store;
|
||||
this.hvm = template.isRequiresHvm();
|
||||
this.checksum = template.getChecksum();
|
||||
this.id = template.getId();
|
||||
this.description = template.getDisplayText();
|
||||
this.setSecUrl(secUrl);
|
||||
this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
|
||||
this.resourceId = template.getId();
|
||||
}
|
||||
|
||||
public DownloadCommand(String secUrl, Volume volume, Long maxDownloadSizeInBytes, String checkSum, String url, ImageFormat format) {
|
||||
|
|
@ -216,4 +221,26 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
|
|||
public void setResourceType(ResourceType resourceType) {
|
||||
this.resourceType = resourceType;
|
||||
}
|
||||
|
||||
|
||||
public DataStoreTO getDataStore() {
|
||||
return _store;
|
||||
}
|
||||
|
||||
|
||||
public void setDataStore(DataStoreTO _store) {
|
||||
this._store = _store;
|
||||
}
|
||||
|
||||
|
||||
public Long getResourceId() {
|
||||
return resourceId;
|
||||
}
|
||||
|
||||
|
||||
public void setResourceId(Long resourceId) {
|
||||
this.resourceId = resourceId;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
package com.cloud.agent.api.to;
|
||||
|
||||
import com.cloud.storage.DataStoreRole;
|
||||
|
||||
public final class NfsTO implements DataStoreTO {
|
||||
|
||||
private String _url;
|
||||
private DataStoreRole _role;
|
||||
|
||||
public NfsTO() {
|
||||
|
||||
super();
|
||||
|
||||
}
|
||||
|
||||
public NfsTO(String url, DataStoreRole role) {
|
||||
|
||||
super();
|
||||
|
||||
this._url = url;
|
||||
this._role = role;
|
||||
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return _url;
|
||||
}
|
||||
|
||||
public void setUrl(String _url) {
|
||||
this._url = _url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DataStoreRole getRole() {
|
||||
return _role;
|
||||
}
|
||||
|
||||
public void setRole(DataStoreRole _role) {
|
||||
this._role = _role;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
@ -34,7 +34,10 @@ import java.io.IOException;
|
|||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
import java.net.InetAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -90,6 +93,8 @@ import com.cloud.agent.api.storage.ListVolumeAnswer;
|
|||
import com.cloud.agent.api.storage.ListVolumeCommand;
|
||||
import com.cloud.agent.api.storage.UploadCommand;
|
||||
import com.cloud.agent.api.storage.ssCommand;
|
||||
import com.cloud.agent.api.to.DataStoreTO;
|
||||
import com.cloud.agent.api.to.NfsTO;
|
||||
import com.cloud.agent.api.to.S3TO;
|
||||
import com.cloud.agent.api.to.SwiftTO;
|
||||
import com.cloud.exception.InternalErrorException;
|
||||
|
|
@ -106,6 +111,7 @@ import com.cloud.storage.template.UploadManager;
|
|||
import com.cloud.storage.template.UploadManagerImpl;
|
||||
import com.cloud.utils.NumbersUtil;
|
||||
import com.cloud.utils.S3Utils;
|
||||
import com.cloud.utils.UriUtils;
|
||||
import com.cloud.utils.S3Utils.FileNamingStrategy;
|
||||
import com.cloud.utils.S3Utils.ObjectNamingStrategy;
|
||||
import com.cloud.utils.component.ComponentContext;
|
||||
|
|
@ -126,7 +132,7 @@ SecondaryStorageResource {
|
|||
|
||||
int _timeout;
|
||||
|
||||
String _instance;
|
||||
String _instance;
|
||||
String _dc;
|
||||
String _pod;
|
||||
String _guid;
|
||||
|
|
@ -162,8 +168,8 @@ SecondaryStorageResource {
|
|||
if (cmd instanceof DownloadProgressCommand) {
|
||||
return _dlMgr.handleDownloadCommand(this, (DownloadProgressCommand)cmd);
|
||||
} else if (cmd instanceof DownloadCommand) {
|
||||
return _dlMgr.handleDownloadCommand(this, (DownloadCommand)cmd);
|
||||
} else if (cmd instanceof UploadCommand) {
|
||||
return execute((DownloadCommand)cmd);
|
||||
} else if (cmd instanceof UploadCommand) {
|
||||
return _upldMgr.handleUploadCommand(this, (UploadCommand)cmd);
|
||||
} else if (cmd instanceof CreateEntityDownloadURLCommand){
|
||||
return _upldMgr.handleCreateEntityURLCommand((CreateEntityDownloadURLCommand)cmd);
|
||||
|
|
@ -325,6 +331,51 @@ SecondaryStorageResource {
|
|||
}
|
||||
}
|
||||
|
||||
private Answer execute(DownloadCommand cmd){
|
||||
DataStoreTO dstore = cmd.getDataStore();
|
||||
if ( dstore instanceof NfsTO ){
|
||||
return _dlMgr.handleDownloadCommand(this, cmd);
|
||||
}
|
||||
else if ( dstore instanceof S3TO ){
|
||||
//TODO: how to handle download progress for S3
|
||||
S3TO s3 = (S3TO)cmd.getDataStore();
|
||||
String url = cmd.getUrl();
|
||||
String user = null;
|
||||
String password = null;
|
||||
if (cmd.getAuth() != null) {
|
||||
user = cmd.getAuth().getUserName();
|
||||
password = new String(cmd.getAuth().getPassword());
|
||||
}
|
||||
// get input stream from the given url
|
||||
InputStream in = UriUtils.getInputStreamFromUrl(url, user, password);
|
||||
URI uri;
|
||||
URL urlObj;
|
||||
try {
|
||||
uri = new URI(url);
|
||||
urlObj = new URL(url);
|
||||
} catch (URISyntaxException e) {
|
||||
throw new CloudRuntimeException("URI is incorrect: " + url);
|
||||
} catch (MalformedURLException e) {
|
||||
throw new CloudRuntimeException("URL is incorrect: " + url);
|
||||
}
|
||||
|
||||
final String bucket = s3.getBucketName();
|
||||
String key = join(asList(determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId()), urlObj.getFile()), S3Utils.SEPARATOR);
|
||||
S3Utils.putObject(s3, in, bucket, key);
|
||||
return new Answer(cmd, true, format("Uploaded the contents of input stream from %1$s for template id %2$s to S3 bucket %3$s", url,
|
||||
cmd.getResourceId(), bucket));
|
||||
}
|
||||
else if ( dstore instanceof SwiftTO ){
|
||||
//TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand) here, but we need to handle
|
||||
// source is url, most likely we need to modify our existing swiftUpload python script.
|
||||
return new Answer(cmd, false, "Swift is not currently support DownloadCommand");
|
||||
}
|
||||
else{
|
||||
return new Answer(cmd, false, "Unsupport image data store: " + dstore);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
private Answer execute(uploadTemplateToSwiftFromSecondaryStorageCommand cmd) {
|
||||
SwiftTO swift = cmd.getSwift();
|
||||
String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
|
||||
|
|
@ -834,33 +885,33 @@ SecondaryStorageResource {
|
|||
String absoluteTemplatePath = parent + relativeTemplatePath;
|
||||
MessageDigest digest;
|
||||
String checksum = null;
|
||||
File f = new File(absoluteTemplatePath);
|
||||
File f = new File(absoluteTemplatePath);
|
||||
InputStream is = null;
|
||||
byte[] buffer = new byte[8192];
|
||||
int read = 0;
|
||||
if(s_logger.isDebugEnabled()){
|
||||
s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath );
|
||||
s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath );
|
||||
}
|
||||
|
||||
|
||||
try {
|
||||
digest = MessageDigest.getInstance("MD5");
|
||||
is = new FileInputStream(f);
|
||||
digest = MessageDigest.getInstance("MD5");
|
||||
is = new FileInputStream(f);
|
||||
while( (read = is.read(buffer)) > 0) {
|
||||
digest.update(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
byte[] md5sum = digest.digest();
|
||||
BigInteger bigInt = new BigInteger(1, md5sum);
|
||||
checksum = bigInt.toString(16);
|
||||
if(s_logger.isDebugEnabled()){
|
||||
s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+ " - " +checksum );
|
||||
s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+ " - " +checksum );
|
||||
}
|
||||
|
||||
}catch(IOException e) {
|
||||
String logMsg = "Unable to process file for MD5 - " + absoluteTemplatePath;
|
||||
s_logger.error(logMsg);
|
||||
return new Answer(cmd, false, checksum);
|
||||
}catch (NoSuchAlgorithmException e) {
|
||||
return new Answer(cmd, false, checksum);
|
||||
}catch (NoSuchAlgorithmException e) {
|
||||
return new Answer(cmd, false, checksum);
|
||||
}
|
||||
finally {
|
||||
|
|
@ -869,10 +920,10 @@ SecondaryStorageResource {
|
|||
is.close();
|
||||
} catch (IOException e) {
|
||||
if(s_logger.isDebugEnabled()){
|
||||
s_logger.debug("Could not close the file " +absoluteTemplatePath);
|
||||
s_logger.debug("Could not close the file " +absoluteTemplatePath);
|
||||
}
|
||||
return new Answer(cmd, false, checksum);
|
||||
}
|
||||
return new Answer(cmd, false, checksum);
|
||||
}
|
||||
}
|
||||
|
||||
return new Answer(cmd, true, checksum);
|
||||
|
|
@ -1141,7 +1192,7 @@ SecondaryStorageResource {
|
|||
if (nfsIps.contains(cidr)) {
|
||||
/*
|
||||
* if the internal download ip is the same with secondary storage ip, adding internal sites will flush
|
||||
* ip route to nfs through storage ip.
|
||||
* ip route to nfs through storage ip.
|
||||
*/
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1243,7 +1294,7 @@ SecondaryStorageResource {
|
|||
|
||||
for (PortConfig pCfg:cmd.getPortConfigs()){
|
||||
if (pCfg.isAdd()) {
|
||||
ipList.add(pCfg.getSourceIp());
|
||||
ipList.add(pCfg.getSourceIp());
|
||||
}
|
||||
}
|
||||
boolean success = true;
|
||||
|
|
@ -1401,7 +1452,7 @@ SecondaryStorageResource {
|
|||
String nfsPath = nfsHostIp + ":" + uri.getPath();
|
||||
String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString();
|
||||
String root = _parent + "/" + dir;
|
||||
mount(root, nfsPath);
|
||||
mount(root, nfsPath);
|
||||
return root;
|
||||
} catch (Exception e) {
|
||||
String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString();
|
||||
|
|
@ -1470,7 +1521,7 @@ SecondaryStorageResource {
|
|||
String eth2ip = (String) params.get("eth2ip");
|
||||
if (eth2ip != null) {
|
||||
params.put("public.network.device", "eth2");
|
||||
}
|
||||
}
|
||||
_publicIp = (String) params.get("eth2ip");
|
||||
_hostname = (String) params.get("name");
|
||||
|
||||
|
|
@ -1677,7 +1728,7 @@ SecondaryStorageResource {
|
|||
command.add(String.valueOf(isAppend));
|
||||
for (String ip : ipList){
|
||||
command.add(ip);
|
||||
}
|
||||
}
|
||||
|
||||
String result = command.execute();
|
||||
if (result != null) {
|
||||
|
|
@ -1693,7 +1744,7 @@ SecondaryStorageResource {
|
|||
s_logger.debug("create mount point: " + root);
|
||||
} else {
|
||||
s_logger.debug("Unable to create mount point: " + root);
|
||||
return null;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1813,13 +1864,13 @@ SecondaryStorageResource {
|
|||
@Override
|
||||
public void setName(String name) {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setConfigParams(Map<String, Object> params) {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -1837,6 +1888,6 @@ SecondaryStorageResource {
|
|||
@Override
|
||||
public void setRunLevel(int level) {
|
||||
// TODO Auto-generated method stub
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,9 +16,15 @@
|
|||
// under the License.
|
||||
package com.cloud.storage.template;
|
||||
|
||||
import static com.cloud.utils.S3Utils.putDirectory;
|
||||
import static com.cloud.utils.StringUtils.join;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Arrays.asList;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FilenameFilter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
|
|
@ -41,14 +47,21 @@ import java.util.concurrent.Executors;
|
|||
import javax.ejb.Local;
|
||||
import javax.naming.ConfigurationException;
|
||||
|
||||
import org.apache.commons.httpclient.Credentials;
|
||||
import org.apache.commons.httpclient.HttpClient;
|
||||
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
|
||||
import org.apache.commons.httpclient.UsernamePasswordCredentials;
|
||||
import org.apache.commons.httpclient.auth.AuthScope;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.cloud.agent.api.Answer;
|
||||
import com.cloud.agent.api.storage.DownloadAnswer;
|
||||
import com.cloud.agent.api.storage.DownloadCommand;
|
||||
import com.cloud.agent.api.storage.DownloadCommand.Proxy;
|
||||
import com.cloud.agent.api.storage.DownloadCommand.ResourceType;
|
||||
import com.cloud.agent.api.storage.DownloadProgressCommand;
|
||||
import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType;
|
||||
import com.cloud.agent.api.to.S3TO;
|
||||
import com.cloud.exception.InternalErrorException;
|
||||
import com.cloud.storage.Storage.ImageFormat;
|
||||
import com.cloud.storage.StorageLayer;
|
||||
|
|
@ -59,6 +72,9 @@ import com.cloud.storage.template.Processor.FormatInfo;
|
|||
import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback;
|
||||
import com.cloud.storage.template.TemplateDownloader.Status;
|
||||
import com.cloud.utils.NumbersUtil;
|
||||
import com.cloud.utils.S3Utils;
|
||||
import com.cloud.utils.UriUtils;
|
||||
import com.cloud.utils.S3Utils.ObjectNamingStrategy;
|
||||
import com.cloud.utils.component.ManagerBase;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
import com.cloud.utils.script.OutputInterpreter;
|
||||
|
|
@ -224,7 +240,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
|
|||
|
||||
/**
|
||||
* Get notified of change of job status. Executed in context of downloader thread
|
||||
*
|
||||
*
|
||||
* @param jobId
|
||||
* the id of the job
|
||||
* @param status
|
||||
|
|
@ -279,21 +295,21 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
|
|||
byte[] buffer = new byte[8192];
|
||||
int read = 0;
|
||||
MessageDigest digest;
|
||||
String checksum = null;
|
||||
String checksum = null;
|
||||
InputStream is = null;
|
||||
try {
|
||||
digest = MessageDigest.getInstance("MD5");
|
||||
is = new FileInputStream(f);
|
||||
digest = MessageDigest.getInstance("MD5");
|
||||
is = new FileInputStream(f);
|
||||
while( (read = is.read(buffer)) > 0) {
|
||||
digest.update(buffer, 0, read);
|
||||
}
|
||||
}
|
||||
byte[] md5sum = digest.digest();
|
||||
BigInteger bigInt = new BigInteger(1, md5sum);
|
||||
checksum = String.format("%032x",bigInt);
|
||||
return checksum;
|
||||
}catch(IOException e) {
|
||||
return null;
|
||||
}catch (NoSuchAlgorithmException e) {
|
||||
}catch (NoSuchAlgorithmException e) {
|
||||
return null;
|
||||
}
|
||||
finally {
|
||||
|
|
@ -302,19 +318,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
|
|||
is.close();
|
||||
} catch (IOException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Post download activity (install and cleanup). Executed in context of downloader thread
|
||||
*
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
private String postDownload(String jobId) {
|
||||
DownloadJob dnld = jobs.get(jobId);
|
||||
TemplateDownloader td = dnld.getTemplateDownloader();
|
||||
String resourcePath = null;
|
||||
String resourcePath = null;
|
||||
ResourceType resourceType = dnld.getResourceType();
|
||||
|
||||
// once template path is set, remove the parent dir so that the template is installed with a relative path
|
||||
|
|
@ -458,7 +474,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
|
|||
return "Unable to create " + tmpDir;
|
||||
}
|
||||
// TO DO - define constant for volume properties.
|
||||
File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) :
|
||||
File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) :
|
||||
_storage.getFile(tmpDir + File.separator + "volume.properties");
|
||||
if ( file.exists() ) {
|
||||
file.delete();
|
||||
|
|
@ -583,6 +599,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
|
|||
return convertStatus(getDownloadStatus(jobId));
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) {
|
||||
ResourceType resourceType = cmd.getResourceType();
|
||||
|
|
@ -774,7 +791,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
|
|||
}
|
||||
|
||||
@Override
|
||||
public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {
|
||||
public Map<Long, TemplateProp> gatherVolumeInfo(String rootDir) {
|
||||
Map<Long, TemplateProp> result = new HashMap<Long, TemplateProp>();
|
||||
String volumeDir = rootDir + File.separator + _volumeDir;
|
||||
|
||||
|
|
@ -1046,4 +1063,5 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,19 +31,21 @@ import com.cloud.agent.api.SecStorageFirewallCfgCommand;
|
|||
import com.cloud.agent.api.UpdateHostPasswordCommand;
|
||||
import com.cloud.agent.api.storage.DownloadAnswer;
|
||||
import com.cloud.agent.api.storage.DownloadCommand;
|
||||
import com.cloud.agent.api.to.NfsTO;
|
||||
import com.cloud.exception.UnsupportedVersionException;
|
||||
import com.cloud.hypervisor.Hypervisor.HypervisorType;
|
||||
import com.cloud.serializer.GsonHelper;
|
||||
import com.cloud.storage.Storage.ImageFormat;
|
||||
import com.cloud.storage.Storage.TemplateType;
|
||||
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
|
||||
import com.cloud.storage.DataStoreRole;
|
||||
import com.cloud.storage.VMTemplateVO;
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
|
||||
public class RequestTest extends TestCase {
|
||||
|
|
@ -51,7 +53,7 @@ public class RequestTest extends TestCase {
|
|||
|
||||
public void testSerDeser() {
|
||||
s_logger.info("Testing serializing and deserializing works as expected");
|
||||
|
||||
|
||||
s_logger.info("UpdateHostPasswordCommand should have two parameters that doesn't show in logging");
|
||||
UpdateHostPasswordCommand cmd1 = new UpdateHostPasswordCommand("abc", "def");
|
||||
s_logger.info("SecStorageFirewallCfgCommand has a context map that shouldn't show up in debug level");
|
||||
|
|
@ -89,7 +91,7 @@ public class RequestTest extends TestCase {
|
|||
logger.setLevel(level);
|
||||
|
||||
byte[] bytes = sreq.getBytes();
|
||||
|
||||
|
||||
assert Request.getSequence(bytes) == 892403717;
|
||||
assert Request.getManagementServerId(bytes) == 3;
|
||||
assert Request.getAgentId(bytes) == 2;
|
||||
|
|
@ -130,7 +132,7 @@ public class RequestTest extends TestCase {
|
|||
s_logger.info("Testing Download answer");
|
||||
VMTemplateVO template = new VMTemplateVO(1, "templatename", ImageFormat.QCOW2, true, true, true, TemplateType.USER, "url", true, 32, 1, "chksum", "displayText", true, 30, true,
|
||||
HypervisorType.KVM, null);
|
||||
DownloadCommand cmd = new DownloadCommand("secUrl", template, 30000000l);
|
||||
DownloadCommand cmd = new DownloadCommand(new NfsTO("secUrl", DataStoreRole.Image), "secUrl", template, 30000000l);
|
||||
Request req = new Request(1, 1, cmd, true);
|
||||
|
||||
req.logD("Debug for Download");
|
||||
|
|
@ -161,7 +163,7 @@ public class RequestTest extends TestCase {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void testLogging() {
|
||||
s_logger.info("Testing Logging");
|
||||
GetHostStatsCommand cmd3 = new GetHostStatsCommand("hostguid", "hostname", 101);
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy;
|
|||
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole;
|
||||
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
|
||||
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
|
||||
|
|
@ -65,6 +65,7 @@ import com.cloud.configuration.dao.ConfigurationDao;
|
|||
import com.cloud.exception.StorageUnavailableException;
|
||||
import com.cloud.host.HostVO;
|
||||
import com.cloud.host.dao.HostDao;
|
||||
import com.cloud.storage.DataStoreRole;
|
||||
import com.cloud.storage.DiskOfferingVO;
|
||||
import com.cloud.storage.SnapshotVO;
|
||||
import com.cloud.storage.Storage.ImageFormat;
|
||||
|
|
@ -130,7 +131,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
VolumeManager volumeMgr;
|
||||
@Inject
|
||||
private SwiftManager _swiftMgr;
|
||||
@Inject
|
||||
@Inject
|
||||
private S3Manager _s3Mgr;
|
||||
@Inject
|
||||
StorageCacheManager cacheMgr;
|
||||
|
|
@ -314,12 +315,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected Answer cloneVolume(DataObject template, DataObject volume) {
|
||||
VolumeInfo volInfo = (VolumeInfo)volume;
|
||||
DiskOfferingVO offering = diskOfferingDao.findById(volInfo.getDiskOfferingId());
|
||||
VMTemplateStoragePoolVO tmpltStoredOn = templatePoolDao.findByPoolTemplate(template.getDataStore().getId(), template.getId());
|
||||
|
||||
|
||||
DiskProfile diskProfile = new DiskProfile(volInfo, offering,
|
||||
null);
|
||||
CreateCommand cmd = new CreateCommand(diskProfile,
|
||||
|
|
@ -334,7 +335,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
s_logger.debug("Failed to send to storage pool", e);
|
||||
throw new CloudRuntimeException("Failed to send to storage pool", e);
|
||||
}
|
||||
|
||||
|
||||
if (answer.getResult()) {
|
||||
VolumeVO vol = this.volDao.findById(volume.getId());
|
||||
CreateAnswer createAnswer = (CreateAnswer) answer;
|
||||
|
|
@ -345,7 +346,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
vol.setPoolId(pool.getId());
|
||||
vol.setPodId(pool.getPodId());
|
||||
this.volDao.update(vol.getId(), vol);
|
||||
|
||||
|
||||
} else {
|
||||
if (tmpltStoredOn != null
|
||||
&& (answer instanceof CreateAnswer)
|
||||
|
|
@ -354,15 +355,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
if (!templateMgr
|
||||
.resetTemplateDownloadStateOnPool(tmpltStoredOn
|
||||
.getId())) {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
errMsg = answer.getDetails();
|
||||
}
|
||||
|
||||
|
||||
return answer;
|
||||
}
|
||||
|
||||
|
||||
protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData) {
|
||||
VolumeInfo volume = (VolumeInfo)srcData;
|
||||
VolumeInfo destVolume = (VolumeInfo)destData;
|
||||
|
|
@ -370,9 +371,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
.getDataCenterId());
|
||||
StoragePool srcPool = (StoragePool)this.dataStoreMgr.getDataStore(volume
|
||||
.getPoolId(), DataStoreRole.Primary);
|
||||
|
||||
|
||||
StoragePool destPool = (StoragePool)this.dataStoreMgr.getDataStore(destVolume.getPoolId(), DataStoreRole.Primary);
|
||||
|
||||
|
||||
String value = this.configDao.getValue(Config.CopyVolumeWait.toString());
|
||||
int _copyvolumewait = NumbersUtil.parseInt(value,
|
||||
Integer.parseInt(Config.CopyVolumeWait.getDefaultValue()));
|
||||
|
|
@ -394,7 +395,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
}
|
||||
|
||||
String secondaryStorageVolumePath = cvAnswer.getVolumePath();
|
||||
|
||||
|
||||
cvCmd = new CopyVolumeCommand(volume.getId(),
|
||||
secondaryStorageVolumePath, destPool,
|
||||
secondaryStorageURL, false, _copyvolumewait);
|
||||
|
|
@ -409,7 +410,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
throw new CloudRuntimeException(
|
||||
"Failed to copy the volume from secondary storage to the destination primary storage pool.");
|
||||
}
|
||||
|
||||
|
||||
VolumeVO destVol = this.volDao.findById(destVolume.getId());
|
||||
destVol.setPath(cvAnswer.getVolumePath());
|
||||
this.volDao.update(destVol.getId(), destVol);
|
||||
|
|
@ -437,7 +438,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
} else if (srcData.getType() == DataObjectType.VOLUME
|
||||
&& destData.getType() == DataObjectType.TEMPLATE) {
|
||||
answer = createTemplateFromVolume(srcData, destData);
|
||||
} else if (srcData.getType() == DataObjectType.TEMPLATE
|
||||
} else if (srcData.getType() == DataObjectType.TEMPLATE
|
||||
&& destData.getType() == DataObjectType.VOLUME) {
|
||||
answer = cloneVolume(srcData, destData);
|
||||
} else if (destData.getType() == DataObjectType.VOLUME
|
||||
|
|
@ -667,21 +668,21 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
return sendCommand(cmd, pool, template.getId(), zoneId,
|
||||
secondaryStorageHost.getId());
|
||||
}
|
||||
|
||||
|
||||
private HostVO getSecHost(long volumeId, long dcId) {
|
||||
Long id = snapshotDao.getSecHostId(volumeId);
|
||||
if ( id != null) {
|
||||
if ( id != null) {
|
||||
return hostDao.findById(id);
|
||||
}
|
||||
return this.templateMgr.getSecondaryStorageHost(dcId);
|
||||
}
|
||||
|
||||
|
||||
protected Answer copySnapshot(DataObject srcObject, DataObject destObject) {
|
||||
SnapshotInfo srcSnapshot = (SnapshotInfo)srcObject;
|
||||
VolumeInfo baseVolume = srcSnapshot.getBaseVolume();
|
||||
Long dcId = baseVolume.getDataCenterId();
|
||||
Long accountId = baseVolume.getAccountId();
|
||||
|
||||
|
||||
HostVO secHost = getSecHost(baseVolume.getId(), baseVolume.getDataCenterId());
|
||||
Long secHostId = secHost.getId();
|
||||
String secondaryStoragePoolUrl = secHost.getStorageUrl();
|
||||
|
|
@ -696,12 +697,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
|
||||
SwiftTO swift = _swiftMgr.getSwiftTO();
|
||||
S3TO s3 = _s3Mgr.getS3TO();
|
||||
|
||||
|
||||
long prevSnapshotId = srcSnapshot.getPrevSnapshotId();
|
||||
if (prevSnapshotId > 0) {
|
||||
prevSnapshot = snapshotDao.findByIdIncludingRemoved(prevSnapshotId);
|
||||
if ( prevSnapshot.getBackupSnapshotId() != null && swift == null) {
|
||||
if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2")) {
|
||||
if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2")) {
|
||||
prevBackupUuid = prevSnapshot.getBackupSnapshotId();
|
||||
prevSnapshotUuid = prevSnapshot.getPath();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ public class ImageStoreImpl implements ImageStoreEntity {
|
|||
|
||||
@Override
|
||||
public DataStoreTO getTO() {
|
||||
return null;
|
||||
return getDriver().getStoreTO(this);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -83,10 +83,10 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
|
|||
|
||||
private VolumeDao volumeDao;
|
||||
|
||||
public PrimaryDataStoreImpl() {
|
||||
|
||||
public PrimaryDataStoreImpl() {
|
||||
|
||||
}
|
||||
|
||||
|
||||
public void configure(StoragePoolVO pdsv,
|
||||
PrimaryDataStoreDriver driver, DataStoreProvider provider) {
|
||||
this.pdsv = pdsv;
|
||||
|
|
@ -245,9 +245,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
return objectInStoreMgr.get(obj, this);
|
||||
}
|
||||
|
||||
|
|
@ -341,9 +341,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore {
|
|||
public String getStorageProviderName() {
|
||||
return this.pdsv.getStorageProviderName();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public DataStoreTO getTO() {
|
||||
return null;
|
||||
return getDriver().getStoreTO(this);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import com.cloud.agent.api.Answer;
|
|||
import com.cloud.agent.api.DeleteSnapshotBackupCommand;
|
||||
import com.cloud.agent.api.storage.DeleteVolumeCommand;
|
||||
import com.cloud.agent.api.to.DataStoreTO;
|
||||
import com.cloud.agent.api.to.NfsTO;
|
||||
import com.cloud.agent.api.to.S3TO;
|
||||
import com.cloud.agent.api.to.SwiftTO;
|
||||
import com.cloud.host.HostVO;
|
||||
|
|
@ -106,10 +107,9 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver {
|
|||
@Override
|
||||
public DataStoreTO getStoreTO(DataStore store) {
|
||||
ImageStoreImpl nfsStore = (ImageStoreImpl)store;
|
||||
ImageStoreTO nfsTO = new ImageStoreTO();
|
||||
nfsTO.setProviderName("CloudStack");
|
||||
NfsTO nfsTO = new NfsTO();
|
||||
nfsTO.setRole(DataStoreRole.Image);
|
||||
nfsTO.setUri(nfsStore.getUri());
|
||||
nfsTO.setUrl(nfsStore.getUri());
|
||||
return nfsTO;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -366,7 +366,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor
|
|||
if(vmTemplateStore != null) {
|
||||
start();
|
||||
DownloadCommand dcmd =
|
||||
new DownloadCommand(secUrl, template, maxTemplateSizeInBytes);
|
||||
new DownloadCommand(store.getTO(), secUrl, template, maxTemplateSizeInBytes);
|
||||
dcmd.setProxy(getHttpProxy());
|
||||
if (downloadJobExists) {
|
||||
dcmd = new DownloadProgressCommand(dcmd, vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART);
|
||||
|
|
|
|||
|
|
@ -138,6 +138,24 @@ public final class S3Utils {
|
|||
|
||||
}
|
||||
|
||||
public static void putObject(final ClientOptions clientOptions,
|
||||
final InputStream sourceStream, final String bucketName, final String key) {
|
||||
|
||||
assert clientOptions != null;
|
||||
assert sourceStream != null;
|
||||
assert !isBlank(bucketName);
|
||||
assert !isBlank(key);
|
||||
|
||||
if (LOGGER.isDebugEnabled()) {
|
||||
LOGGER.debug(format("Sending stream as S3 object %1$s in "
|
||||
+ "bucket %2$s", key, bucketName));
|
||||
}
|
||||
|
||||
acquireClient(clientOptions).putObject(bucketName, key, sourceStream, null);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
public static File getFile(final ClientOptions clientOptions,
|
||||
final String bucketName, final String key,
|
||||
|
|
@ -239,6 +257,7 @@ public final class S3Utils {
|
|||
|
||||
}
|
||||
|
||||
|
||||
public static void putDirectory(final ClientOptions clientOptions,
|
||||
final String bucketName, final File directory,
|
||||
final FilenameFilter fileNameFilter,
|
||||
|
|
@ -284,6 +303,8 @@ public final class S3Utils {
|
|||
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static void deleteObject(final ClientOptions clientOptions,
|
||||
final String bucketName, final String key) {
|
||||
|
||||
|
|
|
|||
|
|
@ -18,16 +18,34 @@ package com.cloud.utils;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.Inet6Address;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URLEncoder;
|
||||
import java.net.UnknownHostException;
|
||||
|
||||
import javax.net.ssl.HttpsURLConnection;
|
||||
|
||||
import org.apache.commons.httpclient.Credentials;
|
||||
import org.apache.commons.httpclient.HttpClient;
|
||||
import org.apache.commons.httpclient.HttpStatus;
|
||||
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
|
||||
import org.apache.commons.httpclient.UsernamePasswordCredentials;
|
||||
import org.apache.commons.httpclient.auth.AuthScope;
|
||||
import org.apache.commons.httpclient.methods.GetMethod;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
|
||||
public class UriUtils {
|
||||
|
||||
public static final Logger s_logger = Logger.getLogger(UriUtils.class.getName());
|
||||
|
||||
public static String formNfsUri(String host, String path) {
|
||||
try {
|
||||
URI uri = new URI("nfs", host, path, null);
|
||||
|
|
@ -111,4 +129,67 @@ public class UriUtils {
|
|||
}
|
||||
return remoteSize;
|
||||
}
|
||||
|
||||
public static Pair<String, Integer> validateUrl(String url) throws IllegalArgumentException {
|
||||
try {
|
||||
URI uri = new URI(url);
|
||||
if (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") ) {
|
||||
throw new IllegalArgumentException("Unsupported scheme for url");
|
||||
}
|
||||
int port = uri.getPort();
|
||||
if (!(port == 80 || port == 443 || port == -1)) {
|
||||
throw new IllegalArgumentException("Only ports 80 and 443 are allowed");
|
||||
}
|
||||
|
||||
if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) {
|
||||
port = 443;
|
||||
} else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) {
|
||||
port = 80;
|
||||
}
|
||||
|
||||
String host = uri.getHost();
|
||||
try {
|
||||
InetAddress hostAddr = InetAddress.getByName(host);
|
||||
if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) {
|
||||
throw new IllegalArgumentException("Illegal host specified in url");
|
||||
}
|
||||
if (hostAddr instanceof Inet6Address) {
|
||||
throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")");
|
||||
}
|
||||
return new Pair<String, Integer>(host, port);
|
||||
} catch (UnknownHostException uhe) {
|
||||
throw new IllegalArgumentException("Unable to resolve " + host);
|
||||
}
|
||||
} catch (URISyntaxException use) {
|
||||
throw new IllegalArgumentException("Invalid URL: " + url);
|
||||
}
|
||||
}
|
||||
|
||||
public static InputStream getInputStreamFromUrl(String url, String user, String password) {
|
||||
|
||||
try{
|
||||
Pair<String, Integer> hostAndPort = validateUrl(url);
|
||||
HttpClient httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
|
||||
if ((user != null) && (password != null)) {
|
||||
httpclient.getParams().setAuthenticationPreemptive(true);
|
||||
Credentials defaultcreds = new UsernamePasswordCredentials(user, password);
|
||||
httpclient.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds);
|
||||
s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second());
|
||||
}
|
||||
// Execute the method.
|
||||
GetMethod method = new GetMethod(url);
|
||||
int statusCode = httpclient.executeMethod(method);
|
||||
|
||||
if (statusCode != HttpStatus.SC_OK) {
|
||||
s_logger.error("Failed to read from URL: " + url);
|
||||
return null;
|
||||
}
|
||||
|
||||
return method.getResponseBodyAsStream();
|
||||
}
|
||||
catch (Exception ex){
|
||||
s_logger.error("Failed to read from URL: " + url);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue