diff --git a/api/src/com/cloud/agent/api/storage/DownloadCommand.java b/api/src/com/cloud/agent/api/storage/DownloadCommand.java index c6ffe45a9ef..77e7ae32b68 100644 --- a/api/src/com/cloud/agent/api/storage/DownloadCommand.java +++ b/api/src/com/cloud/agent/api/storage/DownloadCommand.java @@ -20,6 +20,7 @@ import java.net.URI; import org.apache.cloudstack.api.InternalIdentity; +import com.cloud.agent.api.to.DataStoreTO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Volume; import com.cloud.template.VirtualMachineTemplate; @@ -105,6 +106,8 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal private Long maxDownloadSizeInBytes = null; private long id; private ResourceType resourceType = ResourceType.TEMPLATE; + private DataStoreTO _store; + private Long resourceId; protected DownloadCommand() { } @@ -122,14 +125,16 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal this.resourceType = that.resourceType; } - public DownloadCommand(String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes) { + public DownloadCommand(DataStoreTO store, String secUrl, VirtualMachineTemplate template, Long maxDownloadSizeInBytes) { super(template.getUniqueName(), template.getUrl(), template.getFormat(), template.getAccountId()); + this._store = store; this.hvm = template.isRequiresHvm(); this.checksum = template.getChecksum(); this.id = template.getId(); this.description = template.getDisplayText(); this.setSecUrl(secUrl); this.maxDownloadSizeInBytes = maxDownloadSizeInBytes; + this.resourceId = template.getId(); } public DownloadCommand(String secUrl, Volume volume, Long maxDownloadSizeInBytes, String checkSum, String url, ImageFormat format) { @@ -216,4 +221,26 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal public void setResourceType(ResourceType resourceType) { this.resourceType = resourceType; } + + + public DataStoreTO getDataStore() { + return _store; + } + + + public void setDataStore(DataStoreTO _store) { + this._store = _store; + } + + + public Long getResourceId() { + return resourceId; + } + + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } + + } diff --git a/api/src/com/cloud/agent/api/to/NfsTO.java b/api/src/com/cloud/agent/api/to/NfsTO.java new file mode 100644 index 00000000000..5490fd1e588 --- /dev/null +++ b/api/src/com/cloud/agent/api/to/NfsTO.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +import com.cloud.storage.DataStoreRole; + +public final class NfsTO implements DataStoreTO { + + private String _url; + private DataStoreRole _role; + + public NfsTO() { + + super(); + + } + + public NfsTO(String url, DataStoreRole role) { + + super(); + + this._url = url; + this._role = role; + + } + + public String getUrl() { + return _url; + } + + public void setUrl(String _url) { + this._url = _url; + } + + @Override + public DataStoreRole getRole() { + return _role; + } + + public void setRole(DataStoreRole _role) { + this._role = _role; + } + + + +} diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index a8788c66b64..776ce29ec6a 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -34,7 +34,10 @@ import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; import java.net.InetAddress; +import java.net.MalformedURLException; import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -90,6 +93,8 @@ import com.cloud.agent.api.storage.ListVolumeAnswer; import com.cloud.agent.api.storage.ListVolumeCommand; import com.cloud.agent.api.storage.UploadCommand; import com.cloud.agent.api.storage.ssCommand; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; @@ -106,6 +111,7 @@ import com.cloud.storage.template.UploadManager; import com.cloud.storage.template.UploadManagerImpl; import com.cloud.utils.NumbersUtil; import com.cloud.utils.S3Utils; +import com.cloud.utils.UriUtils; import com.cloud.utils.S3Utils.FileNamingStrategy; import com.cloud.utils.S3Utils.ObjectNamingStrategy; import com.cloud.utils.component.ComponentContext; @@ -126,7 +132,7 @@ SecondaryStorageResource { int _timeout; - String _instance; + String _instance; String _dc; String _pod; String _guid; @@ -162,8 +168,8 @@ SecondaryStorageResource { if (cmd instanceof DownloadProgressCommand) { return _dlMgr.handleDownloadCommand(this, (DownloadProgressCommand)cmd); } else if (cmd instanceof DownloadCommand) { - return _dlMgr.handleDownloadCommand(this, (DownloadCommand)cmd); - } else if (cmd instanceof UploadCommand) { + return execute((DownloadCommand)cmd); + } else if (cmd instanceof UploadCommand) { return _upldMgr.handleUploadCommand(this, (UploadCommand)cmd); } else if (cmd instanceof CreateEntityDownloadURLCommand){ return _upldMgr.handleCreateEntityURLCommand((CreateEntityDownloadURLCommand)cmd); @@ -325,6 +331,51 @@ SecondaryStorageResource { } } + private Answer execute(DownloadCommand cmd){ + DataStoreTO dstore = cmd.getDataStore(); + if ( dstore instanceof NfsTO ){ + return _dlMgr.handleDownloadCommand(this, cmd); + } + else if ( dstore instanceof S3TO ){ + //TODO: how to handle download progress for S3 + S3TO s3 = (S3TO)cmd.getDataStore(); + String url = cmd.getUrl(); + String user = null; + String password = null; + if (cmd.getAuth() != null) { + user = cmd.getAuth().getUserName(); + password = new String(cmd.getAuth().getPassword()); + } + // get input stream from the given url + InputStream in = UriUtils.getInputStreamFromUrl(url, user, password); + URI uri; + URL urlObj; + try { + uri = new URI(url); + urlObj = new URL(url); + } catch (URISyntaxException e) { + throw new CloudRuntimeException("URI is incorrect: " + url); + } catch (MalformedURLException e) { + throw new CloudRuntimeException("URL is incorrect: " + url); + } + + final String bucket = s3.getBucketName(); + String key = join(asList(determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId()), urlObj.getFile()), S3Utils.SEPARATOR); + S3Utils.putObject(s3, in, bucket, key); + return new Answer(cmd, true, format("Uploaded the contents of input stream from %1$s for template id %2$s to S3 bucket %3$s", url, + cmd.getResourceId(), bucket)); + } + else if ( dstore instanceof SwiftTO ){ + //TODO: need to move code from execute(uploadTemplateToSwiftFromSecondaryStorageCommand) here, but we need to handle + // source is url, most likely we need to modify our existing swiftUpload python script. + return new Answer(cmd, false, "Swift is not currently support DownloadCommand"); + } + else{ + return new Answer(cmd, false, "Unsupport image data store: " + dstore); + } + + } + private Answer execute(uploadTemplateToSwiftFromSecondaryStorageCommand cmd) { SwiftTO swift = cmd.getSwift(); String secondaryStorageUrl = cmd.getSecondaryStorageUrl(); @@ -834,33 +885,33 @@ SecondaryStorageResource { String absoluteTemplatePath = parent + relativeTemplatePath; MessageDigest digest; String checksum = null; - File f = new File(absoluteTemplatePath); + File f = new File(absoluteTemplatePath); InputStream is = null; byte[] buffer = new byte[8192]; int read = 0; if(s_logger.isDebugEnabled()){ - s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath ); + s_logger.debug("parent path " +parent+ " relative template path " +relativeTemplatePath ); } try { - digest = MessageDigest.getInstance("MD5"); - is = new FileInputStream(f); + digest = MessageDigest.getInstance("MD5"); + is = new FileInputStream(f); while( (read = is.read(buffer)) > 0) { digest.update(buffer, 0, read); - } + } byte[] md5sum = digest.digest(); BigInteger bigInt = new BigInteger(1, md5sum); checksum = bigInt.toString(16); if(s_logger.isDebugEnabled()){ - s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+ " - " +checksum ); + s_logger.debug("Successfully calculated checksum for file " +absoluteTemplatePath+ " - " +checksum ); } }catch(IOException e) { String logMsg = "Unable to process file for MD5 - " + absoluteTemplatePath; s_logger.error(logMsg); - return new Answer(cmd, false, checksum); - }catch (NoSuchAlgorithmException e) { + return new Answer(cmd, false, checksum); + }catch (NoSuchAlgorithmException e) { return new Answer(cmd, false, checksum); } finally { @@ -869,10 +920,10 @@ SecondaryStorageResource { is.close(); } catch (IOException e) { if(s_logger.isDebugEnabled()){ - s_logger.debug("Could not close the file " +absoluteTemplatePath); + s_logger.debug("Could not close the file " +absoluteTemplatePath); } - return new Answer(cmd, false, checksum); - } + return new Answer(cmd, false, checksum); + } } return new Answer(cmd, true, checksum); @@ -1141,7 +1192,7 @@ SecondaryStorageResource { if (nfsIps.contains(cidr)) { /* * if the internal download ip is the same with secondary storage ip, adding internal sites will flush - * ip route to nfs through storage ip. + * ip route to nfs through storage ip. */ continue; } @@ -1243,7 +1294,7 @@ SecondaryStorageResource { for (PortConfig pCfg:cmd.getPortConfigs()){ if (pCfg.isAdd()) { - ipList.add(pCfg.getSourceIp()); + ipList.add(pCfg.getSourceIp()); } } boolean success = true; @@ -1401,7 +1452,7 @@ SecondaryStorageResource { String nfsPath = nfsHostIp + ":" + uri.getPath(); String dir = UUID.nameUUIDFromBytes(nfsPath.getBytes()).toString(); String root = _parent + "/" + dir; - mount(root, nfsPath); + mount(root, nfsPath); return root; } catch (Exception e) { String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString(); @@ -1470,7 +1521,7 @@ SecondaryStorageResource { String eth2ip = (String) params.get("eth2ip"); if (eth2ip != null) { params.put("public.network.device", "eth2"); - } + } _publicIp = (String) params.get("eth2ip"); _hostname = (String) params.get("name"); @@ -1677,7 +1728,7 @@ SecondaryStorageResource { command.add(String.valueOf(isAppend)); for (String ip : ipList){ command.add(ip); - } + } String result = command.execute(); if (result != null) { @@ -1693,7 +1744,7 @@ SecondaryStorageResource { s_logger.debug("create mount point: " + root); } else { s_logger.debug("Unable to create mount point: " + root); - return null; + return null; } } @@ -1813,13 +1864,13 @@ SecondaryStorageResource { @Override public void setName(String name) { // TODO Auto-generated method stub - + } @Override public void setConfigParams(Map params) { // TODO Auto-generated method stub - + } @Override @@ -1837,6 +1888,6 @@ SecondaryStorageResource { @Override public void setRunLevel(int level) { // TODO Auto-generated method stub - + } } diff --git a/core/src/com/cloud/storage/template/DownloadManagerImpl.java b/core/src/com/cloud/storage/template/DownloadManagerImpl.java index db264c17bb1..7eb68418b42 100755 --- a/core/src/com/cloud/storage/template/DownloadManagerImpl.java +++ b/core/src/com/cloud/storage/template/DownloadManagerImpl.java @@ -16,9 +16,15 @@ // under the License. package com.cloud.storage.template; +import static com.cloud.utils.S3Utils.putDirectory; +import static com.cloud.utils.StringUtils.join; +import static java.lang.String.format; +import static java.util.Arrays.asList; + import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.math.BigInteger; @@ -41,14 +47,21 @@ import java.util.concurrent.Executors; import javax.ejb.Local; import javax.naming.ConfigurationException; +import org.apache.commons.httpclient.Credentials; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthScope; import org.apache.log4j.Logger; +import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.DownloadCommand; import com.cloud.agent.api.storage.DownloadCommand.Proxy; import com.cloud.agent.api.storage.DownloadCommand.ResourceType; import com.cloud.agent.api.storage.DownloadProgressCommand; import com.cloud.agent.api.storage.DownloadProgressCommand.RequestType; +import com.cloud.agent.api.to.S3TO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; @@ -59,6 +72,9 @@ import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.storage.template.TemplateDownloader.DownloadCompleteCallback; import com.cloud.storage.template.TemplateDownloader.Status; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.S3Utils; +import com.cloud.utils.UriUtils; +import com.cloud.utils.S3Utils.ObjectNamingStrategy; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; @@ -224,7 +240,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager /** * Get notified of change of job status. Executed in context of downloader thread - * + * * @param jobId * the id of the job * @param status @@ -279,21 +295,21 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager byte[] buffer = new byte[8192]; int read = 0; MessageDigest digest; - String checksum = null; + String checksum = null; InputStream is = null; try { - digest = MessageDigest.getInstance("MD5"); - is = new FileInputStream(f); + digest = MessageDigest.getInstance("MD5"); + is = new FileInputStream(f); while( (read = is.read(buffer)) > 0) { digest.update(buffer, 0, read); - } + } byte[] md5sum = digest.digest(); BigInteger bigInt = new BigInteger(1, md5sum); checksum = String.format("%032x",bigInt); return checksum; }catch(IOException e) { return null; - }catch (NoSuchAlgorithmException e) { + }catch (NoSuchAlgorithmException e) { return null; } finally { @@ -302,19 +318,19 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager is.close(); } catch (IOException e) { return null; - } + } } } /** * Post download activity (install and cleanup). Executed in context of downloader thread - * + * * @throws IOException */ private String postDownload(String jobId) { DownloadJob dnld = jobs.get(jobId); TemplateDownloader td = dnld.getTemplateDownloader(); - String resourcePath = null; + String resourcePath = null; ResourceType resourceType = dnld.getResourceType(); // once template path is set, remove the parent dir so that the template is installed with a relative path @@ -458,7 +474,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return "Unable to create " + tmpDir; } // TO DO - define constant for volume properties. - File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : + File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + "volume.properties"); if ( file.exists() ) { file.delete(); @@ -583,6 +599,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager return convertStatus(getDownloadStatus(jobId)); } + @Override public DownloadAnswer handleDownloadCommand(SecondaryStorageResource resource, DownloadCommand cmd) { ResourceType resourceType = cmd.getResourceType(); @@ -774,7 +791,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } @Override - public Map gatherVolumeInfo(String rootDir) { + public Map gatherVolumeInfo(String rootDir) { Map result = new HashMap(); String volumeDir = rootDir + File.separator + _volumeDir; @@ -1046,4 +1063,5 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } } + } diff --git a/core/test/com/cloud/agent/transport/RequestTest.java b/core/test/com/cloud/agent/transport/RequestTest.java index 64c1e0bddef..048bd21db7c 100644 --- a/core/test/com/cloud/agent/transport/RequestTest.java +++ b/core/test/com/cloud/agent/transport/RequestTest.java @@ -31,19 +31,21 @@ import com.cloud.agent.api.SecStorageFirewallCfgCommand; import com.cloud.agent.api.UpdateHostPasswordCommand; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.DownloadCommand; +import com.cloud.agent.api.to.NfsTO; import com.cloud.exception.UnsupportedVersionException; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.serializer.GsonHelper; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.VMTemplateVO; /** - * - * - * - * + * + * + * + * */ public class RequestTest extends TestCase { @@ -51,7 +53,7 @@ public class RequestTest extends TestCase { public void testSerDeser() { s_logger.info("Testing serializing and deserializing works as expected"); - + s_logger.info("UpdateHostPasswordCommand should have two parameters that doesn't show in logging"); UpdateHostPasswordCommand cmd1 = new UpdateHostPasswordCommand("abc", "def"); s_logger.info("SecStorageFirewallCfgCommand has a context map that shouldn't show up in debug level"); @@ -89,7 +91,7 @@ public class RequestTest extends TestCase { logger.setLevel(level); byte[] bytes = sreq.getBytes(); - + assert Request.getSequence(bytes) == 892403717; assert Request.getManagementServerId(bytes) == 3; assert Request.getAgentId(bytes) == 2; @@ -130,7 +132,7 @@ public class RequestTest extends TestCase { s_logger.info("Testing Download answer"); VMTemplateVO template = new VMTemplateVO(1, "templatename", ImageFormat.QCOW2, true, true, true, TemplateType.USER, "url", true, 32, 1, "chksum", "displayText", true, 30, true, HypervisorType.KVM, null); - DownloadCommand cmd = new DownloadCommand("secUrl", template, 30000000l); + DownloadCommand cmd = new DownloadCommand(new NfsTO("secUrl", DataStoreRole.Image), "secUrl", template, 30000000l); Request req = new Request(1, 1, cmd, true); req.logD("Debug for Download"); @@ -161,7 +163,7 @@ public class RequestTest extends TestCase { } } } - + public void testLogging() { s_logger.info("Testing Logging"); GetHostStatsCommand cmd3 = new GetHostStatsCommand("hostguid", "hostname", 101); diff --git a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index efc2f99973b..e72bb54ddec 100644 --- a/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -28,7 +28,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectType; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreRole; + import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; @@ -65,6 +65,7 @@ import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.exception.StorageUnavailableException; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage.ImageFormat; @@ -130,7 +131,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { VolumeManager volumeMgr; @Inject private SwiftManager _swiftMgr; - @Inject + @Inject private S3Manager _s3Mgr; @Inject StorageCacheManager cacheMgr; @@ -314,12 +315,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } } } - + protected Answer cloneVolume(DataObject template, DataObject volume) { VolumeInfo volInfo = (VolumeInfo)volume; DiskOfferingVO offering = diskOfferingDao.findById(volInfo.getDiskOfferingId()); VMTemplateStoragePoolVO tmpltStoredOn = templatePoolDao.findByPoolTemplate(template.getDataStore().getId(), template.getId()); - + DiskProfile diskProfile = new DiskProfile(volInfo, offering, null); CreateCommand cmd = new CreateCommand(diskProfile, @@ -334,7 +335,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { s_logger.debug("Failed to send to storage pool", e); throw new CloudRuntimeException("Failed to send to storage pool", e); } - + if (answer.getResult()) { VolumeVO vol = this.volDao.findById(volume.getId()); CreateAnswer createAnswer = (CreateAnswer) answer; @@ -345,7 +346,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { vol.setPoolId(pool.getId()); vol.setPodId(pool.getPodId()); this.volDao.update(vol.getId(), vol); - + } else { if (tmpltStoredOn != null && (answer instanceof CreateAnswer) @@ -354,15 +355,15 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { if (!templateMgr .resetTemplateDownloadStateOnPool(tmpltStoredOn .getId())) { - + } } errMsg = answer.getDetails(); } - + return answer; } - + protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData) { VolumeInfo volume = (VolumeInfo)srcData; VolumeInfo destVolume = (VolumeInfo)destData; @@ -370,9 +371,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { .getDataCenterId()); StoragePool srcPool = (StoragePool)this.dataStoreMgr.getDataStore(volume .getPoolId(), DataStoreRole.Primary); - + StoragePool destPool = (StoragePool)this.dataStoreMgr.getDataStore(destVolume.getPoolId(), DataStoreRole.Primary); - + String value = this.configDao.getValue(Config.CopyVolumeWait.toString()); int _copyvolumewait = NumbersUtil.parseInt(value, Integer.parseInt(Config.CopyVolumeWait.getDefaultValue())); @@ -394,7 +395,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } String secondaryStorageVolumePath = cvAnswer.getVolumePath(); - + cvCmd = new CopyVolumeCommand(volume.getId(), secondaryStorageVolumePath, destPool, secondaryStorageURL, false, _copyvolumewait); @@ -409,7 +410,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException( "Failed to copy the volume from secondary storage to the destination primary storage pool."); } - + VolumeVO destVol = this.volDao.findById(destVolume.getId()); destVol.setPath(cvAnswer.getVolumePath()); this.volDao.update(destVol.getId(), destVol); @@ -437,7 +438,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } else if (srcData.getType() == DataObjectType.VOLUME && destData.getType() == DataObjectType.TEMPLATE) { answer = createTemplateFromVolume(srcData, destData); - } else if (srcData.getType() == DataObjectType.TEMPLATE + } else if (srcData.getType() == DataObjectType.TEMPLATE && destData.getType() == DataObjectType.VOLUME) { answer = cloneVolume(srcData, destData); } else if (destData.getType() == DataObjectType.VOLUME @@ -667,21 +668,21 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { return sendCommand(cmd, pool, template.getId(), zoneId, secondaryStorageHost.getId()); } - + private HostVO getSecHost(long volumeId, long dcId) { Long id = snapshotDao.getSecHostId(volumeId); - if ( id != null) { + if ( id != null) { return hostDao.findById(id); } return this.templateMgr.getSecondaryStorageHost(dcId); } - + protected Answer copySnapshot(DataObject srcObject, DataObject destObject) { SnapshotInfo srcSnapshot = (SnapshotInfo)srcObject; VolumeInfo baseVolume = srcSnapshot.getBaseVolume(); Long dcId = baseVolume.getDataCenterId(); Long accountId = baseVolume.getAccountId(); - + HostVO secHost = getSecHost(baseVolume.getId(), baseVolume.getDataCenterId()); Long secHostId = secHost.getId(); String secondaryStoragePoolUrl = secHost.getStorageUrl(); @@ -696,12 +697,12 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { SwiftTO swift = _swiftMgr.getSwiftTO(); S3TO s3 = _s3Mgr.getS3TO(); - + long prevSnapshotId = srcSnapshot.getPrevSnapshotId(); if (prevSnapshotId > 0) { prevSnapshot = snapshotDao.findByIdIncludingRemoved(prevSnapshotId); if ( prevSnapshot.getBackupSnapshotId() != null && swift == null) { - if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2")) { + if (prevSnapshot.getVersion() != null && prevSnapshot.getVersion().equals("2.2")) { prevBackupUuid = prevSnapshot.getBackupSnapshotId(); prevSnapshotUuid = prevSnapshot.getPath(); } diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java index 4989b2629a2..bc8d2e2a10f 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java @@ -175,7 +175,7 @@ public class ImageStoreImpl implements ImageStoreEntity { @Override public DataStoreTO getTO() { - return null; + return getDriver().getStoreTO(this); } diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 0feefd0528c..8be1c8a7e47 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -83,10 +83,10 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { private VolumeDao volumeDao; - public PrimaryDataStoreImpl() { - + public PrimaryDataStoreImpl() { + } - + public void configure(StoragePoolVO pdsv, PrimaryDataStoreDriver driver, DataStoreProvider provider) { this.pdsv = pdsv; @@ -245,9 +245,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { } } } - + } - + return objectInStoreMgr.get(obj, this); } @@ -341,9 +341,9 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { public String getStorageProviderName() { return this.pdsv.getStorageProviderName(); } - + @Override public DataStoreTO getTO() { - return null; + return getDriver().getStoreTO(this); } } diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java index af6a3b63492..f90238571dc 100644 --- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java +++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java @@ -46,6 +46,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.DeleteSnapshotBackupCommand; import com.cloud.agent.api.storage.DeleteVolumeCommand; import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.host.HostVO; @@ -106,10 +107,9 @@ public class CloudStackImageStoreDriverImpl implements ImageStoreDriver { @Override public DataStoreTO getStoreTO(DataStore store) { ImageStoreImpl nfsStore = (ImageStoreImpl)store; - ImageStoreTO nfsTO = new ImageStoreTO(); - nfsTO.setProviderName("CloudStack"); + NfsTO nfsTO = new NfsTO(); nfsTO.setRole(DataStoreRole.Image); - nfsTO.setUri(nfsStore.getUri()); + nfsTO.setUrl(nfsStore.getUri()); return nfsTO; } diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index 507d57531fe..58477b43e90 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -366,7 +366,7 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor if(vmTemplateStore != null) { start(); DownloadCommand dcmd = - new DownloadCommand(secUrl, template, maxTemplateSizeInBytes); + new DownloadCommand(store.getTO(), secUrl, template, maxTemplateSizeInBytes); dcmd.setProxy(getHttpProxy()); if (downloadJobExists) { dcmd = new DownloadProgressCommand(dcmd, vmTemplateStore.getJobId(), RequestType.GET_OR_RESTART); diff --git a/utils/src/com/cloud/utils/S3Utils.java b/utils/src/com/cloud/utils/S3Utils.java index b7273a14869..423ea7138a9 100644 --- a/utils/src/com/cloud/utils/S3Utils.java +++ b/utils/src/com/cloud/utils/S3Utils.java @@ -138,6 +138,24 @@ public final class S3Utils { } + public static void putObject(final ClientOptions clientOptions, + final InputStream sourceStream, final String bucketName, final String key) { + + assert clientOptions != null; + assert sourceStream != null; + assert !isBlank(bucketName); + assert !isBlank(key); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(format("Sending stream as S3 object %1$s in " + + "bucket %2$s", key, bucketName)); + } + + acquireClient(clientOptions).putObject(bucketName, key, sourceStream, null); + + } + + @SuppressWarnings("unchecked") public static File getFile(final ClientOptions clientOptions, final String bucketName, final String key, @@ -239,6 +257,7 @@ public final class S3Utils { } + public static void putDirectory(final ClientOptions clientOptions, final String bucketName, final File directory, final FilenameFilter fileNameFilter, @@ -284,6 +303,8 @@ public final class S3Utils { } + + public static void deleteObject(final ClientOptions clientOptions, final String bucketName, final String key) { diff --git a/utils/src/com/cloud/utils/UriUtils.java b/utils/src/com/cloud/utils/UriUtils.java index 3bcee7aeac4..ef1aba24a00 100644 --- a/utils/src/com/cloud/utils/UriUtils.java +++ b/utils/src/com/cloud/utils/UriUtils.java @@ -18,16 +18,34 @@ package com.cloud.utils; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.net.HttpURLConnection; +import java.net.Inet6Address; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; +import java.net.UnknownHostException; import javax.net.ssl.HttpsURLConnection; +import org.apache.commons.httpclient.Credentials; +import org.apache.commons.httpclient.HttpClient; +import org.apache.commons.httpclient.HttpStatus; +import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; +import org.apache.commons.httpclient.UsernamePasswordCredentials; +import org.apache.commons.httpclient.auth.AuthScope; +import org.apache.commons.httpclient.methods.GetMethod; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.log4j.Logger; + import com.cloud.utils.exception.CloudRuntimeException; public class UriUtils { + + public static final Logger s_logger = Logger.getLogger(UriUtils.class.getName()); + public static String formNfsUri(String host, String path) { try { URI uri = new URI("nfs", host, path, null); @@ -111,4 +129,67 @@ public class UriUtils { } return remoteSize; } + + public static Pair validateUrl(String url) throws IllegalArgumentException { + try { + URI uri = new URI(url); + if (!uri.getScheme().equalsIgnoreCase("http") && !uri.getScheme().equalsIgnoreCase("https") ) { + throw new IllegalArgumentException("Unsupported scheme for url"); + } + int port = uri.getPort(); + if (!(port == 80 || port == 443 || port == -1)) { + throw new IllegalArgumentException("Only ports 80 and 443 are allowed"); + } + + if (port == -1 && uri.getScheme().equalsIgnoreCase("https")) { + port = 443; + } else if (port == -1 && uri.getScheme().equalsIgnoreCase("http")) { + port = 80; + } + + String host = uri.getHost(); + try { + InetAddress hostAddr = InetAddress.getByName(host); + if (hostAddr.isAnyLocalAddress() || hostAddr.isLinkLocalAddress() || hostAddr.isLoopbackAddress() || hostAddr.isMulticastAddress()) { + throw new IllegalArgumentException("Illegal host specified in url"); + } + if (hostAddr instanceof Inet6Address) { + throw new IllegalArgumentException("IPV6 addresses not supported (" + hostAddr.getHostAddress() + ")"); + } + return new Pair(host, port); + } catch (UnknownHostException uhe) { + throw new IllegalArgumentException("Unable to resolve " + host); + } + } catch (URISyntaxException use) { + throw new IllegalArgumentException("Invalid URL: " + url); + } + } + + public static InputStream getInputStreamFromUrl(String url, String user, String password) { + + try{ + Pair hostAndPort = validateUrl(url); + HttpClient httpclient = new HttpClient(new MultiThreadedHttpConnectionManager()); + if ((user != null) && (password != null)) { + httpclient.getParams().setAuthenticationPreemptive(true); + Credentials defaultcreds = new UsernamePasswordCredentials(user, password); + httpclient.getState().setCredentials(new AuthScope(hostAndPort.first(), hostAndPort.second(), AuthScope.ANY_REALM), defaultcreds); + s_logger.info("Added username=" + user + ", password=" + password + "for host " + hostAndPort.first() + ":" + hostAndPort.second()); + } + // Execute the method. + GetMethod method = new GetMethod(url); + int statusCode = httpclient.executeMethod(method); + + if (statusCode != HttpStatus.SC_OK) { + s_logger.error("Failed to read from URL: " + url); + return null; + } + + return method.getResponseBodyAsStream(); + } + catch (Exception ex){ + s_logger.error("Failed to read from URL: " + url); + return null; + } + } }