From 8f4575e877ca397ec1913cfe522b7829ed476b6a Mon Sep 17 00:00:00 2001 From: Edison Su Date: Sat, 13 Jul 2013 01:51:55 -0700 Subject: [PATCH] CLOUDSTACK-3400: add swift support --- api/src/com/cloud/agent/api/to/SwiftTO.java | 16 +- client/tomcatconf/applicationContext.xml.in | 3 - client/tomcatconf/commands.properties.in | 4 - .../storage/command/DownloadCommand.java | 9 +- .../com/cloud/storage/VMTemplateSwiftVO.java | 108 ------- .../cloud/storage/dao/VMTemplateSwiftDao.java | 37 --- .../storage/dao/VMTemplateSwiftDaoImpl.java | 87 ------ .../storage/test/ChildTestConfiguration.java | 6 - .../driver/SwiftImageStoreDriverImpl.java | 6 +- server/src/com/cloud/api/ApiDBUtils.java | 9 - .../com/cloud/storage/VolumeManagerImpl.java | 3 - .../cloud/template/TemplateManagerImpl.java | 3 - .../resource/NfsSecondaryStorageResource.java | 270 ++++++++++-------- utils/src/com/cloud/utils/SwiftUtil.java | 111 +++++++ 14 files changed, 279 insertions(+), 393 deletions(-) delete mode 100755 engine/schema/src/com/cloud/storage/VMTemplateSwiftVO.java delete mode 100755 engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDao.java delete mode 100755 engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDaoImpl.java create mode 100644 utils/src/com/cloud/utils/SwiftUtil.java diff --git a/api/src/com/cloud/agent/api/to/SwiftTO.java b/api/src/com/cloud/agent/api/to/SwiftTO.java index 5d74003216e..7349d7779ac 100644 --- a/api/src/com/cloud/agent/api/to/SwiftTO.java +++ b/api/src/com/cloud/agent/api/to/SwiftTO.java @@ -17,26 +17,25 @@ package com.cloud.agent.api.to; import com.cloud.storage.DataStoreRole; +import com.cloud.utils.SwiftUtil; -public class SwiftTO implements DataStoreTO { +public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg { Long id; String url; String account; String userName; String key; - String container; public SwiftTO() { } - public SwiftTO(Long id, String url, String account, String userName, String key, - String container) { + public SwiftTO(Long id, String url, String account, String userName, String key + ) { this.id = id; this.url = url; this.account = account; this.userName = userName; this.key = key; - this.container = container; } public Long getId() { @@ -64,9 +63,8 @@ public class SwiftTO implements DataStoreTO { return DataStoreRole.Image; } - public String getContainer() { - return this.container; + @Override + public String getEndPoint() { + return this.url; } - - } diff --git a/client/tomcatconf/applicationContext.xml.in b/client/tomcatconf/applicationContext.xml.in index 610fdfdcb5b..c717b560a1c 100644 --- a/client/tomcatconf/applicationContext.xml.in +++ b/client/tomcatconf/applicationContext.xml.in @@ -316,7 +316,6 @@ - @@ -356,7 +355,6 @@ - @@ -793,7 +791,6 @@ - diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index e20af7c2d58..ad7047151b3 100644 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -241,10 +241,6 @@ deleteAlerts=1 #### system capacity commands listCapacity=3 -#### swift commands -addSwift=1 -listSwifts=1 - #### s3 commands addS3=1 listS3s=1 diff --git a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java index 7e3d65c7bde..84dd59db9f6 100644 --- a/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java +++ b/core/src/org/apache/cloudstack/storage/command/DownloadCommand.java @@ -45,6 +45,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal private ResourceType resourceType = ResourceType.TEMPLATE; private String installPath; private DataStoreTO _store; + private DataStoreTO cacheStore; protected DownloadCommand() { } @@ -178,7 +179,11 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal this.installPath = installPath; } + public void setCacheStore(DataStoreTO cacheStore) { + this.cacheStore = cacheStore; + } - - + public DataStoreTO getCacheStore() { + return this.cacheStore; + } } diff --git a/engine/schema/src/com/cloud/storage/VMTemplateSwiftVO.java b/engine/schema/src/com/cloud/storage/VMTemplateSwiftVO.java deleted file mode 100755 index de55fb6622f..00000000000 --- a/engine/schema/src/com/cloud/storage/VMTemplateSwiftVO.java +++ /dev/null @@ -1,108 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.storage; - -import java.util.Date; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -import com.cloud.utils.db.GenericDaoBase; -import org.apache.cloudstack.api.InternalIdentity; - -/** - * Join table for swift and templates - * - * - */ -@Entity -@Table(name = "template_swift_ref") -public class VMTemplateSwiftVO implements InternalIdentity { - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - Long id; - - @Column(name = "swift_id") - private long swiftId; - - @Column(name = "template_id") - private long templateId; - - @Column(name = GenericDaoBase.CREATED_COLUMN) - private Date created = null; - - @Column(name = "path") - private String path; - - @Column(name = "size") - private long size; - - @Column(name = "physical_size") - private long physicalSize; - - public VMTemplateSwiftVO(long swiftId, long templateId, Date created, String path, long size, long physicalSize) { - this.swiftId = swiftId; - this.templateId = templateId; - this.created = created; - this.path = path; - this.size = size; - this.physicalSize = physicalSize; - } - - protected VMTemplateSwiftVO() { - - } - - public long getTemplateId() { - return templateId; - } - - public long getId() { - return id; - } - - public Date getCreated() { - return created; - } - - public String getPath() { - return path; - } - - public long getSwiftId() { - return swiftId; - } - - public long getSize() { - return size; - } - - public long getPhysicalSize() { - return physicalSize; - } - - @Override - public String toString() { - return new StringBuilder("TmplSwift[").append(id).append("-").append(templateId).append("-").append(swiftId) - .append("]").toString(); - } - -} diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDao.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDao.java deleted file mode 100755 index ec181895c4c..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDao.java +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.storage.dao; - -import java.util.List; - -import com.cloud.storage.VMTemplateSwiftVO; -import com.cloud.utils.db.GenericDao; - -/** - * - * - */ - -public interface VMTemplateSwiftDao extends GenericDao { - List listBySwiftId(long id); - - List listByTemplateId(long templateId); - - VMTemplateSwiftVO findBySwiftTemplate(long swiftId, long templateId); - - VMTemplateSwiftVO findOneByTemplateId(long templateId); -} diff --git a/engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDaoImpl.java deleted file mode 100755 index c65527af202..00000000000 --- a/engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDaoImpl.java +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.storage.dao; - -import java.util.Collections; -import java.util.List; - -import javax.ejb.Local; - -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - -import com.cloud.storage.VMTemplateSwiftVO; -import com.cloud.utils.db.GenericDaoBase; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; - -/** - * - * - */ - -@Component -@Local(value = { VMTemplateSwiftDao.class }) -public class VMTemplateSwiftDaoImpl extends GenericDaoBase implements VMTemplateSwiftDao { - public static final Logger s_logger = Logger.getLogger(VMTemplateSwiftDaoImpl.class.getName()); - - protected final SearchBuilder AllFieldSearch; - - public VMTemplateSwiftDaoImpl() { - AllFieldSearch = createSearchBuilder(); - AllFieldSearch.and("swift_id", AllFieldSearch.entity().getSwiftId(), SearchCriteria.Op.EQ); - AllFieldSearch.and("template_id", AllFieldSearch.entity().getTemplateId(), SearchCriteria.Op.EQ); - AllFieldSearch.done(); - - } - - @Override - public List listBySwiftId(long id) { - SearchCriteria sc = AllFieldSearch.create(); - sc.setParameters("swift_id", id); - return listBy(sc); - } - - @Override - public List listByTemplateId(long templateId) { - SearchCriteria sc = AllFieldSearch.create(); - sc.setParameters("template_id", templateId); - return listBy(sc); - } - - @Override - public VMTemplateSwiftVO findOneByTemplateId(long templateId) { - SearchCriteria sc = AllFieldSearch.create(); - sc.setParameters("template_id", templateId); - List list = listBy(sc); - if (list == null || list.size() < 1) { - return null; - } else { - Collections.shuffle(list); - return list.get(0); - } - } - - @Override - public VMTemplateSwiftVO findBySwiftTemplate(long swiftId, long templateId) { - SearchCriteria sc = AllFieldSearch.create(); - sc.setParameters("swift_id", swiftId); - sc.setParameters("template_id", templateId); - return findOneBy(sc); - } - -} diff --git a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java index 60f86d86ddb..ebd6e39155b 100644 --- a/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java +++ b/engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java @@ -79,7 +79,6 @@ import com.cloud.storage.download.DownloadMonitorImpl; import com.cloud.storage.s3.S3Manager; import com.cloud.storage.secondary.SecondaryStorageVmManager; import com.cloud.storage.snapshot.SnapshotManager; -import com.cloud.storage.swift.SwiftManager; import com.cloud.tags.dao.ResourceTagsDaoImpl; import com.cloud.template.TemplateManager; import com.cloud.user.AccountManager; @@ -177,11 +176,6 @@ public class ChildTestConfiguration extends TestConfiguration { return Mockito.mock(VolumeManager.class); } - @Bean - public SwiftManager switfMgr() { - return Mockito.mock(SwiftManager.class); - } - @Bean public ManagementServer server() { return Mockito.mock(ManagementServer.class); diff --git a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java index d9718507823..527d3aad8ec 100644 --- a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java +++ b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java @@ -53,13 +53,15 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { ImageStoreDetailsDao _imageStoreDetailsDao; @Inject EndPointSelector _epSelector; + @Inject + StorageCacheManager cacheManager; @Override public DataStoreTO getStoreTO(DataStore store) { ImageStoreImpl imgStore = (ImageStoreImpl) store; Map details = _imageStoreDetailsDao.getDetails(imgStore.getId()); return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT), - details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.S3_BUCKET_NAME)); + details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY)); } @Override @@ -71,7 +73,9 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl { public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes(); VirtualMachineTemplate tmpl = _templateDao.findById(data.getId()); + DataStore cacheStore = cacheManager.getCacheStorage(dataStore.getScope()); DownloadCommand dcmd = new DownloadCommand((TemplateObjectTO)(data.getTO()), maxTemplateSizeInBytes); + dcmd.setCacheStore(cacheStore.getTO()); dcmd.setProxy(getHttpProxy()); EndPoint ep = _epSelector.select(data); diff --git a/server/src/com/cloud/api/ApiDBUtils.java b/server/src/com/cloud/api/ApiDBUtils.java index 8e4a23e89bd..2cab44d5e4d 100755 --- a/server/src/com/cloud/api/ApiDBUtils.java +++ b/server/src/com/cloud/api/ApiDBUtils.java @@ -236,7 +236,6 @@ import com.cloud.storage.StorageStats; import com.cloud.storage.UploadVO; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateS3VO; -import com.cloud.storage.VMTemplateSwiftVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.Volume.Type; @@ -253,7 +252,6 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VMTemplateS3Dao; -import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeHostDao; import com.cloud.storage.snapshot.SnapshotPolicy; @@ -336,7 +334,6 @@ public class ApiDBUtils { static PrimaryDataStoreDao _storagePoolDao; static VMTemplateDao _templateDao; static VMTemplateDetailsDao _templateDetailsDao; - static VMTemplateSwiftDao _templateSwiftDao; static VMTemplateS3Dao _templateS3Dao; static UploadDao _uploadDao; static UserDao _userDao; @@ -445,7 +442,6 @@ public class ApiDBUtils { @Inject private PrimaryDataStoreDao storagePoolDao; @Inject private VMTemplateDao templateDao; @Inject private VMTemplateDetailsDao templateDetailsDao; - @Inject private VMTemplateSwiftDao templateSwiftDao; @Inject private VMTemplateS3Dao templateS3Dao; @Inject private UploadDao uploadDao; @Inject private UserDao userDao; @@ -554,7 +550,6 @@ public class ApiDBUtils { _storagePoolDao = storagePoolDao; _templateDao = templateDao; _templateDetailsDao = templateDetailsDao; - _templateSwiftDao = templateSwiftDao; _templateS3Dao = templateS3Dao; _uploadDao = uploadDao; _userDao = userDao; @@ -894,10 +889,6 @@ public class ApiDBUtils { return template; } - public static VMTemplateSwiftVO findTemplateSwiftRef(long templateId) { - return _templateSwiftDao.findOneByTemplateId(templateId); - } - public static VMTemplateS3VO findTemplateS3Ref(long templateId) { return _templateS3Dao.findOneByTemplateId(templateId); } diff --git a/server/src/com/cloud/storage/VolumeManagerImpl.java b/server/src/com/cloud/storage/VolumeManagerImpl.java index c935703031d..72d27bca1bb 100644 --- a/server/src/com/cloud/storage/VolumeManagerImpl.java +++ b/server/src/com/cloud/storage/VolumeManagerImpl.java @@ -139,7 +139,6 @@ import com.cloud.storage.dao.UploadDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VMTemplateS3Dao; -import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.storage.download.DownloadMonitor; @@ -237,8 +236,6 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager { @Inject protected VMTemplatePoolDao _vmTemplatePoolDao = null; @Inject - protected VMTemplateSwiftDao _vmTemplateSwiftDao = null; - @Inject protected VMTemplateS3Dao _vmTemplateS3Dao; @Inject protected S3Manager _s3Mgr; diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 8dcb0ed07d9..18a69ca092e 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -151,7 +151,6 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.dao.VMTemplateS3Dao; -import com.cloud.storage.dao.VMTemplateSwiftDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; @@ -232,8 +231,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject SnapshotDao _snapshotDao; @Inject - VMTemplateSwiftDao _tmpltSwiftDao; - @Inject VMTemplateS3Dao _vmS3TemplateDao; @Inject ConfigurationDao _configDao; diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index cefc08a131a..77c019e2f56 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -23,12 +23,7 @@ import static java.lang.String.format; import static java.util.Arrays.asList; import static org.apache.commons.lang.StringUtils.substringAfterLast; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.math.BigInteger; import java.net.InetAddress; import java.net.URI; @@ -43,6 +38,10 @@ import java.util.concurrent.Callable; import javax.naming.ConfigurationException; +import com.cloud.agent.api.storage.*; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.template.*; +import com.cloud.utils.SwiftUtil; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DeleteCommand; @@ -56,7 +55,13 @@ import org.apache.cloudstack.storage.template.UploadManagerImpl; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.DefaultHttpClient; import org.apache.log4j.Logger; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -84,13 +89,6 @@ import com.cloud.agent.api.SecStorageVMSetupCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupSecondaryStorageCommand; import com.cloud.agent.api.UploadTemplateToSwiftFromSecondaryStorageCommand; -import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand; -import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand; -import com.cloud.agent.api.storage.ListTemplateAnswer; -import com.cloud.agent.api.storage.ListTemplateCommand; -import com.cloud.agent.api.storage.ListVolumeAnswer; -import com.cloud.agent.api.storage.ListVolumeCommand; -import com.cloud.agent.api.storage.UploadCommand; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; @@ -105,11 +103,7 @@ import com.cloud.resource.ServerResourceBase; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; -import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; -import com.cloud.storage.template.TemplateLocation; -import com.cloud.storage.template.TemplateProp; -import com.cloud.storage.template.VhdProcessor; import com.cloud.utils.NumbersUtil; import com.cloud.utils.S3Utils; import com.cloud.utils.S3Utils.FileNamingStrategy; @@ -219,6 +213,86 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + + protected CopyCmdAnswer postProcessing(File destFile, String downloadPath, String destPath, DataTO srcData, DataTO destData) throws ConfigurationException { + // do post processing to unzip the file if it is compressed + String scriptsDir = "scripts/storage/secondary"; + String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh"); + if (createTmpltScr == null) { + throw new ConfigurationException("Unable to find createtmplt.sh"); + } + s_logger.info("createtmplt.sh found in " + createTmpltScr); + String createVolScr = Script.findScript(scriptsDir, "createvolume.sh"); + if (createVolScr == null) { + throw new ConfigurationException("Unable to find createvolume.sh"); + } + s_logger.info("createvolume.sh found in " + createVolScr); + String script = srcData.getObjectType() == DataObjectType.TEMPLATE ? createTmpltScr : createVolScr; + + int installTimeoutPerGig = 180 * 60 * 1000; + int imgSizeGigs = (int) Math.ceil(destFile.length() * 1.0d / (1024 * 1024 * 1024)); + imgSizeGigs++; // add one just in case + long timeout = imgSizeGigs * installTimeoutPerGig; + + String origPath = destFile.getAbsolutePath(); + String extension = null; + if (srcData.getObjectType() == DataObjectType.TEMPLATE) { + extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension(); + } else { + extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension(); + } + + String templateName = UUID.randomUUID().toString(); + String templateFilename = templateName + "." + extension; + Script scr = new Script(script, timeout, s_logger); + scr.add("-s", Integer.toString(imgSizeGigs)); // not used for now + scr.add("-n", templateFilename); + + scr.add("-t", downloadPath); + scr.add("-f", origPath); // this is the temporary + // template file downloaded + String result; + result = scr.execute(); + + if (result != null) { + // script execution failure + throw new CloudRuntimeException("Failed to run script " + script); + } + + String finalFileName = templateFilename; + String finalDownloadPath = destPath + File.separator + templateFilename; + // compute the size of + long size = this._storage.getSize(downloadPath + File.separator + templateFilename); + + DataTO newDestTO = null; + + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + TemplateObjectTO newTemplTO = new TemplateObjectTO(); + newTemplTO.setPath(finalDownloadPath); + newTemplTO.setName(finalFileName); + newTemplTO.setSize(size); + newDestTO = newTemplTO; + } else { + return new CopyCmdAnswer("not implemented yet"); + } + + return new CopyCmdAnswer(newDestTO); + } + protected Answer copyFromSwiftToNfs(CopyCommand cmd, DataTO srcData, SwiftTO swiftTO, DataTO destData, NfsTO destImageStore) { + final String storagePath = destImageStore.getUrl(); + final String destPath = destData.getPath(); + try { + String downloadPath = determineStorageTemplatePath(storagePath, destPath); + final File downloadDirectory = _storage.getFile(downloadPath); + downloadDirectory.mkdirs(); + File destFile = SwiftUtil.getObject(swiftTO, downloadDirectory, srcData.getPath()); + return postProcessing(destFile,downloadPath,destPath,srcData,destData); + } catch (Exception e) { + s_logger.debug("Failed to copy swift to nfs", e); + return new CopyCmdAnswer(e.toString()); + } + } + protected Answer copyFromS3ToNfs(CopyCommand cmd, DataTO srcData, S3TO s3, DataTO destData, NfsTO destImageStore) { final String storagePath = destImageStore.getUrl(); final String destPath = destData.getPath(); @@ -249,68 +323,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new CopyCmdAnswer("Can't find template"); } - // do post processing to unzip the file if it is compressed - String scriptsDir = "scripts/storage/secondary"; - String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh"); - if (createTmpltScr == null) { - throw new ConfigurationException("Unable to find createtmplt.sh"); - } - s_logger.info("createtmplt.sh found in " + createTmpltScr); - String createVolScr = Script.findScript(scriptsDir, "createvolume.sh"); - if (createVolScr == null) { - throw new ConfigurationException("Unable to find createvolume.sh"); - } - s_logger.info("createvolume.sh found in " + createVolScr); - String script = srcData.getObjectType() == DataObjectType.TEMPLATE ? createTmpltScr : createVolScr; - - int installTimeoutPerGig = 180 * 60 * 1000; - int imgSizeGigs = (int) Math.ceil(destFile.length() * 1.0d / (1024 * 1024 * 1024)); - imgSizeGigs++; // add one just in case - long timeout = imgSizeGigs * installTimeoutPerGig; - - String origPath = destFile.getAbsolutePath(); - String extension = null; - if (srcData.getObjectType() == DataObjectType.TEMPLATE) { - extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension(); - } else { - extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension(); - } - - String templateName = UUID.randomUUID().toString(); - String templateFilename = templateName + "." + extension; - Script scr = new Script(script, timeout, s_logger); - scr.add("-s", Integer.toString(imgSizeGigs)); // not used for now - scr.add("-n", templateFilename); - - scr.add("-t", downloadPath); - scr.add("-f", origPath); // this is the temporary - // template file downloaded - String result; - result = scr.execute(); - - if (result != null) { - // script execution failure - throw new CloudRuntimeException("Failed to run script " + script); - } - - String finalFileName = templateFilename; - String finalDownloadPath = destPath + File.separator + templateFilename; - // compute the size of - long size = this._storage.getSize(downloadPath + File.separator + templateFilename); - - DataTO newDestTO = null; - - if (destData.getObjectType() == DataObjectType.TEMPLATE) { - TemplateObjectTO newTemplTO = new TemplateObjectTO(); - newTemplTO.setPath(finalDownloadPath); - newTemplTO.setName(finalFileName); - newTemplTO.setSize(size); - newDestTO = newTemplTO; - } else { - return new CopyCmdAnswer("not implemented yet"); - } - - return new CopyCmdAnswer(newDestTO); + return postProcessing(destFile,downloadPath,destPath,srcData,destData); } catch (Exception e) { final String errMsg = format("Failed to download" + "due to $2%s", e.getMessage()); @@ -432,11 +445,15 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return createTemplateFromSnapshot(cmd); } - if (srcDataStore instanceof S3TO && destDataStore instanceof NfsTO + if (destDataStore instanceof NfsTO && destDataStore.getRole() == DataStoreRole.ImageCache) { - S3TO s3 = (S3TO) srcDataStore; NfsTO destImageStore = (NfsTO) destDataStore; - return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore); + if (srcDataStore instanceof S3TO) { + S3TO s3 = (S3TO) srcDataStore; + return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore); + } else if (srcDataStore instanceof SwiftTO) { + return copyFromSwiftToNfs(cmd, srcData, (SwiftTO)srcDataStore, destData, destImageStore); + } } if (srcDataStore.getRole() == DataStoreRole.ImageCache && destDataStore.getRole() == DataStoreRole.Image) { @@ -519,51 +536,62 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } - protected Answer registerTemplateOnSwift(DownloadCommand cmd) { + protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path) { + HttpClient client = new DefaultHttpClient(); + HttpGet get = new HttpGet(url); + try { + HttpResponse response = client.execute(get); + HttpEntity entity = response.getEntity(); + if (entity == null) { + s_logger.debug("Faled to get entity"); + throw new CloudRuntimeException("Failed to get url: " + url); + } - return null; + String nfsMountPath = getRootDir(nfs.getUrl()); + String filePath = nfsMountPath + File.separator + path; + FileOutputStream outputStream = new FileOutputStream(filePath); + entity.writeTo(outputStream); + return new File(filePath); + } catch (IOException e) { + s_logger.debug("Faild to get url:"+ url + ", due to " + e.toString()); + throw new CloudRuntimeException(e); + } } + protected Answer registerTemplateOnSwift(DownloadCommand cmd) { + SwiftTO swiftTO = (SwiftTO)cmd.getDataStore(); + String path = cmd.getInstallPath(); + DataStoreTO cacheStore = cmd.getCacheStore(); + if (cacheStore == null || !(cacheStore instanceof NfsTO)) { + return new DownloadAnswer("cache store can't be null", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } + + try { + NfsTO nfsCacheStore = (NfsTO)cacheStore; + File file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path); + String swiftPath = SwiftUtil.putObject(swiftTO, file, "T-" + cmd.getId()); + String md5sum = null; + try { + md5sum = DigestUtils.md5Hex(new FileInputStream(file)); + } catch (IOException e) { + s_logger.debug("Failed to get md5sum: " + file.getAbsoluteFile()); + } + + file.delete(); + + return new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, + swiftPath, swiftPath, file.length(), file.length(), md5sum + ); + } catch (Exception e) { + s_logger.debug("Failed to register template into swift", e); + return new DownloadAnswer(e.toString(), VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } + } + private Answer execute(DownloadCommand cmd) { DataStoreTO dstore = cmd.getDataStore(); if (dstore instanceof NfsTO || dstore instanceof S3TO) { return _dlMgr.handleDownloadCommand(this, cmd); - } - /* - * else if (dstore instanceof S3TO) { // TODO: start download job to - * handle this // TODO: how to handle download progress for S3 S3TO s3 = - * (S3TO) cmd.getDataStore(); String url = cmd.getUrl(); String user = - * null; String password = null; if (cmd.getAuth() != null) { user = - * cmd.getAuth().getUserName(); password = new - * String(cmd.getAuth().getPassword()); } // get input stream from the - * given url InputStream in = UriUtils.getInputStreamFromUrl(url, user, - * password); URI uri; URL urlObj; try { uri = new URI(url); urlObj = - * new URL(url); } catch (URISyntaxException e) { throw new - * CloudRuntimeException("URI is incorrect: " + url); } catch - * (MalformedURLException e) { throw new - * CloudRuntimeException("URL is incorrect: " + url); } - * - * final String bucket = s3.getBucketName(); String path = null; if - * (cmd.getResourceType() == ResourceType.TEMPLATE) { // convention is - * no / in the end for install path based on // S3Utils implementation. - * // template key is // - * TEMPLATE_ROOT_DIR/account_id/template_id/template_name, by // adding - * template_name in the key, I can avoid generating a // - * template.properties file // for listTemplateCommand. path = - * determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId(), - * cmd.getName()); } else { path = - * determineS3VolumeDirectory(cmd.getAccountId(), cmd.getResourceId()); - * } - * - * String key = join(asList(path, urlObj.getFile()), S3Utils.SEPARATOR); - * S3Utils.putObject(s3, in, bucket, key); List s3Obj = - * S3Utils.getDirectory(s3, bucket, path); if (s3Obj == null || - * s3Obj.size() == 0) { return new Answer(cmd, false, - * "Failed to download to S3 bucket: " + bucket + " with key: " + key); - * } else { return new DownloadAnswer(null, 100, null, - * Status.DOWNLOADED, path, path, s3Obj.get(0).getSize(), - * s3Obj.get(0).getSize(), s3Obj .get(0).getETag()); } } - */ - else if (dstore instanceof SwiftTO) { + } else if (dstore instanceof SwiftTO) { return registerTemplateOnSwift(cmd); } else { return new Answer(cmd, false, "Unsupported image data store: " + dstore); diff --git a/utils/src/com/cloud/utils/SwiftUtil.java b/utils/src/com/cloud/utils/SwiftUtil.java new file mode 100644 index 00000000000..b95249b2c6c --- /dev/null +++ b/utils/src/com/cloud/utils/SwiftUtil.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.utils; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.log4j.Logger; + +import java.io.File; + + +public class SwiftUtil { + private static Logger logger = Logger.getLogger(SwiftUtil.class); + private static long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L; + public interface SwiftClientCfg { + String getAccount(); + String getUserName(); + String getKey(); + String getEndPoint(); + } + + private static String getSwiftCLIPath() { + String swiftCLI = Script.findScript("scripts/storage/secondary", "swift"); + if (swiftCLI == null) { + logger.debug("Can't find swift cli at scripts/storage/secondary/swift"); + throw new CloudRuntimeException("Can't find swift cli at scripts/storage/secondary/swift"); + } + return swiftCLI; + } + + public static String putObject(SwiftClientCfg cfg, File srcFile, String container) { + String swiftCli = getSwiftCLIPath(); + String srcDirectory = srcFile.getParent(); + Script command = new Script("/bin/bash", logger); + long size = srcFile.length(); + if (size <= SWIFT_MAX_SIZE) { + command.add("cd " + srcDirectory + + ";/usr/bin/python " + swiftCli + " -A " + + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + + cfg.getKey() + " upload " + container + " " + srcFile.getName()); + } else { + command.add("cd " + srcDirectory + + ";/usr/bin/python " + swiftCli + " -A " + + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + srcFile.getName()); + } + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = command.execute(parser); + if (result != null) { + throw new CloudRuntimeException("Failed to upload file: " + result); + } + + if (parser.getLines() != null) { + String[] lines = parser.getLines().split("\\n"); + for (String line : lines) { + if (line.contains("Errno") || line.contains("failed")) { + throw new CloudRuntimeException("Failed to upload file: " + lines.toString()); + } + } + } + return container + File.separator + srcFile.getName(); + } + + public static File getObject(SwiftClientCfg cfg, File destDirectory, String swiftPath) { + int firstIndexOfSeparator = swiftPath.indexOf(File.separator); + String container = swiftPath.substring(0, firstIndexOfSeparator); + String srcPath = swiftPath.substring(firstIndexOfSeparator + 1); + String destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath; + String swiftCli = getSwiftCLIPath(); + Script command = new Script("/bin/bash", logger); + command.add("-c"); + command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint() + + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " download " + + container + " " + srcPath + " -o " + destFilePath); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = command.execute(parser); + if (result != null) { + String errMsg = "swiftDownload failed err=" + result; + logger.debug(errMsg); + throw new CloudRuntimeException("failed to get object: " + swiftPath); + } + if (parser.getLines() != null) { + String[] lines = parser.getLines().split("\\n"); + for (String line : lines) { + if (line.contains("Errno") || line.contains("failed")) { + String errMsg = "swiftDownload failed , err=" + lines.toString(); + logger.debug(errMsg); + throw new CloudRuntimeException("Failed to get object: " + swiftPath); + } + } + } + return new File(destFilePath); + } +}