etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-3400: add swift support

This commit is contained in:
Edison Su 2013-07-13 01:51:55 -07:00
parent 0c1ae20e77
commit 8f4575e877
14 changed files with 279 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
api/src/com/cloud/agent/api/to/SwiftTO.java
View File

@ -17,26 +17,25 @@
package com.cloud.agent.api.to;
import com.cloud.storage.DataStoreRole;
import com.cloud.utils.SwiftUtil;
public class SwiftTO implements DataStoreTO {
public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
Long id;
String url;
String account;
String userName;
String key;
String container;
public SwiftTO() { }
public SwiftTO(Long id, String url, String account, String userName, String key,
String container) {
public SwiftTO(Long id, String url, String account, String userName, String key
) {
this.id = id;
this.url = url;
this.account = account;
this.userName = userName;
this.key = key;
this.container = container;
}
public Long getId() {
@ -64,9 +63,8 @@ public class SwiftTO implements DataStoreTO {
return DataStoreRole.Image;
}
public String getContainer() {
return this.container;
@Override
public String getEndPoint() {
return this.url;
}
}

3
client/tomcatconf/applicationContext.xml.in
View File

@ -316,7 +316,6 @@
<bean id="storagePoolHostDaoImpl" class="com.cloud.storage.dao.StoragePoolHostDaoImpl" />
<bean id="storagePoolJoinDaoImpl" class="com.cloud.api.query.dao.StoragePoolJoinDaoImpl" />
<bean id="storagePoolWorkDaoImpl" class="com.cloud.storage.dao.StoragePoolWorkDaoImpl" />
<bean id="swiftDaoImpl" class="com.cloud.storage.dao.SwiftDaoImpl" />
<bean id="syncQueueDaoImpl" class="com.cloud.async.dao.SyncQueueDaoImpl" />
<bean id="syncQueueItemDaoImpl" class="com.cloud.async.dao.SyncQueueItemDaoImpl" />
<bean id="templatePrimaryDataStoreDaoImpl" class="org.apache.cloudstack.storage.volume.db.TemplatePrimaryDataStoreDaoImpl" />
@ -356,7 +355,6 @@
<bean id="vMTemplateHostDaoImpl" class="com.cloud.storage.dao.VMTemplateHostDaoImpl" />
<bean id="vMTemplatePoolDaoImpl" class="com.cloud.storage.dao.VMTemplatePoolDaoImpl" />
<bean id="vMTemplateS3DaoImpl" class="com.cloud.storage.dao.VMTemplateS3DaoImpl" />
<bean id="vMTemplateSwiftDaoImpl" class="com.cloud.storage.dao.VMTemplateSwiftDaoImpl" />
<bean id="vMTemplateZoneDaoImpl" class="com.cloud.storage.dao.VMTemplateZoneDaoImpl" />
<bean id="versionDaoImpl" class="com.cloud.upgrade.dao.VersionDaoImpl" />
<bean id="virtualRouterProviderDaoImpl" class="com.cloud.network.dao.VirtualRouterProviderDaoImpl" />
@ -793,7 +791,6 @@
<bean id="snapshotManagerImpl" class="com.cloud.storage.snapshot.SnapshotManagerImpl" />
<bean id="snapshotSchedulerImpl" class="com.cloud.storage.snapshot.SnapshotSchedulerImpl" />
<bean id="storageNetworkManagerImpl" class="com.cloud.network.StorageNetworkManagerImpl" />
<bean id="swiftManagerImpl" class="com.cloud.storage.swift.SwiftManagerImpl" />
<bean id="syncQueueManagerImpl" class="com.cloud.async.SyncQueueManagerImpl" />
<bean id="taggedResourceManagerImpl" class="com.cloud.tags.TaggedResourceManagerImpl" />
<bean id="resourceMetaDataManagerImpl" class="com.cloud.metadata.ResourceMetaDataManagerImpl" />

4
client/tomcatconf/commands.properties.in
View File

@ -241,10 +241,6 @@ deleteAlerts=1
#### system capacity commands
listCapacity=3
#### swift commands
addSwift=1
listSwifts=1
#### s3 commands
addS3=1
listS3s=1

9
core/src/org/apache/cloudstack/storage/command/DownloadCommand.java
View File

@ -45,6 +45,7 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
private ResourceType resourceType = ResourceType.TEMPLATE;
private String installPath;
private DataStoreTO _store;
private DataStoreTO cacheStore;
protected DownloadCommand() {
}
@ -178,7 +179,11 @@ public class DownloadCommand extends AbstractDownloadCommand implements Internal
this.installPath = installPath;
}
public void setCacheStore(DataStoreTO cacheStore) {
this.cacheStore = cacheStore;
}
public DataStoreTO getCacheStore() {
return this.cacheStore;
}
}

108
engine/schema/src/com/cloud/storage/VMTemplateSwiftVO.java
View File

@ -1,108 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.storage;
import java.util.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import com.cloud.utils.db.GenericDaoBase;
import org.apache.cloudstack.api.InternalIdentity;
/**
* Join table for swift and templates
*
*
*/
@Entity
@Table(name = "template_swift_ref")
public class VMTemplateSwiftVO implements InternalIdentity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
Long id;
@Column(name = "swift_id")
private long swiftId;
@Column(name = "template_id")
private long templateId;
@Column(name = GenericDaoBase.CREATED_COLUMN)
private Date created = null;
@Column(name = "path")
private String path;
@Column(name = "size")
private long size;
@Column(name = "physical_size")
private long physicalSize;
public VMTemplateSwiftVO(long swiftId, long templateId, Date created, String path, long size, long physicalSize) {
this.swiftId = swiftId;
this.templateId = templateId;
this.created = created;
this.path = path;
this.size = size;
this.physicalSize = physicalSize;
}
protected VMTemplateSwiftVO() {
}
public long getTemplateId() {
return templateId;
}
public long getId() {
return id;
}
public Date getCreated() {
return created;
}
public String getPath() {
return path;
}
public long getSwiftId() {
return swiftId;
}
public long getSize() {
return size;
}
public long getPhysicalSize() {
return physicalSize;
}
@Override
public String toString() {
return new StringBuilder("TmplSwift[").append(id).append("-").append(templateId).append("-").append(swiftId)
.append("]").toString();
}
}

37
engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDao.java
View File

@ -1,37 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.storage.dao;
import java.util.List;
import com.cloud.storage.VMTemplateSwiftVO;
import com.cloud.utils.db.GenericDao;
/**
*
*
*/
public interface VMTemplateSwiftDao extends GenericDao<VMTemplateSwiftVO, Long> {
List<VMTemplateSwiftVO> listBySwiftId(long id);
List<VMTemplateSwiftVO> listByTemplateId(long templateId);
VMTemplateSwiftVO findBySwiftTemplate(long swiftId, long templateId);
VMTemplateSwiftVO findOneByTemplateId(long templateId);
}

87
engine/schema/src/com/cloud/storage/dao/VMTemplateSwiftDaoImpl.java
View File

@ -1,87 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.storage.dao;
import java.util.Collections;
import java.util.List;
import javax.ejb.Local;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.storage.VMTemplateSwiftVO;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
/**
*
*
*/
@Component
@Local(value = { VMTemplateSwiftDao.class })
public class VMTemplateSwiftDaoImpl extends GenericDaoBase<VMTemplateSwiftVO, Long> implements VMTemplateSwiftDao {
public static final Logger s_logger = Logger.getLogger(VMTemplateSwiftDaoImpl.class.getName());
protected final SearchBuilder<VMTemplateSwiftVO> AllFieldSearch;
public VMTemplateSwiftDaoImpl() {
AllFieldSearch = createSearchBuilder();
AllFieldSearch.and("swift_id", AllFieldSearch.entity().getSwiftId(), SearchCriteria.Op.EQ);
AllFieldSearch.and("template_id", AllFieldSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
AllFieldSearch.done();
}
@Override
public List<VMTemplateSwiftVO> listBySwiftId(long id) {
SearchCriteria<VMTemplateSwiftVO> sc = AllFieldSearch.create();
sc.setParameters("swift_id", id);
return listBy(sc);
}
@Override
public List<VMTemplateSwiftVO> listByTemplateId(long templateId) {
SearchCriteria<VMTemplateSwiftVO> sc = AllFieldSearch.create();
sc.setParameters("template_id", templateId);
return listBy(sc);
}
@Override
public VMTemplateSwiftVO findOneByTemplateId(long templateId) {
SearchCriteria<VMTemplateSwiftVO> sc = AllFieldSearch.create();
sc.setParameters("template_id", templateId);
List<VMTemplateSwiftVO> list = listBy(sc);
if (list == null || list.size() < 1) {
return null;
} else {
Collections.shuffle(list);
return list.get(0);
}
}
@Override
public VMTemplateSwiftVO findBySwiftTemplate(long swiftId, long templateId) {
SearchCriteria<VMTemplateSwiftVO> sc = AllFieldSearch.create();
sc.setParameters("swift_id", swiftId);
sc.setParameters("template_id", templateId);
return findOneBy(sc);
}
}

6
engine/storage/integration-test/test/org/apache/cloudstack/storage/test/ChildTestConfiguration.java
View File

@ -79,7 +79,6 @@ import com.cloud.storage.download.DownloadMonitorImpl;
import com.cloud.storage.s3.S3Manager;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.storage.swift.SwiftManager;
import com.cloud.tags.dao.ResourceTagsDaoImpl;
import com.cloud.template.TemplateManager;
import com.cloud.user.AccountManager;
@ -177,11 +176,6 @@ public class ChildTestConfiguration extends TestConfiguration {
return Mockito.mock(VolumeManager.class);
}
@Bean
public SwiftManager switfMgr() {
return Mockito.mock(SwiftManager.class);
}
@Bean
public ManagementServer server() {
return Mockito.mock(ManagementServer.class);

6
plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
View File

@ -53,13 +53,15 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
ImageStoreDetailsDao _imageStoreDetailsDao;
@Inject
EndPointSelector _epSelector;
@Inject
StorageCacheManager cacheManager;
@Override
public DataStoreTO getStoreTO(DataStore store) {
ImageStoreImpl imgStore = (ImageStoreImpl) store;
Map<String, String> details = _imageStoreDetailsDao.getDetails(imgStore.getId());
return new SwiftTO(imgStore.getId(), imgStore.getUri(), details.get(ApiConstants.ACCOUNT),
details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY), details.get(ApiConstants.S3_BUCKET_NAME));
details.get(ApiConstants.USERNAME), details.get(ApiConstants.KEY));
}
@Override
@ -71,7 +73,9 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback<CreateCmdResult> callback) {
Long maxTemplateSizeInBytes = getMaxTemplateSizeInBytes();
VirtualMachineTemplate tmpl = _templateDao.findById(data.getId());
DataStore cacheStore = cacheManager.getCacheStorage(dataStore.getScope());
DownloadCommand dcmd = new DownloadCommand((TemplateObjectTO)(data.getTO()), maxTemplateSizeInBytes);
dcmd.setCacheStore(cacheStore.getTO());
dcmd.setProxy(getHttpProxy());
EndPoint ep = _epSelector.select(data);

9
server/src/com/cloud/api/ApiDBUtils.java
View File

@ -236,7 +236,6 @@ import com.cloud.storage.StorageStats;
import com.cloud.storage.UploadVO;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateS3VO;
import com.cloud.storage.VMTemplateSwiftVO;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.Volume;
import com.cloud.storage.Volume.Type;
@ -253,7 +252,6 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateDetailsDao;
import com.cloud.storage.dao.VMTemplateHostDao;
import com.cloud.storage.dao.VMTemplateS3Dao;
import com.cloud.storage.dao.VMTemplateSwiftDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeHostDao;
import com.cloud.storage.snapshot.SnapshotPolicy;
@ -336,7 +334,6 @@ public class ApiDBUtils {
static PrimaryDataStoreDao _storagePoolDao;
static VMTemplateDao _templateDao;
static VMTemplateDetailsDao _templateDetailsDao;
static VMTemplateSwiftDao _templateSwiftDao;
static VMTemplateS3Dao _templateS3Dao;
static UploadDao _uploadDao;
static UserDao _userDao;
@ -445,7 +442,6 @@ public class ApiDBUtils {
@Inject private PrimaryDataStoreDao storagePoolDao;
@Inject private VMTemplateDao templateDao;
@Inject private VMTemplateDetailsDao templateDetailsDao;
@Inject private VMTemplateSwiftDao templateSwiftDao;
@Inject private VMTemplateS3Dao templateS3Dao;
@Inject private UploadDao uploadDao;
@Inject private UserDao userDao;
@ -554,7 +550,6 @@ public class ApiDBUtils {
_storagePoolDao = storagePoolDao;
_templateDao = templateDao;
_templateDetailsDao = templateDetailsDao;
_templateSwiftDao = templateSwiftDao;
_templateS3Dao = templateS3Dao;
_uploadDao = uploadDao;
_userDao = userDao;
@ -894,10 +889,6 @@ public class ApiDBUtils {
return template;
}
public static VMTemplateSwiftVO findTemplateSwiftRef(long templateId) {
return _templateSwiftDao.findOneByTemplateId(templateId);
}
public static VMTemplateS3VO findTemplateS3Ref(long templateId) {
return _templateS3Dao.findOneByTemplateId(templateId);
}

3
server/src/com/cloud/storage/VolumeManagerImpl.java
View File

@ -139,7 +139,6 @@ import com.cloud.storage.dao.UploadDao;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VMTemplateS3Dao;
import com.cloud.storage.dao.VMTemplateSwiftDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeDetailsDao;
import com.cloud.storage.download.DownloadMonitor;
@ -237,8 +236,6 @@ public class VolumeManagerImpl extends ManagerBase implements VolumeManager {
@Inject
protected VMTemplatePoolDao _vmTemplatePoolDao = null;
@Inject
protected VMTemplateSwiftDao _vmTemplateSwiftDao = null;
@Inject
protected VMTemplateS3Dao _vmTemplateS3Dao;
@Inject
protected S3Manager _s3Mgr;

3
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -151,7 +151,6 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateDetailsDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VMTemplateS3Dao;
import com.cloud.storage.dao.VMTemplateSwiftDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.download.DownloadMonitor;
@ -232,8 +231,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Inject
SnapshotDao _snapshotDao;
@Inject
VMTemplateSwiftDao _tmpltSwiftDao;
@Inject
VMTemplateS3Dao _vmS3TemplateDao;
@Inject
ConfigurationDao _configDao;

270
services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -23,12 +23,7 @@ import static java.lang.String.format;
import static java.util.Arrays.asList;
import static org.apache.commons.lang.StringUtils.substringAfterLast;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.*;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.URI;
@ -43,6 +38,10 @@ import java.util.concurrent.Callable;
import javax.naming.ConfigurationException;
import com.cloud.agent.api.storage.*;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.template.*;
import com.cloud.utils.SwiftUtil;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.DeleteCommand;
@ -56,7 +55,13 @@ import org.apache.cloudstack.storage.template.UploadManagerImpl;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.log4j.Logger;
import com.amazonaws.services.s3.model.S3ObjectSummary;
@ -84,13 +89,6 @@ import com.cloud.agent.api.SecStorageVMSetupCommand;
import com.cloud.agent.api.StartupCommand;
import com.cloud.agent.api.StartupSecondaryStorageCommand;
import com.cloud.agent.api.UploadTemplateToSwiftFromSecondaryStorageCommand;
import com.cloud.agent.api.storage.CreateEntityDownloadURLCommand;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
import com.cloud.agent.api.storage.ListTemplateAnswer;
import com.cloud.agent.api.storage.ListTemplateCommand;
import com.cloud.agent.api.storage.ListVolumeAnswer;
import com.cloud.agent.api.storage.ListVolumeCommand;
import com.cloud.agent.api.storage.UploadCommand;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.DataTO;
@ -105,11 +103,7 @@ import com.cloud.resource.ServerResourceBase;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageLayer;
import com.cloud.storage.template.Processor;
import com.cloud.storage.template.Processor.FormatInfo;
import com.cloud.storage.template.TemplateLocation;
import com.cloud.storage.template.TemplateProp;
import com.cloud.storage.template.VhdProcessor;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.S3Utils;
import com.cloud.utils.S3Utils.FileNamingStrategy;
@ -219,6 +213,86 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
protected CopyCmdAnswer postProcessing(File destFile, String downloadPath, String destPath, DataTO srcData, DataTO destData) throws ConfigurationException {
// do post processing to unzip the file if it is compressed
String scriptsDir = "scripts/storage/secondary";
String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
if (createTmpltScr == null) {
throw new ConfigurationException("Unable to find createtmplt.sh");
}
s_logger.info("createtmplt.sh found in " + createTmpltScr);
String createVolScr = Script.findScript(scriptsDir, "createvolume.sh");
if (createVolScr == null) {
throw new ConfigurationException("Unable to find createvolume.sh");
}
s_logger.info("createvolume.sh found in " + createVolScr);
String script = srcData.getObjectType() == DataObjectType.TEMPLATE ? createTmpltScr : createVolScr;
int installTimeoutPerGig = 180 * 60 * 1000;
int imgSizeGigs = (int) Math.ceil(destFile.length() * 1.0d / (1024 * 1024 * 1024));
imgSizeGigs++; // add one just in case
long timeout = imgSizeGigs * installTimeoutPerGig;
String origPath = destFile.getAbsolutePath();
String extension = null;
if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension();
} else {
extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension();
}
String templateName = UUID.randomUUID().toString();
String templateFilename = templateName + "." + extension;
Script scr = new Script(script, timeout, s_logger);
scr.add("-s", Integer.toString(imgSizeGigs)); // not used for now
scr.add("-n", templateFilename);
scr.add("-t", downloadPath);
scr.add("-f", origPath); // this is the temporary
// template file downloaded
String result;
result = scr.execute();
if (result != null) {
// script execution failure
throw new CloudRuntimeException("Failed to run script " + script);
}
String finalFileName = templateFilename;
String finalDownloadPath = destPath + File.separator + templateFilename;
// compute the size of
long size = this._storage.getSize(downloadPath + File.separator + templateFilename);
DataTO newDestTO = null;
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
TemplateObjectTO newTemplTO = new TemplateObjectTO();
newTemplTO.setPath(finalDownloadPath);
newTemplTO.setName(finalFileName);
newTemplTO.setSize(size);
newDestTO = newTemplTO;
} else {
return new CopyCmdAnswer("not implemented yet");
}
return new CopyCmdAnswer(newDestTO);
}
protected Answer copyFromSwiftToNfs(CopyCommand cmd, DataTO srcData, SwiftTO swiftTO, DataTO destData, NfsTO destImageStore) {
final String storagePath = destImageStore.getUrl();
final String destPath = destData.getPath();
try {
String downloadPath = determineStorageTemplatePath(storagePath, destPath);
final File downloadDirectory = _storage.getFile(downloadPath);
downloadDirectory.mkdirs();
File destFile = SwiftUtil.getObject(swiftTO, downloadDirectory, srcData.getPath());
return postProcessing(destFile,downloadPath,destPath,srcData,destData);
} catch (Exception e) {
s_logger.debug("Failed to copy swift to nfs", e);
return new CopyCmdAnswer(e.toString());
}
}
protected Answer copyFromS3ToNfs(CopyCommand cmd, DataTO srcData, S3TO s3, DataTO destData, NfsTO destImageStore) {
final String storagePath = destImageStore.getUrl();
final String destPath = destData.getPath();
@ -249,68 +323,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return new CopyCmdAnswer("Can't find template");
}
// do post processing to unzip the file if it is compressed
String scriptsDir = "scripts/storage/secondary";
String createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
if (createTmpltScr == null) {
throw new ConfigurationException("Unable to find createtmplt.sh");
}
s_logger.info("createtmplt.sh found in " + createTmpltScr);
String createVolScr = Script.findScript(scriptsDir, "createvolume.sh");
if (createVolScr == null) {
throw new ConfigurationException("Unable to find createvolume.sh");
}
s_logger.info("createvolume.sh found in " + createVolScr);
String script = srcData.getObjectType() == DataObjectType.TEMPLATE ? createTmpltScr : createVolScr;
int installTimeoutPerGig = 180 * 60 * 1000;
int imgSizeGigs = (int) Math.ceil(destFile.length() * 1.0d / (1024 * 1024 * 1024));
imgSizeGigs++; // add one just in case
long timeout = imgSizeGigs * installTimeoutPerGig;
String origPath = destFile.getAbsolutePath();
String extension = null;
if (srcData.getObjectType() == DataObjectType.TEMPLATE) {
extension = ((TemplateObjectTO) srcData).getFormat().getFileExtension();
} else {
extension = ((VolumeObjectTO) srcData).getFormat().getFileExtension();
}
String templateName = UUID.randomUUID().toString();
String templateFilename = templateName + "." + extension;
Script scr = new Script(script, timeout, s_logger);
scr.add("-s", Integer.toString(imgSizeGigs)); // not used for now
scr.add("-n", templateFilename);
scr.add("-t", downloadPath);
scr.add("-f", origPath); // this is the temporary
// template file downloaded
String result;
result = scr.execute();
if (result != null) {
// script execution failure
throw new CloudRuntimeException("Failed to run script " + script);
}
String finalFileName = templateFilename;
String finalDownloadPath = destPath + File.separator + templateFilename;
// compute the size of
long size = this._storage.getSize(downloadPath + File.separator + templateFilename);
DataTO newDestTO = null;
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
TemplateObjectTO newTemplTO = new TemplateObjectTO();
newTemplTO.setPath(finalDownloadPath);
newTemplTO.setName(finalFileName);
newTemplTO.setSize(size);
newDestTO = newTemplTO;
} else {
return new CopyCmdAnswer("not implemented yet");
}
return new CopyCmdAnswer(newDestTO);
return postProcessing(destFile,downloadPath,destPath,srcData,destData);
} catch (Exception e) {
final String errMsg = format("Failed to download" + "due to $2%s", e.getMessage());
@ -432,11 +445,15 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return createTemplateFromSnapshot(cmd);
}
if (srcDataStore instanceof S3TO && destDataStore instanceof NfsTO
if (destDataStore instanceof NfsTO
&& destDataStore.getRole() == DataStoreRole.ImageCache) {
S3TO s3 = (S3TO) srcDataStore;
NfsTO destImageStore = (NfsTO) destDataStore;
return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore);
if (srcDataStore instanceof S3TO) {
S3TO s3 = (S3TO) srcDataStore;
return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore);
} else if (srcDataStore instanceof SwiftTO) {
return copyFromSwiftToNfs(cmd, srcData, (SwiftTO)srcDataStore, destData, destImageStore);
}
}
if (srcDataStore.getRole() == DataStoreRole.ImageCache && destDataStore.getRole() == DataStoreRole.Image) {
@ -519,51 +536,62 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
}
}
protected Answer registerTemplateOnSwift(DownloadCommand cmd) {
protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path) {
HttpClient client = new DefaultHttpClient();
HttpGet get = new HttpGet(url);
try {
HttpResponse response = client.execute(get);
HttpEntity entity = response.getEntity();
if (entity == null) {
s_logger.debug("Faled to get entity");
throw new CloudRuntimeException("Failed to get url: " + url);
}
return null;
String nfsMountPath = getRootDir(nfs.getUrl());
String filePath = nfsMountPath + File.separator + path;
FileOutputStream outputStream = new FileOutputStream(filePath);
entity.writeTo(outputStream);
return new File(filePath);
} catch (IOException e) {
s_logger.debug("Faild to get url:"+ url + ", due to " + e.toString());
throw new CloudRuntimeException(e);
}
}
protected Answer registerTemplateOnSwift(DownloadCommand cmd) {
SwiftTO swiftTO = (SwiftTO)cmd.getDataStore();
String path = cmd.getInstallPath();
DataStoreTO cacheStore = cmd.getCacheStore();
if (cacheStore == null || !(cacheStore instanceof NfsTO)) {
return new DownloadAnswer("cache store can't be null", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
}
try {
NfsTO nfsCacheStore = (NfsTO)cacheStore;
File file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path);
String swiftPath = SwiftUtil.putObject(swiftTO, file, "T-" + cmd.getId());
String md5sum = null;
try {
md5sum = DigestUtils.md5Hex(new FileInputStream(file));
} catch (IOException e) {
s_logger.debug("Failed to get md5sum: " + file.getAbsoluteFile());
}
file.delete();
return new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED,
swiftPath, swiftPath, file.length(), file.length(), md5sum
);
} catch (Exception e) {
s_logger.debug("Failed to register template into swift", e);
return new DownloadAnswer(e.toString(), VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR);
}
}
private Answer execute(DownloadCommand cmd) {
DataStoreTO dstore = cmd.getDataStore();
if (dstore instanceof NfsTO || dstore instanceof S3TO) {
return _dlMgr.handleDownloadCommand(this, cmd);
}
/*
* else if (dstore instanceof S3TO) { // TODO: start download job to
* handle this // TODO: how to handle download progress for S3 S3TO s3 =
* (S3TO) cmd.getDataStore(); String url = cmd.getUrl(); String user =
* null; String password = null; if (cmd.getAuth() != null) { user =
* cmd.getAuth().getUserName(); password = new
* String(cmd.getAuth().getPassword()); } // get input stream from the
* given url InputStream in = UriUtils.getInputStreamFromUrl(url, user,
* password); URI uri; URL urlObj; try { uri = new URI(url); urlObj =
* new URL(url); } catch (URISyntaxException e) { throw new
* CloudRuntimeException("URI is incorrect: " + url); } catch
* (MalformedURLException e) { throw new
* CloudRuntimeException("URL is incorrect: " + url); }
*
* final String bucket = s3.getBucketName(); String path = null; if
* (cmd.getResourceType() == ResourceType.TEMPLATE) { // convention is
* no / in the end for install path based on // S3Utils implementation.
* // template key is //
* TEMPLATE_ROOT_DIR/account_id/template_id/template_name, by // adding
* template_name in the key, I can avoid generating a //
* template.properties file // for listTemplateCommand. path =
* determineS3TemplateDirectory(cmd.getAccountId(), cmd.getResourceId(),
* cmd.getName()); } else { path =
* determineS3VolumeDirectory(cmd.getAccountId(), cmd.getResourceId());
* }
*
* String key = join(asList(path, urlObj.getFile()), S3Utils.SEPARATOR);
* S3Utils.putObject(s3, in, bucket, key); List<S3ObjectSummary> s3Obj =
* S3Utils.getDirectory(s3, bucket, path); if (s3Obj == null ||
* s3Obj.size() == 0) { return new Answer(cmd, false,
* "Failed to download to S3 bucket: " + bucket + " with key: " + key);
* } else { return new DownloadAnswer(null, 100, null,
* Status.DOWNLOADED, path, path, s3Obj.get(0).getSize(),
* s3Obj.get(0).getSize(), s3Obj .get(0).getETag()); } }
*/
else if (dstore instanceof SwiftTO) {
} else if (dstore instanceof SwiftTO) {
return registerTemplateOnSwift(cmd);
} else {
return new Answer(cmd, false, "Unsupported image data store: " + dstore);

111
utils/src/com/cloud/utils/SwiftUtil.java Normal file
View File

@ -0,0 +1,111 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.utils;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
import org.apache.log4j.Logger;
import java.io.File;
public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
private static long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
public interface SwiftClientCfg {
String getAccount();
String getUserName();
String getKey();
String getEndPoint();
}
private static String getSwiftCLIPath() {
String swiftCLI = Script.findScript("scripts/storage/secondary", "swift");
if (swiftCLI == null) {
logger.debug("Can't find swift cli at scripts/storage/secondary/swift");
throw new CloudRuntimeException("Can't find swift cli at scripts/storage/secondary/swift");
}
return swiftCLI;
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String container) {
String swiftCli = getSwiftCLIPath();
String srcDirectory = srcFile.getParent();
Script command = new Script("/bin/bash", logger);
long size = srcFile.length();
if (size <= SWIFT_MAX_SIZE) {
command.add("cd " + srcDirectory
+ ";/usr/bin/python " + swiftCli + " -A "
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
+ cfg.getKey() + " upload " + container + " " + srcFile.getName());
} else {
command.add("cd " + srcDirectory
+ ";/usr/bin/python " + swiftCli + " -A "
+ cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K "
+ cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " + container + " " + srcFile.getName());
}
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
throw new CloudRuntimeException("Failed to upload file: " + result);
}
if (parser.getLines() != null) {
String[] lines = parser.getLines().split("\\n");
for (String line : lines) {
if (line.contains("Errno") || line.contains("failed")) {
throw new CloudRuntimeException("Failed to upload file: " + lines.toString());
}
}
}
return container + File.separator + srcFile.getName();
}
public static File getObject(SwiftClientCfg cfg, File destDirectory, String swiftPath) {
int firstIndexOfSeparator = swiftPath.indexOf(File.separator);
String container = swiftPath.substring(0, firstIndexOfSeparator);
String srcPath = swiftPath.substring(firstIndexOfSeparator + 1);
String destFilePath = destDirectory.getAbsolutePath() + File.separator + srcPath;
String swiftCli = getSwiftCLIPath();
Script command = new Script("/bin/bash", logger);
command.add("-c");
command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint()
+ " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() + " download "
+ container + " " + srcPath + " -o " + destFilePath);
OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
String errMsg = "swiftDownload failed err=" + result;
logger.debug(errMsg);
throw new CloudRuntimeException("failed to get object: " + swiftPath);
}
if (parser.getLines() != null) {
String[] lines = parser.getLines().split("\\n");
for (String line : lines) {
if (line.contains("Errno") || line.contains("failed")) {
String errMsg = "swiftDownload failed , err=" + lines.toString();
logger.debug(errMsg);
throw new CloudRuntimeException("Failed to get object: " + swiftPath);
}
}
}
return new File(destFilePath);
}
}