mirror of https://github.com/apache/cloudstack.git
CLOUDSTACK-2584: fix create template from s3
This commit is contained in:
Edison Su
parent
235825dc33
commit
827a1a229b
|
|
@ -74,5 +74,4 @@ public class ImageStoreTO implements DataStoreTO {
|
|||
public DataStoreRole getRole() {
|
||||
return this.role;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,37 +137,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
return true;
|
||||
}
|
||||
|
||||
@DB
|
||||
protected Answer copyVolumeFromImage(DataObject srcData, DataObject destData) {
|
||||
String value = configDao.getValue(Config.CopyVolumeWait.key());
|
||||
int _copyvolumewait = NumbersUtil.parseInt(value,
|
||||
Integer.parseInt(Config.CopyVolumeWait.getDefaultValue()));
|
||||
|
||||
if (needCacheStorage(srcData, destData)) {
|
||||
//need to copy it to image cache store
|
||||
Scope destScope = destData.getDataStore().getScope();
|
||||
if (destScope instanceof ClusterScope){
|
||||
ClusterScope clusterScope = (ClusterScope)destScope;
|
||||
destScope = new ZoneScope(clusterScope.getZoneId());
|
||||
} else if (destScope instanceof HostScope){
|
||||
HostScope hostScope = (HostScope)destScope;
|
||||
destScope = new ZoneScope(hostScope.getZoneId());
|
||||
}
|
||||
DataObject cacheData = cacheMgr.createCacheObject(srcData, destScope);
|
||||
CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _copyvolumewait);
|
||||
EndPoint ep = selector.select(cacheData, destData);
|
||||
Answer answer = ep.sendMessage(cmd);
|
||||
return answer;
|
||||
} else {
|
||||
//handle copy it to/from cache store
|
||||
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _copyvolumewait);
|
||||
EndPoint ep = selector.select(srcData, destData);
|
||||
Answer answer = ep.sendMessage(cmd);
|
||||
return answer;
|
||||
}
|
||||
}
|
||||
|
||||
private Answer copyTemplate(DataObject srcData, DataObject destData) {
|
||||
protected Answer copyObject(DataObject srcData, DataObject destData) {
|
||||
String value = configDao.getValue(Config.PrimaryStorageDownloadWait.toString());
|
||||
int _primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue()));
|
||||
if (needCacheStorage(srcData, destData)) {
|
||||
|
|
@ -233,7 +203,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
CopyCommand cmd = new CopyCommand(srcData.getTO(), volObj.getTO(), _createVolumeFromSnapshotWait);
|
||||
EndPoint ep = selector.select(snapObj, volObj);
|
||||
Answer answer = ep.sendMessage(cmd);
|
||||
|
||||
|
||||
return answer;
|
||||
} catch (Exception e) {
|
||||
s_logger.error(basicErrMsg, e);
|
||||
|
|
@ -275,30 +245,24 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
Answer answer = null;
|
||||
String errMsg = null;
|
||||
try {
|
||||
if (destData.getType() == DataObjectType.VOLUME
|
||||
&& srcData.getType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Image) {
|
||||
answer = copyVolumeFromImage(srcData, destData);
|
||||
} else if (destData.getType() == DataObjectType.TEMPLATE
|
||||
&& srcData.getType() == DataObjectType.TEMPLATE) {
|
||||
answer = copyTemplate(srcData, destData);
|
||||
} else if (srcData.getType() == DataObjectType.SNAPSHOT
|
||||
|
||||
if (srcData.getType() == DataObjectType.SNAPSHOT
|
||||
&& destData.getType() == DataObjectType.VOLUME) {
|
||||
answer = copyVolumeFromSnapshot(srcData, destData);
|
||||
} else if (srcData.getType() == DataObjectType.SNAPSHOT
|
||||
&& destData.getType() == DataObjectType.TEMPLATE) {
|
||||
answer = createTemplateFromSnapshot(srcData, destData);
|
||||
} else if (srcData.getType() == DataObjectType.VOLUME
|
||||
&& destData.getType() == DataObjectType.TEMPLATE) {
|
||||
answer = createTemplateFromVolume(srcData, destData);
|
||||
} else if (srcData.getType() == DataObjectType.TEMPLATE
|
||||
&& destData.getType() == DataObjectType.VOLUME) {
|
||||
answer = cloneVolume(srcData, destData);
|
||||
} else if (destData.getType() == DataObjectType.VOLUME
|
||||
&& srcData.getType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary) {
|
||||
&& srcData.getType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary && destData.getDataStore().getRole() == DataStoreRole.Primary) {
|
||||
answer = copyVolumeBetweenPools(srcData, destData);
|
||||
} else if (srcData.getType() == DataObjectType.SNAPSHOT &&
|
||||
destData.getType() == DataObjectType.SNAPSHOT) {
|
||||
answer = copySnapshot(srcData, destData);
|
||||
} else {
|
||||
answer = copyObject(srcData, destData);
|
||||
}
|
||||
|
||||
if (answer != null && !answer.getResult()) {
|
||||
|
|
@ -337,58 +301,26 @@ public class AncientDataMotionStrategy implements DataMotionStrategy {
|
|||
return answer;
|
||||
}
|
||||
|
||||
|
||||
private Answer createTemplateFromVolume(DataObject srcData,
|
||||
DataObject destData) {
|
||||
|
||||
String value = configDao
|
||||
.getValue(Config.CreatePrivateTemplateFromVolumeWait.toString());
|
||||
int _createprivatetemplatefromvolumewait = NumbersUtil.parseInt(value,
|
||||
Integer.parseInt(Config.CreatePrivateTemplateFromVolumeWait
|
||||
.getDefaultValue()));
|
||||
|
||||
if (needCacheStorage(srcData, destData)) {
|
||||
//need to copy it to image cache store
|
||||
DataObject cacheData = cacheMgr.createCacheObject(srcData, destData.getDataStore().getScope());
|
||||
CopyCommand cmd = new CopyCommand(cacheData.getTO(), destData.getTO(), _createprivatetemplatefromvolumewait);
|
||||
EndPoint ep = selector.select(cacheData, destData);
|
||||
Answer answer = ep.sendMessage(cmd);
|
||||
return answer;
|
||||
} else {
|
||||
//handle copy it to/from cache store
|
||||
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _createprivatetemplatefromvolumewait);
|
||||
EndPoint ep = selector.select(srcData, destData);
|
||||
Answer answer = ep.sendMessage(cmd);
|
||||
return answer;
|
||||
}
|
||||
}
|
||||
|
||||
protected Answer copySnapshot(DataObject srcData, DataObject destData) {
|
||||
String value = configDao.getValue(Config.BackupSnapshotWait.toString());
|
||||
int _backupsnapshotwait = NumbersUtil.parseInt(value, Integer.parseInt(Config.BackupSnapshotWait.getDefaultValue()));
|
||||
|
||||
DataObject cacheData = null;
|
||||
Answer answer = null;
|
||||
try {
|
||||
if (needCacheStorage(srcData, destData)) {
|
||||
cacheData = cacheMgr.getCacheObject(srcData, destData.getDataStore().getScope());
|
||||
if (needCacheStorage(srcData, destData)) {
|
||||
cacheData = cacheMgr.getCacheObject(srcData, destData.getDataStore().getScope());
|
||||
|
||||
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait);
|
||||
cmd.setCacheTO(cacheData.getTO());
|
||||
EndPoint ep = selector.select(srcData, destData);
|
||||
answer = ep.sendMessage(cmd);
|
||||
} else {
|
||||
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait);
|
||||
EndPoint ep = selector.select(srcData, destData);
|
||||
answer = ep.sendMessage(cmd);
|
||||
}
|
||||
// clean up cache entry in case of failure
|
||||
if (answer == null || !answer.getResult()) {
|
||||
if (cacheData != null) {
|
||||
cacheMgr.deleteCacheObject(cacheData);
|
||||
}
|
||||
}
|
||||
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait);
|
||||
cmd.setCacheTO(cacheData.getTO());
|
||||
EndPoint ep = selector.select(srcData, destData);
|
||||
Answer answer = ep.sendMessage(cmd);
|
||||
return answer;
|
||||
} else {
|
||||
CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _backupsnapshotwait);
|
||||
EndPoint ep = selector.select(srcData, destData);
|
||||
Answer answer = ep.sendMessage(cmd);
|
||||
return answer;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
s_logger.debug("copy snasphot failed: " + e.toString());
|
||||
if (cacheData != null) {
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
package org.apache.cloudstack.storage.resource;
|
||||
|
||||
import static com.cloud.utils.S3Utils.getDirectory;
|
||||
import static com.cloud.utils.S3Utils.putFile;
|
||||
import static com.cloud.utils.StringUtils.join;
|
||||
import static com.cloud.utils.db.GlobalLock.executeWithNoWaitLock;
|
||||
import static java.lang.String.format;
|
||||
|
|
@ -27,6 +28,7 @@ import java.io.BufferedWriter;
|
|||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileWriter;
|
||||
import java.io.FilenameFilter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.math.BigInteger;
|
||||
|
|
@ -118,7 +120,9 @@ import com.cloud.storage.template.TemplateProp;
|
|||
import com.cloud.storage.template.VhdProcessor;
|
||||
import com.cloud.utils.NumbersUtil;
|
||||
import com.cloud.utils.S3Utils;
|
||||
import com.cloud.utils.S3Utils.ClientOptions;
|
||||
import com.cloud.utils.S3Utils.FileNamingStrategy;
|
||||
import com.cloud.utils.S3Utils.ObjectNamingStrategy;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
import com.cloud.utils.net.NetUtils;
|
||||
import com.cloud.utils.script.OutputInterpreter;
|
||||
|
|
@ -214,8 +218,6 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
return execute((DownloadTemplateFromSwiftToSecondaryStorageCommand) cmd);
|
||||
} else if (cmd instanceof UploadTemplateToSwiftFromSecondaryStorageCommand) {
|
||||
return execute((UploadTemplateToSwiftFromSecondaryStorageCommand) cmd);
|
||||
} else if (cmd instanceof UploadTemplateToS3FromSecondaryStorageCommand) {
|
||||
return execute((UploadTemplateToS3FromSecondaryStorageCommand) cmd);
|
||||
} else if (cmd instanceof CleanupSnapshotBackupCommand) {
|
||||
return execute((CleanupSnapshotBackupCommand) cmd);
|
||||
} else if (cmd instanceof CopyCommand) {
|
||||
|
|
@ -424,6 +426,17 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
}
|
||||
return new CopyCmdAnswer("");
|
||||
}
|
||||
|
||||
protected Answer copyFromNfsToImage(CopyCommand cmd) {
|
||||
DataTO destData = cmd.getDestTO();
|
||||
DataStoreTO destDataStore = destData.getDataStore();
|
||||
|
||||
if (destDataStore instanceof S3TO) {
|
||||
return copyFromNfsToS3(cmd);
|
||||
} else {
|
||||
return new CopyCmdAnswer("unsupported ");
|
||||
}
|
||||
}
|
||||
|
||||
protected Answer execute(CopyCommand cmd) {
|
||||
DataTO srcData = cmd.getSrcTO();
|
||||
|
|
@ -440,6 +453,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
NfsTO destImageStore = (NfsTO)destDataStore;
|
||||
return this.copyFromS3ToNfs(cmd, srcData, s3, destData, destImageStore);
|
||||
}
|
||||
|
||||
if (srcDataStore.getRole() == DataStoreRole.ImageCache && destDataStore.getRole() == DataStoreRole.Image) {
|
||||
return copyFromNfsToImage(cmd);
|
||||
}
|
||||
|
||||
|
||||
return Answer.createUnsupportedCommandAnswer(cmd);
|
||||
|
|
@ -608,86 +625,47 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
}
|
||||
}
|
||||
|
||||
private Answer execute(UploadTemplateToS3FromSecondaryStorageCommand cmd) {
|
||||
/*
|
||||
final S3TO s3 = cmd.getS3();
|
||||
final Long accountId = cmd.getAccountId();
|
||||
final Long templateId = cmd.getTemplateId();
|
||||
|
||||
try {
|
||||
final S3TO s3 = cmd.getS3();
|
||||
final Long accountId = cmd.getAccountId();
|
||||
final Long templateId = cmd.getTemplateId();
|
||||
|
||||
protected Answer copyFromNfsToS3(CopyCommand cmd) {
|
||||
final DataTO srcData = cmd.getSrcTO();
|
||||
final DataTO destData = cmd.getDestTO();
|
||||
DataStoreTO srcDataStore = srcData.getDataStore();
|
||||
NfsTO srcStore = (NfsTO)srcDataStore;
|
||||
DataStoreTO destDataStore = destData.getDataStore();
|
||||
|
||||
final S3TO s3 = (S3TO)destDataStore;
|
||||
|
||||
try {
|
||||
|
||||
final String templatePath = determineStorageTemplatePath(
|
||||
cmd.getStoragePath(), accountId, templateId);
|
||||
srcStore.getUrl(), srcData.getPath());
|
||||
|
||||
if (s_logger.isDebugEnabled()) {
|
||||
s_logger.debug("Found template id " + templateId
|
||||
+ " account id " + accountId + " from directory "
|
||||
s_logger.debug("Found " + srcData.getObjectType() + " from directory "
|
||||
+ templatePath + " to upload to S3.");
|
||||
}
|
||||
|
||||
if (!_storage.isDirectory(templatePath)) {
|
||||
final String errMsg = format("S3 Sync Failure: Directory %1$s"
|
||||
+ "for template id %2$s does not exist.", templatePath,
|
||||
templateId);
|
||||
s_logger.error(errMsg);
|
||||
return new Answer(cmd, false, errMsg);
|
||||
}
|
||||
|
||||
if (!_storage.isFile(templatePath + "/template.properties")) {
|
||||
final String errMsg = format("S3 Sync Failure: Template id "
|
||||
+ "%1$s does not exist on the file system.",
|
||||
templatePath);
|
||||
s_logger.error(errMsg);
|
||||
return new Answer(cmd, false, errMsg);
|
||||
}
|
||||
|
||||
if (s_logger.isDebugEnabled()) {
|
||||
s_logger.debug(format(
|
||||
"Pushing template id %1$s from %2$s to S3...",
|
||||
templateId, templatePath));
|
||||
}
|
||||
|
||||
final String bucket = s3.getBucketName();
|
||||
putDirectory(s3, bucket, _storage.getFile(templatePath),
|
||||
new FilenameFilter() {
|
||||
@Override
|
||||
public boolean accept(final File directory,
|
||||
final String fileName) {
|
||||
File fileToUpload = new File(directory.getAbsolutePath() + "/" + fileName);
|
||||
return !fileName.startsWith(".") && !fileToUpload.isDirectory();
|
||||
}
|
||||
}, new ObjectNamingStrategy() {
|
||||
@Override
|
||||
public String determineKey(final File file) {
|
||||
s_logger.debug(String
|
||||
.format("Determining key using account id %1$s and template id %2$s",
|
||||
accountId, templateId));
|
||||
return join(
|
||||
asList(determineS3TemplateDirectory(
|
||||
accountId, templateId), file
|
||||
.getName()), S3Utils.SEPARATOR);
|
||||
}
|
||||
});
|
||||
|
||||
return new Answer(
|
||||
cmd,
|
||||
true,
|
||||
format("Uploaded the contents of directory %1$s for template id %2$s to S3 bucket %3$s",
|
||||
templatePath, templateId, bucket));
|
||||
|
||||
final File srcFile = _storage.getFile(templatePath);
|
||||
String key = destData.getPath() + S3Utils.SEPARATOR + srcFile.getName();
|
||||
putFile(s3, srcFile, bucket, key);
|
||||
|
||||
DataTO retObj = null;
|
||||
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
|
||||
TemplateObjectTO newTemplate = new TemplateObjectTO();
|
||||
newTemplate.setPath(key);
|
||||
newTemplate.setSize(srcFile.length());
|
||||
retObj = newTemplate;
|
||||
} else if (destData.getObjectType() == DataObjectType.VOLUME) {
|
||||
VolumeObjectTO newVol = new VolumeObjectTO();
|
||||
newVol.setPath(key);
|
||||
newVol.setSize(srcFile.length());
|
||||
retObj = newVol;
|
||||
}
|
||||
|
||||
return new CopyCmdAnswer(retObj);
|
||||
} catch (Exception e) {
|
||||
|
||||
final String errMsg = format("Failed to upload template id %1$s",
|
||||
templateId);
|
||||
s_logger.error(errMsg, e);
|
||||
return new Answer(cmd, false, errMsg);
|
||||
*/
|
||||
return new Answer(cmd, false, "not supported ");
|
||||
s_logger.error("failed to upload" + srcData.getPath(), e);
|
||||
return new CopyCmdAnswer("failed to upload" + srcData.getPath() + e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue