etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SWIFT : sync swift with secondary storage

This commit is contained in:
anthony 2011-12-28 11:47:26 -08:00
parent ff93f7ec54
commit 1f7a811bac
7 changed files with 93 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api/src/com/cloud/agent/api/downloadTemplateFromSwiftToSecondaryStorageCommand.java
View File

@ -34,18 +34,20 @@ public class downloadTemplateFromSwiftToSecondaryStorageCommand extends Command
private Long dcId;
private Long accountId;
private Long templateId;
private String path;
protected downloadTemplateFromSwiftToSecondaryStorageCommand() {
}
public downloadTemplateFromSwiftToSecondaryStorageCommand(SwiftTO swift, String secondaryStorageUrl, Long dcId, Long accountId, Long templateId, int wait) {
public downloadTemplateFromSwiftToSecondaryStorageCommand(SwiftTO swift, String secondaryStorageUrl, Long dcId, Long accountId, Long templateId, String path, int wait) {
this.swift = swift;
this.secondaryStorageUrl = secondaryStorageUrl;
this.dcId = dcId;
this.accountId = accountId;
this.templateId = templateId;
this.path = path;
setWait(wait);
}
@ -73,6 +75,14 @@ public class downloadTemplateFromSwiftToSecondaryStorageCommand extends Command
return templateId;
}
public String getPath() {
return path;
}
public void setPath(String path) {
this.path = path;
}
@Override
public boolean executeInSequence() {
// TODO Auto-generated method stub

10
core/src/com/cloud/storage/VMTemplateSwiftVO.java
View File

@ -51,16 +51,20 @@ public class VMTemplateSwiftVO {
@Column(name = GenericDaoBase.CREATED_COLUMN)
private Date created = null;
@Column(name = "path")
private String path;
@Column(name = "size")
private long size;
@Column(name = "physical_size")
private long physicalSize;
public VMTemplateSwiftVO(long swiftId, long templateId, Date created, long size, long physicalSize) {
public VMTemplateSwiftVO(long swiftId, long templateId, Date created, String path, long size, long physicalSize) {
this.swiftId = swiftId;
this.templateId = templateId;
this.created = created;
this.path = path;
this.size = size;
this.physicalSize = physicalSize;
}
@ -81,6 +85,10 @@ public class VMTemplateSwiftVO {
return created;
}
public String getPath() {
return path;
}
public long getSwiftId() {
return swiftId;
}

31
core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java
View File

@ -174,18 +174,39 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
String secondaryStorageUrl = cmd.getSecondaryStorageUrl();
Long accountId = cmd.getAccountId();
Long templateId = cmd.getTemplateId();
String path = cmd.getPath();
String errMsg;
String lDir = null;
try {
String parent = getRootDir(secondaryStorageUrl);
String lPath = parent + "/template/tmpl/" + accountId.toString() + "/" + templateId.toString();
String result = swiftDownloadContainer(swift, "T-" + templateId.toString(), lPath);
lDir = parent + "/template/tmpl/" + accountId.toString() + "/" + templateId.toString();
String result = createLocalDir(lDir);
if (result != null) {
String errMsg = "failed to download template from Swift to secondary storage " + lPath + " , err=" + result;
errMsg = "downloadTemplateFromSwiftToSecondaryStorageCommand failed due to Create local directory failed";
s_logger.warn(errMsg);
return new Answer(cmd, false, errMsg);
throw new InternalErrorException(errMsg);
}
String lPath = lDir + "/" + path;
result = swiftDownload(swift, "T-" + templateId.toString(), path, lPath);
if (result != null) {
errMsg = "failed to download template " + path + " from Swift to secondary storage " + lPath + " , err=" + result;
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
path = "template.properties";
lPath = lDir + "/" + path;
result = swiftDownload(swift, "T-" + templateId.toString(), path, lPath);
if (result != null) {
errMsg = "failed to download template " + path + " from Swift to secondary storage " + lPath + " , err=" + result;
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
return new Answer(cmd, true, "success");
} catch (Exception e) {
String errMsg = cmd + " Command failed due to " + e.toString();
if (lDir != null) {
deleteLocalDir(lDir);
}
errMsg = cmd + " Command failed due to " + e.toString();
s_logger.warn(errMsg, e);
return new Answer(cmd, false, errMsg);
}

4
server/src/com/cloud/storage/allocator/AbstractStoragePoolAllocator.java
View File

@ -244,14 +244,14 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement
if (templateHostVO == null) {
VMTemplateSwiftVO templateSwiftVO = _swiftMgr.findByTmpltId(template.getId());
if (templateSwiftVO == null) {
s_logger.info("Did not find template downloaded on secondary hosts in zone " + plan.getDataCenterId());
s_logger.error("Did not find template downloaded on secondary hosts in zone " + plan.getDataCenterId());
return false;
}
long templateSize = templateSwiftVO.getPhysicalSize();
if (templateSize == 0) {
templateSize = templateSwiftVO.getSize();
}
totalAllocatedSize += (templateSize + _extraBytesPerVolume);
return false;
} else {
long templateSize = templateHostVO.getPhysicalSize();
if ( templateSize == 0 ){

43
server/src/com/cloud/storage/dao/VMTemplateHostDaoImpl.java
View File

@ -88,13 +88,14 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
TemplateSearch = createSearchBuilder();
TemplateSearch.and("template_id", TemplateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
TemplateSearch.and("destroyed", TemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
TemplateSearch.and("destroyed", TemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
TemplateSearch.done();
HostTemplateSearch = createSearchBuilder();
HostTemplateSearch.and("host_id", HostTemplateSearch.entity().getHostId(), SearchCriteria.Op.EQ);
HostTemplateSearch.and("template_id", HostTemplateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
HostTemplateSearch.done();
HostTemplateSearch.and("template_id", HostTemplateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
HostTemplateSearch.and("destroyed", HostTemplateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
HostTemplateSearch.done();
HostDestroyedSearch = createSearchBuilder();
HostDestroyedSearch.and("host_id", HostDestroyedSearch.entity().getHostId(), SearchCriteria.Op.EQ);
@ -103,18 +104,21 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
TemplateStatusSearch = createSearchBuilder();
TemplateStatusSearch.and("template_id", TemplateStatusSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
TemplateStatusSearch.and("download_state", TemplateStatusSearch.entity().getDownloadState(), SearchCriteria.Op.EQ);
TemplateStatusSearch.done();
TemplateStatusSearch.and("download_state", TemplateStatusSearch.entity().getDownloadState(), SearchCriteria.Op.EQ);
TemplateStatusSearch.and("destroyed", TemplateStatusSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
TemplateStatusSearch.done();
TemplateStatesSearch = createSearchBuilder();
TemplateStatesSearch.and("template_id", TemplateStatesSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
TemplateStatesSearch.and("states", TemplateStatesSearch.entity().getDownloadState(), SearchCriteria.Op.IN);
TemplateStatesSearch.and("destroyed", TemplateStatesSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
TemplateStatesSearch.done();
HostTemplateStateSearch = createSearchBuilder();
HostTemplateStateSearch.and("template_id", HostTemplateStateSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
HostTemplateStateSearch.and("host_id", HostTemplateStateSearch.entity().getHostId(), SearchCriteria.Op.EQ);
HostTemplateStateSearch.and("states", HostTemplateStateSearch.entity().getDownloadState(), SearchCriteria.Op.IN);
HostTemplateStateSearch.and("destroyed", HostTemplateStateSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
HostTemplateStateSearch.done();
}
@ -176,7 +180,7 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
public List<VMTemplateHostVO> listByTemplateId(long templateId) {
SearchCriteria<VMTemplateHostVO> sc = TemplateSearch.create();
sc.setParameters("template_id", templateId);
sc.setParameters("destroyed", false);
sc.setParameters("destroyed", false);
return listIncludingRemovedBy(sc);
}
@ -184,7 +188,8 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
@Override
public List<VMTemplateHostVO> listByOnlyTemplateId(long templateId) {
SearchCriteria<VMTemplateHostVO> sc = TemplateSearch.create();
sc.setParameters("template_id", templateId);
sc.setParameters("template_id", templateId);
sc.setParameters("destroyed", false);
return listIncludingRemovedBy(sc);
}
@ -193,16 +198,17 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
SearchCriteria<VMTemplateHostVO> sc = HostTemplateSearch.create();
sc.setParameters("host_id", hostId);
sc.setParameters("template_id", templateId);
sc.setParameters("download_state", Status.DOWNLOADED.toString());
return findOneIncludingRemovedBy(sc);
sc.setParameters("destroyed", false);
return findOneIncludingRemovedBy(sc);
}
@Override
public List<VMTemplateHostVO> listByTemplateStatus(long templateId, VMTemplateHostVO.Status downloadState) {
SearchCriteria<VMTemplateHostVO> sc = TemplateStatusSearch.create();
sc.setParameters("template_id", templateId);
sc.setParameters("download_state", downloadState.toString());
return listIncludingRemovedBy(sc);
sc.setParameters("download_state", downloadState.toString());
sc.setParameters("destroyed", false);
return listIncludingRemovedBy(sc);
}
@Override
@ -232,6 +238,7 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
sc.setParameters("template_id", templateId);
sc.setParameters("host_id", hostId);
sc.setParameters("states", (Object[])states);
sc.setParameters("destroyed", false);
return search(sc, null);
}
@ -285,15 +292,16 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
public List<VMTemplateHostVO> listByTemplateStates(long templateId, VMTemplateHostVO.Status... states) {
SearchCriteria<VMTemplateHostVO> sc = TemplateStatesSearch.create();
sc.setParameters("states", (Object[])states);
sc.setParameters("template_id", templateId);
return search(sc, null);
sc.setParameters("template_id", templateId);
sc.setParameters("destroyed", false);
return search(sc, null);
}
@Override
public List<VMTemplateHostVO> listByState(VMTemplateHostVO.Status state) {
SearchCriteria<VMTemplateHostVO> sc = createSearchCriteria();
sc.addAnd("downloadState", SearchCriteria.Op.EQ, state);
sc.addAnd("destroyed", SearchCriteria.Op.EQ, false);
return search(sc, null);
}
@ -301,8 +309,9 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
public List<VMTemplateHostVO> listByHostTemplate(long hostId, long templateId) {
SearchCriteria<VMTemplateHostVO> sc = HostTemplateSearch.create();
sc.setParameters("host_id", hostId);
sc.setParameters("template_id", templateId);
return listIncludingRemovedBy(sc);
sc.setParameters("template_id", templateId);
sc.setParameters("destroyed", false);
return listIncludingRemovedBy(sc);
}
@Override
@ -329,6 +338,7 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
SearchCriteria<VMTemplateHostVO> sc = HostTemplateSearch.create();
sc.setParameters("host_id", hostId);
sc.setParameters("template_id", templateId);
sc.setParameters("destroyed", false);
if (!lock)
return findOneIncludingRemovedBy(sc);
else
@ -346,6 +356,7 @@ public class VMTemplateHostDaoImpl extends GenericDaoBase<VMTemplateHostVO, Long
sc.setJoinParameters("host", "type", Host.Type.LocalSecondaryStorage);
sc.setParameters("template_id", templateId);
sc.setParameters("state", VMTemplateHostVO.Status.DOWNLOADED);
sc.setParameters("destroyed", false);
return findOneBy(sc);
}

45
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -365,31 +365,7 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe
if (tmpltHostRef == null && _swiftMgr.isSwiftEnabled()) {
SwiftTO swift = _swiftMgr.getSwiftTO(templateId);
if (swift != null && sservers != null) {
for (HostVO secondaryStorageHost : sservers) {
downloadTemplateFromSwiftToSecondaryStorageCommand cmd = new downloadTemplateFromSwiftToSecondaryStorageCommand(swift, secondaryStorageHost.getName(), zoneId,
template.getAccountId(), templateId, _primaryStorageDownloadWait);
try {
Answer answer = _agentMgr.sendToSSVM(zoneId, cmd);
if (answer == null || !answer.getResult()) {
String errMsg = "Failed to download template from Swift to secondary storage due to " + (answer == null ? "answer is null" : answer.getDetails());
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
tmpltHostRef = _tmpltHostDao.findByHostTemplate(secondaryStorageHost.getId(), templateId);
if (tmpltHostRef != null) {
if (tmpltHostRef.getDownloadState() != com.cloud.storage.VMTemplateStorageResourceAssoc.Status.DOWNLOADED) {
tmpltHostRef = null;
} else {
break;
}
}
} catch (Exception e) {
String errMsg = "Failed to download template from Swift to secondary storage due to " + e.toString();
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
}
downloadTemplateFromSwiftToSecondaryStorage(zoneId, templateId);
}
}
if (tmpltHostRef == null) {
@ -496,7 +472,7 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe
}
downloadTemplateFromSwiftToSecondaryStorageCommand cmd = new downloadTemplateFromSwiftToSecondaryStorageCommand(swift, secHost.getName(), dcId, template.getAccountId(), templateId,
_primaryStorageDownloadWait);
tmpltSwift.getPath(), _primaryStorageDownloadWait);
try {
Answer answer = _agentMgr.sendToSSVM(dcId, cmd);
if (answer == null || !answer.getResult()) {
@ -504,6 +480,11 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe
s_logger.warn(errMsg);
throw new CloudRuntimeException(errMsg);
}
String installPath = "template/tmpl/" + dcId + "/" + template.getAccountId() + "/" + tmpltSwift.getPath();
VMTemplateHostVO tmpltHost = new VMTemplateHostVO(secHost.getId(), templateId, new Date(), 100, Status.DOWNLOADED, null, null, null, installPath, template.getUrl());
tmpltHost.setSize(tmpltSwift.getSize());
tmpltHost.setPhysicalSize(tmpltSwift.getPhysicalSize());
_tmpltHostDao.persist(tmpltHost);
} catch (Exception e) {
String errMsg = "Failed to download template from Swift to secondary storage due to " + e.toString();
s_logger.warn(errMsg);
@ -552,7 +533,10 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe
}
return null;
}
VMTemplateSwiftVO tmpltSwift = new VMTemplateSwiftVO(swift.getId(), templateHostRef.getTemplateId(), new Date(), templateHostRef.getSize(), templateHostRef.getPhysicalSize());
String path = templateHostRef.getInstallPath();
int index = path.lastIndexOf('/');
path = path.substring(index + 1);
VMTemplateSwiftVO tmpltSwift = new VMTemplateSwiftVO(swift.getId(), templateHostRef.getTemplateId(), new Date(), path, templateHostRef.getSize(), templateHostRef.getPhysicalSize());
_tmpltSwiftDao.persist(tmpltSwift);
_swiftMgr.propagateTemplateOnAllZones(templateHostRef.getTemplateId());
} catch (Exception e) {
@ -591,12 +575,17 @@ public class TemplateManagerImpl implements TemplateManager, Manager, TemplateSe
templateHostRef = _storageMgr.findVmTemplateHost(templateId, pool);
if (templateHostRef == null) {
if (templateHostRef == null || templateHostRef.getDownloadState() != Status.DOWNLOADED) {
String result = downloadTemplateFromSwiftToSecondaryStorage(dcId, templateId);
if (result != null) {
s_logger.error("Unable to find a secondary storage host who has completely downloaded the template.");
return null;
}
templateHostRef = _storageMgr.findVmTemplateHost(templateId, pool);
if (templateHostRef == null || templateHostRef.getDownloadState() != Status.DOWNLOADED) {
s_logger.error("Unable to find a secondary storage host who has completely downloaded the template.");
return null;
}
}
HostVO sh = _hostDao.findById(templateHostRef.getHostId());

1
setup/db/create-schema.sql
View File

@ -1068,6 +1068,7 @@ CREATE TABLE `cloud`.`template_swift_ref` (
`swift_id` bigint unsigned NOT NULL,
`template_id` bigint unsigned NOT NULL,
`created` DATETIME NOT NULL,
`path` varchar(255),
`size` bigint unsigned,
`physical_size` bigint unsigned DEFAULT 0,
PRIMARY KEY (`id`),