etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Various bug fixes for object store migration.

This commit is contained in:
Min Chen 2013-10-28 17:23:59 -07:00
parent 216d0343ed
commit afcf09edaa
7 changed files with 71 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
View File

@ -51,7 +51,6 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.MigrateVolumeAnswer;
@ -66,7 +65,6 @@ import com.cloud.host.Host;
import com.cloud.host.dao.HostDao;
import com.cloud.server.ManagementService;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VolumeVO;
@ -139,8 +137,9 @@ AncientDataMotionStrategy implements DataMotionStrategy {
DataTO destTO = destData.getTO();
DataStoreTO srcStoreTO = srcTO.getDataStore();
DataStoreTO destStoreTO = destTO.getDataStore();
if (srcStoreTO instanceof NfsTO || srcStoreTO.getRole() == DataStoreRole.ImageCache ||
(srcStoreTO instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO)srcStoreTO).getPoolType() == StoragePoolType.NetworkFilesystem)) {
if (srcStoreTO instanceof NfsTO || srcStoreTO.getRole() == DataStoreRole.ImageCache) {
//||
// (srcStoreTO instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO)srcStoreTO).getPoolType() == StoragePoolType.NetworkFilesystem)) {
return false;
}

14
engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
View File

@ -616,7 +616,12 @@ public class TemplateServiceImpl implements TemplateService {
private AsyncCallFuture<TemplateApiResult> syncToRegionStoreAsync(TemplateInfo template, DataStore store) {
AsyncCallFuture<TemplateApiResult> future = new AsyncCallFuture<TemplateApiResult>();
// no need to create entry on template_store_ref here, since entries are already created when prepareSecondaryStorageForMigration is invoked.
// But we need to set default install path so that sync can be done in the right s3 path
TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store);
String installPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/"
+ TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR
+ template.getAccountId() + "/" + template.getId() + "/" + template.getUniqueName();
((TemplateObject)templateOnStore).setInstallPath(installPath);
TemplateOpContext<TemplateApiResult> context = new TemplateOpContext<TemplateApiResult>(null,
(TemplateObject)templateOnStore, future);
AsyncCallbackDispatcher<TemplateServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
@ -649,11 +654,18 @@ public class TemplateServiceImpl implements TemplateService {
return null;
}
private boolean isRegionStore(DataStore store) {
if (store.getScope().getScopeType() == ScopeType.ZONE && store.getScope().getScopeId() == null)
return true;
else
return false;
}
// This routine is used to push templates currently on cache store, but not in region store to region store.
// used in migrating existing NFS secondary storage to S3.
@Override
public void syncTemplateToRegionStore(long templateId, DataStore store) {
if (store.getScope().getScopeType() == ScopeType.REGION) {
if (isRegionStore(store)) {
// if template is on region wide object store, check if it is really downloaded there (by checking install_path). Sync template to region
// wide store if it is not there physically.
TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store);

8
engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
View File

@ -58,6 +58,7 @@ public class TemplateObject implements TemplateInfo {
private VMTemplateVO imageVO;
private DataStore dataStore;
private String url;
private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case
@Inject
VMTemplateDao imageDao;
@Inject
@ -293,6 +294,9 @@ public class TemplateObject implements TemplateInfo {
@Override
public String getInstallPath() {
if (installPath != null)
return installPath;
if (dataStore == null) {
return null;
}
@ -300,6 +304,10 @@ public class TemplateObject implements TemplateInfo {
return obj.getInstallPath();
}
public void setInstallPath(String installPath) {
this.installPath = installPath;
}
@Override
public long getAccountId() {
return imageVO.getAccountId();

13
engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
View File

@ -258,8 +258,14 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
if (snapshots != null) {
s_logger.info("Duplicate " + snapshots.size() + " snapshot cache store records to region store");
for (SnapshotDataStoreVO snap : snapshots) {
SnapshotDataStoreVO snapStore = findByStoreSnapshot(DataStoreRole.Image, storeId, snap.getSnapshotId());
if (snapStore != null) {
s_logger.info("There is already entry for snapshot " + snap.getSnapshotId() + " on region store " + storeId);
continue;
}
s_logger.info("Persisting an entry for snapshot " + snap.getSnapshotId() + " on region store " + storeId);
SnapshotDataStoreVO ss = new SnapshotDataStoreVO();
ss.setSnapshotId(snap.getId());
ss.setSnapshotId(snap.getSnapshotId());
ss.setDataStoreId(storeId);
ss.setRole(DataStoreRole.Image);
ss.setVolumeId(snap.getVolumeId());
@ -267,8 +273,11 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
ss.setState(snap.getState());
ss.setSize(snap.getSize());
ss.setPhysicalSize(snap.getPhysicalSize());
ss.setRefCnt(snap.getRefCnt() + 1); // increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store
ss.setRefCnt(snap.getRefCnt());
persist(ss);
// increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store
snap.incrRefCnt();
update(snap.getId(), snap);
}
}

31
engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
View File

@ -40,6 +40,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
@ -380,10 +381,25 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
sc.setParameters("store_role", DataStoreRole.ImageCache);
sc.setParameters("destroyed", false);
List<TemplateDataStoreVO> tmpls = listBy(sc);
// create an entry for each record, but with empty install path since the content is not yet on region-wide store yet
// create an entry for each template record, but with empty install path since the content is not yet on region-wide store yet
if (tmpls != null) {
s_logger.info("Duplicate " + tmpls.size() + " template cache store records to region store");
for (TemplateDataStoreVO tmpl : tmpls) {
long templateId = tmpl.getTemplateId();
VMTemplateVO template = _tmpltDao.findById(templateId);
if (template == null) {
throw new CloudRuntimeException("No template is found for template id: " + templateId);
}
if (template.getTemplateType() == TemplateType.SYSTEM) {
s_logger.info("No need to duplicate system template since it will be automatically downloaded while adding region store");
continue;
}
TemplateDataStoreVO tmpStore = findByStoreTemplate(storeId, tmpl.getTemplateId());
if (tmpStore != null) {
s_logger.info("There is already entry for template " + tmpl.getTemplateId() + " on region store " + storeId);
continue;
}
s_logger.info("Persisting an entry for template " + tmpl.getTemplateId() + " on region store " + storeId);
TemplateDataStoreVO ts = new TemplateDataStoreVO();
ts.setTemplateId(tmpl.getTemplateId());
ts.setDataStoreId(storeId);
@ -395,17 +411,12 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
ts.setPhysicalSize(tmpl.getPhysicalSize());
ts.setErrorString(tmpl.getErrorString());
ts.setDownloadUrl(tmpl.getDownloadUrl());
ts.setRefCnt(tmpl.getRefCnt() + 1); // increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store
ts.setRefCnt(tmpl.getRefCnt());
persist(ts);
}
// increase ref_cnt of cache store entry so that this will not be recycled before the content is pushed to region-wide store
tmpl.incrRefCnt();
this.update(tmpl.getId(), tmpl);
// mark template as cross-zones and add template_zone association
for (TemplateDataStoreVO tmpl : tmpls) {
long templateId = tmpl.getTemplateId();
VMTemplateVO template = _tmpltDao.findById(templateId);
if (template == null) {
throw new CloudRuntimeException("No template is found for template id: " + templateId);
}
// mark the template as cross-zones
template.setCrossZones(true);
_tmpltDao.update(templateId, template);

13
engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
View File

@ -211,8 +211,14 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
if (vols != null) {
s_logger.info("Duplicate " + vols.size() + " volume cache store records to region store");
for (VolumeDataStoreVO vol : vols) {
VolumeDataStoreVO volStore = findByStoreVolume(storeId, vol.getVolumeId());
if (volStore != null) {
s_logger.info("There is already entry for volume " + vol.getVolumeId() + " on region store " + storeId);
continue;
}
s_logger.info("Persisting an entry for volume " + vol.getVolumeId() + " on region store " + storeId);
VolumeDataStoreVO vs = new VolumeDataStoreVO();
vs.setVolumeId(vol.getId());
vs.setVolumeId(vol.getVolumeId());
vs.setDataStoreId(storeId);
vs.setState(vol.getState());
vs.setDownloadPercent(vol.getDownloadPercent());
@ -220,8 +226,11 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
vs.setSize(vol.getSize());
vs.setPhysicalSize(vol.getPhysicalSize());
vs.setErrorString(vol.getErrorString());
vs.setRefCnt(vol.getRefCnt() + 1); // increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store
vs.setRefCnt(vol.getRefCnt());
persist(vs);
// increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store
vol.incrRefCnt();
this.update(vol.getId(), vol);
}
}

9
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -31,6 +31,7 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd;
import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd;
@ -51,7 +52,6 @@ import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@ -176,7 +176,6 @@ import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
@ -484,9 +483,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
// Handle NFS to S3 object store migration case, we trigger template sync from NFS to S3 during extract template or copy template
_tmpltSvr.syncTemplateToRegionStore(templateId, tmpltStore);
DataObject templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
return tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat(), templateObject);
return tmpltStore.createEntityExtractUrl(templateObject.getInstallPath(), template.getFormat(), templateObject);
}
public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId) {
@ -1292,7 +1291,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
Account owner = _accountMgr.getAccount(ownerId);
final Domain domain = _domainDao.findById(owner.getDomainId());
if ("add".equalsIgnoreCase(operation)) {
final List<String> accountNamesFinal = accountNames;
final List<String> accountNamesFinal = accountNames;
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {