diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java index d5255f40407..59e59a60662 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java @@ -30,4 +30,6 @@ public interface SnapshotDataFactory { SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role); List listSnapshotOnCache(long snapshotId); + + SnapshotInfo getReadySnapshotOnCache(long snapshotId); } diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java index e953eb6e21b..000b9ec4e60 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotService.java @@ -25,4 +25,6 @@ public interface SnapshotService { boolean deleteSnapshot(SnapshotInfo snapshot); boolean revertSnapshot(Long snapshotId); + + void syncVolumeSnapshotsToRegionStore(long volumeId, DataStore store); } diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index e7126a244f1..72aaf3d4a13 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -30,12 +30,15 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; + import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; @@ -52,7 +55,6 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.log4j.Logger; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; @@ -140,6 +142,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati SnapshotDataFactory snapshotFactory; @Inject ConfigDepot _configDepot; + @Inject + SnapshotService _snapshotSrv; private final StateMachine2 _volStateMachine; protected List _storagePoolAllocators; @@ -149,7 +153,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } public void setStoragePoolAllocators(List storagePoolAllocators) { - this._storagePoolAllocators = storagePoolAllocators; + _storagePoolAllocators = storagePoolAllocators; } protected List _podAllocators; @@ -159,7 +163,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } public void setPodAllocators(List podAllocators) { - this._podAllocators = podAllocators; + _podAllocators = podAllocators; } protected VolumeOrchestrator() { @@ -326,6 +330,16 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati VolumeInfo vol = volFactory.getVolume(volume.getId()); DataStore store = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); SnapshotInfo snapInfo = snapshotFactory.getSnapshot(snapshot.getId(), DataStoreRole.Image); + // sync snapshot to region store if necessary + DataStore snapStore = snapInfo.getDataStore(); + long snapVolId = snapInfo.getVolumeId(); + try { + _snapshotSrv.syncVolumeSnapshotsToRegionStore(snapVolId, snapStore); + } catch (Exception ex) { + // log but ignore the sync error to avoid any potential S3 down issue, it should be sync next time + s_logger.warn(ex.getMessage(), ex); + } + // create volume on primary from snapshot AsyncCallFuture future = volService.createVolumeFromSnapshot(vol, store, snapInfo); try { VolumeApiResult result = future.get(); diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 83c337d3f4a..e24c0351e46 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -49,6 +49,8 @@ public interface SnapshotDataStoreDao extends GenericDao listOnCache(long snapshotId); void updateStoreRoleToCache(long storeId); diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 1875636bf4f..2c829141a1e 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -689,6 +689,9 @@ public class TemplateServiceImpl implements TemplateService { @Override public void syncTemplateToRegionStore(long templateId, DataStore store) { if (_storeMgr.isRegionStore(store)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Sync template " + templateId + " from cache to object store..."); + } // if template is on region wide object store, check if it is really downloaded there (by checking install_path). Sync template to region // wide store if it is not there physically. TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store); diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java index 6205fe40deb..16c14f3c36d 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java @@ -79,6 +79,18 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { return so; } + @Override + public SnapshotInfo getReadySnapshotOnCache(long snapshotId) { + SnapshotDataStoreVO snapStore = snapshotStoreDao.findReadyOnCache(snapshotId); + if (snapStore != null) { + DataStore store = storeMgr.getDataStore(snapStore.getDataStoreId(), DataStoreRole.ImageCache); + return getSnapshot(snapshotId, store); + } else { + return null; + } + + } + @Override public List listSnapshotOnCache(long snapshotId) { List cacheSnapshots = snapshotStoreDao.listOnCache(snapshotId); diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index c05e0487b82..9cac20de95c 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -78,6 +78,7 @@ public class SnapshotObject implements SnapshotInfo { SnapshotDataStoreDao snapshotStoreDao; @Inject StorageStrategyFactory storageStrategyFactory; + private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case public SnapshotObject() { @@ -198,6 +199,9 @@ public class SnapshotObject implements SnapshotInfo { @Override public String getPath() { + if (installPath != null) + return installPath; + DataObjectInStore objectInStore = objectInStoreMgr.findObject(this, getDataStore()); if (objectInStore != null) { return objectInStore.getInstallPath(); @@ -205,6 +209,10 @@ public class SnapshotObject implements SnapshotInfo { return null; } + public void setPath(String installPath) { + this.installPath = installPath; + } + @Override public String getName() { return snapshot.getName(); @@ -354,12 +362,12 @@ public class SnapshotObject implements SnapshotInfo { @Override public void addPayload(Object data) { - this.payload = data; + payload = data; } @Override public Object getPayload() { - return this.payload; + return payload; } @Override diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index fb68958027f..d482e70b464 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.snapshot; +import java.util.List; import java.util.concurrent.ExecutionException; import javax.inject.Inject; @@ -37,6 +38,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; @@ -49,6 +51,9 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.template.TemplateConstants; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; @@ -56,13 +61,17 @@ import com.cloud.utils.fsm.NoTransitionException; public class SnapshotServiceImpl implements SnapshotService { private static final Logger s_logger = Logger.getLogger(SnapshotServiceImpl.class); @Inject + protected SnapshotDao _snapshotDao; + @Inject protected SnapshotDataStoreDao _snapshotStoreDao; @Inject - SnapshotDataFactory snapshotfactory; + SnapshotDataFactory _snapshotFactory; @Inject DataStoreManager dataStoreMgr; @Inject DataMotionService motionSrv; + @Inject + StorageCacheManager _cacheMgr; static private class CreateSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; @@ -244,7 +253,7 @@ public class SnapshotServiceImpl implements SnapshotService { try { snapObj.processEvent(Snapshot.Event.BackupToSecondary); - DataStore imageStore = this.findSnapshotImageStore(snapshot); + DataStore imageStore = findSnapshotImageStore(snapshot); if (imageStore == null) { throw new CloudRuntimeException("can not find an image stores"); } @@ -255,7 +264,7 @@ public class SnapshotServiceImpl implements SnapshotService { CopySnapshotContext context = new CopySnapshotContext(null, snapshot, snapshotOnImageStore, future); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().copySnapshotAsyncCallback(null, null)).setContext(context); - this.motionSrv.copyAsync(snapshot, snapshotOnImageStore, caller); + motionSrv.copyAsync(snapshot, snapshotOnImageStore, caller); } catch (Exception e) { s_logger.debug("Failed to copy snapshot", e); result.setResult("Failed to copy snapshot:" + e.toString()); @@ -306,7 +315,7 @@ public class SnapshotServiceImpl implements SnapshotService { CopyCmdAnswer answer = (CopyCmdAnswer)result.getAnswer(); destSnapshot.processEvent(Event.OperationSuccessed, result.getAnswer()); srcSnapshot.processEvent(Snapshot.Event.OperationSucceeded); - snapResult = new SnapshotResult(this.snapshotfactory.getSnapshot(destSnapshot.getId(), destSnapshot.getDataStore()), answer); + snapResult = new SnapshotResult(_snapshotFactory.getSnapshot(destSnapshot.getId(), destSnapshot.getDataStore()), answer); future.complete(snapResult); } catch (Exception e) { s_logger.debug("Failed to update snapshot state", e); @@ -391,7 +400,7 @@ public class SnapshotServiceImpl implements SnapshotService { @Override public boolean revertSnapshot(Long snapshotId) { - SnapshotInfo snapshot = snapshotfactory.getSnapshot(snapshotId, DataStoreRole.Primary); + SnapshotInfo snapshot = _snapshotFactory.getSnapshot(snapshotId, DataStoreRole.Primary); PrimaryDataStore store = (PrimaryDataStore)snapshot.getDataStore(); AsyncCallFuture future = new AsyncCallFuture(); @@ -417,4 +426,92 @@ public class SnapshotServiceImpl implements SnapshotService { return false; } + // This routine is used to push snapshots currently on cache store, but not in region store to region store. + // used in migrating existing NFS secondary storage to S3. We chose to push all volume related snapshots to handle delta snapshots smoothly. + @Override + public void syncVolumeSnapshotsToRegionStore(long volumeId, DataStore store) { + if (dataStoreMgr.isRegionStore(store)) { + // list all backed up snapshots for the given volume + List snapshots = _snapshotDao.listByStatus(volumeId, Snapshot.State.BackedUp); + if (snapshots != null) { + for (SnapshotVO snapshot : snapshots) { + syncSnapshotToRegionStore(snapshot.getId(), store); + } + } + } + } + + // push one individual snapshots currently on cache store to region store if it is not there already + private void syncSnapshotToRegionStore(long snapshotId, DataStore store) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("sync snapshot " + snapshotId + " from cache to object store..."); + } + // if snapshot is already on region wide object store, check if it is really downloaded there (by checking install_path). Sync snapshot to region + // wide store if it is not there physically. + SnapshotInfo snapOnStore = _snapshotFactory.getSnapshot(snapshotId, store); + if (snapOnStore == null) { + throw new CloudRuntimeException("Cannot find an entry in snapshot_store_ref for snapshot " + snapshotId + " on region store: " + store.getName()); + } + if (snapOnStore.getPath() == null || snapOnStore.getPath().length() == 0) { + // snapshot is not on region store yet, sync to region store + SnapshotInfo srcSnapshot = _snapshotFactory.getReadySnapshotOnCache(snapshotId); + if (srcSnapshot == null) { + throw new CloudRuntimeException("Cannot find snapshot " + snapshotId + " on cache store"); + } + AsyncCallFuture future = syncToRegionStoreAsync(srcSnapshot, store); + try { + SnapshotResult result = future.get(); + if (result.isFailed()) { + throw new CloudRuntimeException("sync snapshot from cache to region wide store failed for image store " + store.getName() + ":" + + result.getResult()); + } + _cacheMgr.releaseCacheObject(srcSnapshot); // reduce reference count for template on cache, so it can recycled by schedule + } catch (Exception ex) { + throw new CloudRuntimeException("sync snapshot from cache to region wide store failed for image store " + store.getName()); + } + } + + } + + private AsyncCallFuture syncToRegionStoreAsync(SnapshotInfo snapshot, DataStore store) { + AsyncCallFuture future = new AsyncCallFuture(); + // no need to create entry on snapshot_store_ref here, since entries are already created when updateCloudToUseObjectStore is invoked. + // But we need to set default install path so that sync can be done in the right s3 path + SnapshotInfo snapshotOnStore = _snapshotFactory.getSnapshot(snapshot, store); + String installPath = TemplateConstants.DEFAULT_SNAPSHOT_ROOT_DIR + "/" + + snapshot.getAccountId() + "/" + snapshot.getVolumeId(); + ((SnapshotObject)snapshotOnStore).setPath(installPath); + CopySnapshotContext context = new CopySnapshotContext(null, snapshot, + snapshotOnStore, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher + .create(this); + caller.setCallback(caller.getTarget().syncSnapshotCallBack(null, null)).setContext(context); + motionSrv.copyAsync(snapshot, snapshotOnStore, caller); + return future; + } + + protected Void syncSnapshotCallBack(AsyncCallbackDispatcher callback, + CopySnapshotContext context) { + CopyCommandResult result = callback.getResult(); + SnapshotInfo destSnapshot = context.destSnapshot; + SnapshotResult res = new SnapshotResult(destSnapshot, null); + + AsyncCallFuture future = context.future; + try { + if (result.isFailed()) { + res.setResult(result.getResult()); + // no change to existing snapshot_store_ref, will try to re-sync later if other call triggers this sync operation + } else { + // this will update install path properly, next time it will not sync anymore. + destSnapshot.processEvent(Event.OperationSuccessed, result.getAnswer()); + } + future.complete(res); + } catch (Exception e) { + s_logger.debug("Failed to process sync snapshot callback", e); + res.setResult(e.toString()); + future.complete(res); + } + + return null; + } } diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index b83fb4ec626..51c04bf2fc2 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -27,6 +27,9 @@ import java.util.List; import javax.inject.Inject; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -38,8 +41,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.LocalHostEndpoint; import org.apache.cloudstack.storage.RemoteHostEndPoint; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -211,7 +212,7 @@ public class DefaultEndPointSelector implements EndPointSelector { @Override public EndPoint select(DataObject srcData, DataObject destData, StorageAction action) { - if (action == StorageAction.BACKUPSNAPSHOT) { + if (action == StorageAction.BACKUPSNAPSHOT && srcData.getDataStore().getRole() == DataStoreRole.Primary) { SnapshotInfo srcSnapshot = (SnapshotInfo)srcData; if (srcSnapshot.getHypervisorType() == Hypervisor.HypervisorType.KVM) { VolumeInfo volumeInfo = srcSnapshot.getBaseVolume(); diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java index 519d3dbb019..28d65981065 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java @@ -103,6 +103,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase sc = storeSnapshotSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + sc.setParameters("store_role", DataStoreRole.ImageCache); + sc.setParameters("state", ObjectInDataStoreStateMachine.State.Ready); + return findOneIncludingRemovedBy(sc); + } + @Override public List listOnCache(long snapshotId) { SearchCriteria sc = storeSnapshotSearch.create();