Temporarily backup StorPool volume before expunge (#8843)

* Temporarily backup StorPool volume before expunge

Sometimes the users delete the volumes by mistake. This enhancment
provides a solution to backup the volume before it's deleted. The user
will be able to see the snapshot in CloudStack UI/CLI and create only a
volume from it.
A task will check (by default on every 5mins) if the snapshots are
deleted from StorPool

Global settings to enable the delay delete option:

`storpool.delete.after.interval` - The interval (in seconds) after the StorPool snapshot will be deleted
`storpool.list.snapshots.delete.after.interval` - The interval (in seconds) to fetch the StorPool snapshots with deleteAfter flag

Minor fix when deleting snapshots

* added Apache licence

* addressed comments
This commit is contained in:
slavkap 2024-06-26 11:28:04 +03:00 committed by GitHub
parent 121839277b
commit 6c06e85c80
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 393 additions and 31 deletions

View File

@ -18,9 +18,12 @@
*/
package com.cloud.storage.dao;
import java.util.List;
import org.apache.cloudstack.resourcedetail.ResourceDetailsDao;
import com.cloud.utils.db.GenericDao;
public interface SnapshotDetailsDao extends GenericDao<SnapshotDetailsVO, Long>, ResourceDetailsDao<SnapshotDetailsVO> {
public List<SnapshotDetailsVO> findDetailsByZoneAndKey(long dcId, String key);
}

View File

@ -18,11 +18,44 @@
*/
package com.cloud.storage.dao;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.exception.CloudRuntimeException;
public class SnapshotDetailsDaoImpl extends ResourceDetailsDaoBase<SnapshotDetailsVO> implements SnapshotDetailsDao {
private static final String GET_SNAPSHOT_DETAILS_ON_ZONE = "SELECT s.* FROM snapshot_details s LEFT JOIN snapshots ss ON ss.id=s.snapshot_id WHERE ss.data_center_id = ? AND s.name = ?";
@Override
public void addDetail(long resourceId, String key, String value, boolean display) {
super.addDetail(new SnapshotDetailsVO(resourceId, key, value, display));
}
public List<SnapshotDetailsVO> findDetailsByZoneAndKey(long dcId, String key) {
StringBuilder sql = new StringBuilder(GET_SNAPSHOT_DETAILS_ON_ZONE);
TransactionLegacy txn = TransactionLegacy.currentTxn();
List<SnapshotDetailsVO> snapshotDetailsOnZone = new ArrayList<SnapshotDetailsVO>();
try (PreparedStatement pstmt = txn.prepareStatement(sql.toString());) {
if (pstmt != null) {
pstmt.setLong(1, dcId);
pstmt.setString(2, key);
try (ResultSet rs = pstmt.executeQuery();) {
while (rs.next()) {
snapshotDetailsOnZone.add(toEntityBean(rs, false));
}
} catch (SQLException e) {
throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e);
}
}
return snapshotDetailsOnZone;
} catch (SQLException e) {
throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e);
}
}
}

View File

@ -504,7 +504,9 @@ public class VolumeServiceImpl implements VolumeService {
_snapshotStoreDao.remove(snapStoreVo.getId());
}
} else {
_snapshotStoreDao.remove(snapStoreVo.getId());
if (!StoragePoolType.StorPool.equals(storagePoolVO.getPoolType())) {
_snapshotStoreDao.remove(snapStoreVo.getId());
}
}
}
snapshotApiService.markVolumeSnapshotsAsDestroyed(vo);

View File

@ -0,0 +1,98 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.storage.datastore.api;
import java.io.Serializable;
import java.util.Map;
public class StorPoolSnapshotDef implements Serializable {
private static final long serialVersionUID = 1L;
private String name;
private Integer deleteAfter;
private Map<String, String> tags;
private Boolean bind;
private Integer iops;
private String rename;
private transient String volumeName;
public StorPoolSnapshotDef(String volumeName, Integer deleteAfter, Map<String, String> tags) {
super();
this.volumeName = volumeName;
this.deleteAfter = deleteAfter;
this.tags = tags;
}
public StorPoolSnapshotDef(String name, Integer deleteAfter, Map<String, String> tags, Boolean bind, Integer iops,
String rename) {
super();
this.name = name;
this.deleteAfter = deleteAfter;
this.tags = tags;
this.bind = bind;
this.iops = iops;
this.rename = rename;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getDeleteAfter() {
return deleteAfter;
}
public void setDeleteAfter(Integer deleteAfter) {
this.deleteAfter = deleteAfter;
}
public Map<String, String> getTags() {
return tags;
}
public void setTags(Map<String, String> tags) {
this.tags = tags;
}
public Boolean getBind() {
return bind;
}
public void setBind(Boolean bind) {
this.bind = bind;
}
public Integer getIops() {
return iops;
}
public void setIops(Integer iops) {
this.iops = iops;
}
public String getRename() {
return rename;
}
public void setRename(String rename) {
this.rename = rename;
}
public String getVolumeName() {
return volumeName;
}
public void setVolumeName(String volumeName) {
this.volumeName = volumeName;
}
}

View File

@ -18,6 +18,7 @@
*/
package org.apache.cloudstack.storage.datastore.driver;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -31,6 +32,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
@ -42,6 +44,7 @@ import org.apache.cloudstack.storage.command.CommandResult;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
import org.apache.cloudstack.storage.command.CreateObjectAnswer;
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
@ -87,6 +90,8 @@ import com.cloud.server.ResourceTag;
import com.cloud.server.ResourceTag.ResourceObjectType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ResizeVolumePayload;
import com.cloud.storage.Snapshot;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
@ -95,6 +100,7 @@ import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.SnapshotDetailsDao;
import com.cloud.storage.dao.SnapshotDetailsVO;
import com.cloud.storage.dao.StoragePoolHostDao;
@ -132,9 +138,11 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
@Inject
private HostDao hostDao;
@Inject
private ResourceTagDao _resourceTagDao;
private ResourceTagDao resourceTagDao;
@Inject
private SnapshotDetailsDao _snapshotDetailsDao;
private SnapshotDetailsDao snapshotDetailsDao;
@Inject
private SnapshotDao snapshotDao;
@Inject
private SnapshotDataStoreDao snapshotDataStoreDao;
@Inject
@ -401,7 +409,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
}
try {
SpConnectionDesc conn = StorPoolUtil.getSpConnection(dataStore.getUuid(), dataStore.getId(), storagePoolDetailsDao, primaryStoreDao);
tryToSnapshotVolumeBeforeDelete(vinfo, dataStore, name, conn);
SpApiResponse resp = StorPoolUtil.volumeDelete(name, conn);
if (resp.getError() == null) {
updateStoragePool(dataStore.getId(), - vinfo.getSize());
@ -431,6 +439,54 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
callback.complete(res);
}
private void tryToSnapshotVolumeBeforeDelete(VolumeInfo vinfo, DataStore dataStore, String name, SpConnectionDesc conn) {
Integer deleteAfter = StorPoolConfigurationManager.DeleteAfterInterval.valueIn(dataStore.getId());
if (deleteAfter != null && deleteAfter > 0 && vinfo.getPassphraseId() == null) {
createTemporarySnapshot(vinfo, name, deleteAfter, conn);
} else {
StorPoolUtil.spLog("The volume [%s] is not marked to be snapshot. Check the global setting `storpool.delete.after.interval` or the volume is encrypted [%s]", name, deleteAfter, vinfo.getPassphraseId() != null);
}
}
private void createTemporarySnapshot(VolumeInfo vinfo, String name, Integer deleteAfter, SpConnectionDesc conn) {
Map<String, String> tags = new HashMap<>();
tags.put("cs", StorPoolUtil.DELAY_DELETE);
StorPoolSnapshotDef snapshot = new StorPoolSnapshotDef(name, deleteAfter, tags);
StorPoolUtil.spLog("Creating backup snapshot before delete the volume [%s]", vinfo.getName());
SpApiResponse snapshotResponse = StorPoolUtil.volumeSnapshot(snapshot, conn);
if (snapshotResponse.getError() == null) {
String snapshotName = StorPoolUtil.getSnapshotNameFromResponse(snapshotResponse, false, StorPoolUtil.GLOBAL_ID);
String snapshotPath = StorPoolUtil.devPath(snapshotName);
SnapshotVO snapshotVo = createSnapshotVo(vinfo, snapshotName);
createSnapshotOnPrimaryVo(vinfo, snapshotVo, snapshotPath);
SnapshotDetailsVO snapshotDetails = new SnapshotDetailsVO(snapshotVo.getId(), StorPoolUtil.SP_DELAY_DELETE, "~" + snapshotName, true);
snapshotDetailsDao.persist(snapshotDetails);
}
}
private void createSnapshotOnPrimaryVo(VolumeInfo vinfo, SnapshotVO snapshotVo, String snapshotPath) {
SnapshotDataStoreVO snapshotOnPrimaryVo = new SnapshotDataStoreVO();
snapshotOnPrimaryVo.setSnapshotId(snapshotVo.getId());
snapshotOnPrimaryVo.setDataStoreId(vinfo.getDataCenterId());
snapshotOnPrimaryVo.setRole(vinfo.getDataStore().getRole());
snapshotOnPrimaryVo.setVolumeId(vinfo.getId());
snapshotOnPrimaryVo.setSize(vinfo.getSize());
snapshotOnPrimaryVo.setPhysicalSize(vinfo.getSize());
snapshotOnPrimaryVo.setInstallPath(snapshotPath);
snapshotOnPrimaryVo.setState(ObjectInDataStoreStateMachine.State.Ready);
snapshotDataStoreDao.persist(snapshotOnPrimaryVo);
}
private SnapshotVO createSnapshotVo(VolumeInfo vinfo, String snapshotName) {
SnapshotVO snapshotVo = new SnapshotVO(vinfo.getDataCenterId(), vinfo.getAccountId(), vinfo.getDomainId(), vinfo.getId(),
vinfo.getDiskOfferingId(), snapshotName,
(short)Snapshot.Type.RECURRING.ordinal(), Snapshot.Type.RECURRING.name(),
vinfo.getSize(), vinfo.getMinIops(), vinfo.getMaxIops(), vinfo.getHypervisorType(), Snapshot.LocationType.PRIMARY);
snapshotVo.setState(com.cloud.storage.Snapshot.State.BackedUp);
snapshotVo = snapshotDao.persist(snapshotVo);
return snapshotVo;
}
private void logDataObject(final String pref, DataObject data) {
final DataStore dstore = data.getDataStore();
String name = null;
@ -473,7 +529,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
try {
if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.VOLUME) {
SnapshotInfo sinfo = (SnapshotInfo)srcData;
final String snapshotName = StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), snapshotDataStoreDao, _snapshotDetailsDao);
final String snapshotName = StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), snapshotDataStoreDao, snapshotDetailsDao);
VolumeInfo vinfo = (VolumeInfo)dstData;
final String volumeName = vinfo.getUuid();
@ -491,9 +547,12 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
StorPoolUtil.spLog("Created volume=%s with uuid=%s from snapshot=%s with uuid=%s", StorPoolUtil.getNameFromResponse(resp, false), to.getUuid(), snapshotName, sinfo.getUuid());
} else if (resp.getError().getName().equals("objectDoesNotExist")) {
//check if snapshot is on secondary storage
StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snopshot on secondary storage", snapshotName);
StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snapshot on secondary storage", snapshotName);
SnapshotDataStoreVO snap = getSnapshotImageStoreRef(sinfo.getId(), vinfo.getDataCenterId());
if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) {
SnapshotDetailsVO snapshotDetail = snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
if (snapshotDetail != null) {
err = String.format("Could not create volume from snapshot due to: %s", resp.getError());
} else if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) {
resp = StorPoolUtil.volumeCreate(srcData.getUuid(), null, size, null, "no", "snapshot", sinfo.getBaseVolume().getMaxIops(), conn);
if (resp.getError() == null) {
VolumeObjectTO dstTO = (VolumeObjectTO) dstData.getTO();
@ -514,11 +573,11 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
err = String.format("Could not freeze Storpool volume %s. Error: %s", srcData.getUuid(), resp2.getError());
} else {
String name = StorPoolUtil.getNameFromResponse(resp, false);
SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid());
SnapshotDetailsVO snapshotDetails = snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid());
if (snapshotDetails != null) {
StorPoolHelper.updateSnapshotDetailsValue(snapshotDetails.getId(), StorPoolUtil.devPath(name), "snapshot");
}else {
StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), StorPoolUtil.devPath(name), _snapshotDetailsDao);
StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), StorPoolUtil.devPath(name), snapshotDetailsDao);
}
resp = StorPoolUtil.volumeCreate(volumeName, StorPoolUtil.getNameFromResponse(resp, true), size, null, null, "volume", sinfo.getBaseVolume().getMaxIops(), conn);
if (resp.getError() == null) {
@ -548,8 +607,10 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
err = String.format("Could not create Storpool volume %s from snapshot %s. Error: %s", volumeName, snapshotName, resp.getError());
}
} else if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.SNAPSHOT) {
SnapshotInfo sinfo = (SnapshotInfo)srcData;
SnapshotDetailsVO snapshotDetail = snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
// bypass secondary storage
if (StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
if (StorPoolConfigurationManager.BypassSecondaryStorage.value() || snapshotDetail != null) {
SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData.getTO();
answer = new CopyCmdAnswer(snapshot);
} else {
@ -986,9 +1047,9 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
SnapshotObjectTO snapTo = (SnapshotObjectTO)snapshot.getTO();
snapTo.setPath(StorPoolUtil.devPath(name.split("~")[1]));
answer = new CreateObjectAnswer(snapTo);
StorPoolHelper.addSnapshotDetails(snapshot.getId(), snapshot.getUuid(), snapTo.getPath(), _snapshotDetailsDao);
StorPoolHelper.addSnapshotDetails(snapshot.getId(), snapshot.getUuid(), snapTo.getPath(), snapshotDetailsDao);
//add primary storage of snapshot
StorPoolHelper.addSnapshotDetails(snapshot.getId(), StorPoolUtil.SP_STORAGE_POOL_ID, String.valueOf(snapshot.getDataStore().getId()), _snapshotDetailsDao);
StorPoolHelper.addSnapshotDetails(snapshot.getId(), StorPoolUtil.SP_STORAGE_POOL_ID, String.valueOf(snapshot.getDataStore().getId()), snapshotDetailsDao);
StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.takeSnapshot: snapshot: name=%s, uuid=%s, volume: name=%s, uuid=%s", name, snapshot.getUuid(), volumeName, vinfo.getUuid());
}
} catch (Exception e) {
@ -1003,7 +1064,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
@Override
public void revertSnapshot(final SnapshotInfo snapshot, final SnapshotInfo snapshotOnPrimaryStore, final AsyncCompletionCallback<CommandResult> callback) {
final VolumeInfo vinfo = snapshot.getBaseVolume();
final String snapshotName = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), snapshotDataStoreDao, _snapshotDetailsDao);
final String snapshotName = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), snapshotDataStoreDao, snapshotDetailsDao);
final String volumeName = StorPoolStorageAdaptor.getVolumeNameFromPath(vinfo.getPath(), true);
StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.revertSnapshot: snapshot: name=%s, uuid=%s, volume: name=%s, uuid=%s", snapshotName, snapshot.getUuid(), volumeName, vinfo.getUuid());
String err = null;
@ -1058,7 +1119,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
}
private String getVcPolicyTag(Long vmId) {
ResourceTag resourceTag = vmId != null ? _resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, StorPoolUtil.SP_VC_POLICY) : null;
ResourceTag resourceTag = vmId != null ? resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, StorPoolUtil.SP_VC_POLICY) : null;
return resourceTag != null ? resourceTag.getValue() : "";
}

View File

@ -28,19 +28,29 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.util.StorPoolUtil;
import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.apache.commons.lang3.StringUtils;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.SnapshotDetailsDao;
import com.cloud.storage.dao.SnapshotDetailsVO;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
import com.cloud.utils.exception.CloudRuntimeException;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
@ -55,6 +65,12 @@ public class StorPoolStatsCollector extends ManagerBase {
private StoragePoolDetailsDao storagePoolDetailsDao;
@Inject
private ConfigurationDao configurationDao;
@Inject
private SnapshotDao snapshotDao;
@Inject
private SnapshotDataStoreDao snapshotDataStoreDao;
@Inject
private SnapshotDetailsDao snapshotDetailsDao;
private ScheduledExecutorService executor;
@ -70,7 +86,7 @@ public class StorPoolStatsCollector extends ManagerBase {
public boolean start() {
List<StoragePoolVO> spPools = storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME);
if (CollectionUtils.isNotEmpty(spPools)) {
executor = Executors.newScheduledThreadPool(2,new NamedThreadFactory("StorPoolStatsCollector"));
executor = Executors.newScheduledThreadPool(3, new NamedThreadFactory("StorPoolStatsCollector"));
long storageStatsInterval = NumbersUtil.parseLong(configurationDao.getValue("storage.stats.interval"), 60000L);
long volumeStatsInterval = NumbersUtil.parseLong(configurationDao.getValue("volume.stats.interval"), 60000L);
@ -80,6 +96,13 @@ public class StorPoolStatsCollector extends ManagerBase {
if (StorPoolConfigurationManager.StorageStatsInterval.value() > 0 && storageStatsInterval > 0) {
executor.scheduleAtFixedRate(new StorPoolStorageStatsMonitorTask(), 120, StorPoolConfigurationManager.StorageStatsInterval.value(), TimeUnit.SECONDS);
}
for (StoragePoolVO pool: spPools) {
Integer deleteAfter = StorPoolConfigurationManager.DeleteAfterInterval.valueIn(pool.getId());
if (deleteAfter != null && deleteAfter > 0) {
executor.scheduleAtFixedRate(new StorPoolSnapshotsWithDelayDelete(), 120, StorPoolConfigurationManager.ListSnapshotsWithDeleteAfterInterval.value(), TimeUnit.SECONDS);
break;
}
}
}
return true;
@ -185,4 +208,90 @@ public class StorPoolStatsCollector extends ManagerBase {
}
}
}
class StorPoolSnapshotsWithDelayDelete implements Runnable {
@Override
public void run() {
List<StoragePoolVO> spPools = storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME);
if (CollectionUtils.isNotEmpty(spPools)) {
Map<Long, StoragePoolVO> onePoolForZone = new HashMap<>();
for (StoragePoolVO storagePoolVO : spPools) {
onePoolForZone.put(storagePoolVO.getDataCenterId(), storagePoolVO);
}
for (StoragePoolVO storagePool : onePoolForZone.values()) {
List<SnapshotDetailsVO> snapshotsDetails = snapshotDetailsDao.findDetailsByZoneAndKey(storagePool.getDataCenterId(), StorPoolUtil.SP_DELAY_DELETE);
if (CollectionUtils.isEmpty(snapshotsDetails)) {
return;
}
Map<String, String> snapshotsWithDelayDelete = new HashMap<>();
try {
log.debug(String.format("Collecting snapshots marked to be deleted for zone [%s]", storagePool.getDataCenterId()));
JsonArray arr = StorPoolUtil.snapshotsListAllClusters(StorPoolUtil.getSpConnection(storagePool.getUuid(),
storagePool.getId(), storagePoolDetailsDao, storagePoolDao));
snapshotsWithDelayDelete.putAll(getSnapshotsMarkedForDeletion(arr));
log.debug(String.format("Found snapshot details [%s] and snapshots on StorPool with delay delete flag [%s]", snapshotsDetails, snapshotsWithDelayDelete));
syncSnapshots(snapshotsDetails, snapshotsWithDelayDelete);
} catch (Exception e) {
log.debug("Could not fetch the snapshots with delay delete flag " + e.getMessage());
}
}
}
}
private void syncSnapshots(List<SnapshotDetailsVO> snapshotsDetails,
Map<String, String> snapshotsWithDelayDelete) {
for (SnapshotDetailsVO snapshotDetailsVO : snapshotsDetails) {
if (!snapshotsWithDelayDelete.containsKey(snapshotDetailsVO.getValue())) {
StorPoolUtil.spLog("The snapshot [%s] with delayDelete flag is no longer on StorPool. Removing it from CloudStack", snapshotDetailsVO.getValue());
SnapshotDataStoreVO ss = snapshotDataStoreDao
.findBySourceSnapshot(snapshotDetailsVO.getResourceId(), DataStoreRole.Primary);
if (ss != null) {
ss.setState(State.Destroyed);
snapshotDataStoreDao.update(ss.getId(), ss);
}
SnapshotVO snap = snapshotDao.findById(snapshotDetailsVO.getResourceId());
if (snap != null) {
snap.setState(com.cloud.storage.Snapshot.State.Destroyed);
snapshotDao.update(snap.getId(), snap);
}
snapshotDetailsDao.remove(snapshotDetailsVO.getId());
}
}
}
private Map<String, String> getSnapshotsMarkedForDeletion(JsonArray arr) {
for (JsonElement jsonElement : arr) {
JsonObject error = jsonElement.getAsJsonObject().getAsJsonObject("error");
if (error != null) {
throw new CloudRuntimeException(String.format("Could not collect the snapshots marked for deletion from all storage nodes due to: [%s]", error));
}
}
Map<String, String> snapshotsWithDelayDelete = new HashMap<>();
for (JsonElement jsonElement : arr) {
JsonObject response = jsonElement.getAsJsonObject().getAsJsonObject("response");
if (response == null) {
return snapshotsWithDelayDelete;
}
collectSnapshots(snapshotsWithDelayDelete, response);
}
log.debug("Found snapshots on StorPool" + snapshotsWithDelayDelete);
return snapshotsWithDelayDelete;
}
private void collectSnapshots(Map<String, String> snapshotsWithDelayDelete, JsonObject response) {
JsonArray snapshots = response.getAsJsonObject().getAsJsonArray("data");
for (JsonElement snapshot : snapshots) {
String name = snapshot.getAsJsonObject().get("name").getAsString();
JsonObject tags = snapshot.getAsJsonObject().get("tags").getAsJsonObject();
if (!StringUtils.startsWith(name, "*") && StringUtils.containsNone(name, "@") && tags != null && !tags.entrySet().isEmpty()) {
String tag = tags.getAsJsonPrimitive("cs").getAsString();
if (tag != null && tag.equals(StorPoolUtil.DELAY_DELETE)) {
snapshotsWithDelayDelete.put(name, tag);
}
}
}
}
}
}

View File

@ -28,6 +28,8 @@ import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonPrimitive;
import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
@ -124,6 +126,15 @@ public class StorPoolUtil {
public static final String SP_VOLUME_ON_CLUSTER = "SP_VOLUME_ON_CLUSTER";
private static final String DATA = "data";
private static final String CLUSTERS = "clusters";
public static final String SP_DELAY_DELETE = "SP_DELAY_DELETE";
public static final String DELAY_DELETE = "delayDelete";
public static enum StorpoolRights {
RO("ro"), RW("rw"), DETACH("detach");
@ -416,27 +427,31 @@ public class StorPoolUtil {
public static JsonArray snapshotsList(SpConnectionDesc conn) {
SpApiResponse resp = GET("MultiCluster/SnapshotsList", conn);
JsonObject obj = resp.fullJson.getAsJsonObject();
JsonArray data = obj.getAsJsonArray("data");
return data;
return obj.getAsJsonArray(DATA);
}
public static JsonArray snapshotsListAllClusters(SpConnectionDesc conn) {
SpApiResponse resp = GET("MultiCluster/AllClusters/SnapshotsList", conn);
JsonObject obj = resp.fullJson.getAsJsonObject();
return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
}
public static JsonArray volumesList(SpConnectionDesc conn) {
SpApiResponse resp = GET("MultiCluster/VolumesList", conn);
JsonObject obj = resp.fullJson.getAsJsonObject();
JsonArray data = obj.getAsJsonArray("data");
return data;
return obj.getAsJsonArray(DATA);
}
public static JsonArray volumesSpace(SpConnectionDesc conn) {
SpApiResponse resp = GET("MultiCluster/AllClusters/VolumesSpace", conn);
JsonObject obj = resp.fullJson.getAsJsonObject();
return obj.getAsJsonObject("data").getAsJsonArray("clusters");
return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
}
public static JsonArray templatesStats(SpConnectionDesc conn) {
SpApiResponse resp = GET("MultiCluster/AllClusters/VolumeTemplatesStatus", conn);
JsonObject obj = resp.fullJson.getAsJsonObject();
return obj.getAsJsonObject("data").getAsJsonArray("clusters");
return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
}
private static boolean objectExists(SpApiError err) {
@ -453,7 +468,7 @@ public class StorPoolUtil {
if (resp.getError() != null && !objectExists(resp.getError())) {
return null;
}
JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
return data.getAsJsonPrimitive("size").getAsLong();
}
@ -461,7 +476,7 @@ public class StorPoolUtil {
SpApiResponse resp = GET("MultiCluster/Snapshot/" + name, conn);
JsonObject obj = resp.fullJson.getAsJsonObject();
JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId");
return clusterId != null ? clusterId.getAsString() : null;
}
@ -470,7 +485,7 @@ public class StorPoolUtil {
SpApiResponse resp = GET("MultiCluster/Volume/" + name, conn);
JsonObject obj = resp.fullJson.getAsJsonObject();
JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject();
JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject();
JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId");
return clusterId != null ? clusterId.getAsString() : null;
}
@ -579,6 +594,10 @@ public class StorPoolUtil {
return POST("MultiCluster/VolumeSnapshot/" + volumeName, json, conn);
}
public static SpApiResponse volumeSnapshot(StorPoolSnapshotDef snapshot, SpConnectionDesc conn) {
return POST("MultiCluster/VolumeSnapshot/" + snapshot.getVolumeName(), snapshot, conn);
}
public static SpApiResponse volumesGroupSnapshot(final List<VolumeObjectTO> volumeTOs, final String vmUuid,
final String snapshotName, String csTag, SpConnectionDesc conn) {
Map<String, Object> json = new LinkedHashMap<>();
@ -638,7 +657,7 @@ public class StorPoolUtil {
public static String getSnapshotNameFromResponse(SpApiResponse resp, boolean tildeNeeded, String globalIdOrRemote) {
JsonObject obj = resp.fullJson.getAsJsonObject();
JsonPrimitive data = obj.getAsJsonObject("data").getAsJsonPrimitive(globalIdOrRemote);
JsonPrimitive data = obj.getAsJsonObject(DATA).getAsJsonPrimitive(globalIdOrRemote);
String name = data != null ? data.getAsString() : null;
name = name != null ? !tildeNeeded ? name : "~" + name : name;
return name;
@ -646,7 +665,7 @@ public class StorPoolUtil {
public static String getNameFromResponse(SpApiResponse resp, boolean tildeNeeded) {
JsonObject obj = resp.fullJson.getAsJsonObject();
JsonPrimitive data = obj.getAsJsonObject("data").getAsJsonPrimitive("name");
JsonPrimitive data = obj.getAsJsonObject(DATA).getAsJsonPrimitive("name");
String name = data != null ? data.getAsString() : null;
name = name != null ? name.startsWith("~") && !tildeNeeded ? name.split("~")[1] : name : name;
return name;

View File

@ -148,18 +148,21 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy {
public StrategyPriority canHandle(DataObject srcData, DataObject destData) {
DataObjectType srcType = srcData.getType();
DataObjectType dstType = destData.getType();
if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.TEMPLATE
&& StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.TEMPLATE) {
SnapshotInfo sinfo = (SnapshotInfo) srcData;
VolumeInfo volume = sinfo.getBaseVolume();
StoragePoolVO storagePool = _storagePool.findById(volume.getPoolId());
if (!storagePool.getStorageProviderName().equals(StorPoolUtil.SP_PROVIDER_NAME)) {
return StrategyPriority.CANT_HANDLE;
}
SnapshotDetailsVO snapshotDetail = _snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE);
if (snapshotDetail != null) {
throw new CloudRuntimeException("Cannot create a template from the last snapshot of deleted volume. You can only restore the volume.");
}
String snapshotName = StorPoolHelper.getSnapshotName(sinfo.getId(), sinfo.getUuid(), _snapshotStoreDao,
_snapshotDetailsDao);
StorPoolUtil.spLog("StorPoolDataMotionStrategy.canHandle snapshot name=%s", snapshotName);
if (snapshotName != null) {
if (snapshotName != null && StorPoolConfigurationManager.BypassSecondaryStorage.value()) {
return StrategyPriority.HIGHEST;
}
}

View File

@ -44,6 +44,16 @@ public class StorPoolConfigurationManager implements Configurable {
"The interval in seconds to get StorPool template statistics",
false);
public static final ConfigKey<Integer> DeleteAfterInterval = new ConfigKey<>("Advanced", Integer.class,
"storpool.delete.after.interval", "0",
"The interval (in seconds) after the StorPool snapshot will be deleted",
false, ConfigKey.Scope.StoragePool);
public static final ConfigKey<Integer> ListSnapshotsWithDeleteAfterInterval = new ConfigKey<>("Advanced", Integer.class,
"storpool.list.snapshots.delete.after.interval", "360",
"The interval (in seconds) to fetch the StorPool snapshots with deleteAfter flag",
false);
@Override
public String getConfigComponentName() {
return StorPoolConfigurationManager.class.getSimpleName();
@ -51,6 +61,6 @@ public class StorPoolConfigurationManager implements Configurable {
@Override
public ConfigKey<?>[] getConfigKeys() {
return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval };
return new ConfigKey<?>[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval, DeleteAfterInterval, ListSnapshotsWithDeleteAfterInterval };
}
}

View File

@ -116,6 +116,8 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
if (resp.getError() != null) {
final String err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError());
StorPoolUtil.spLog(err);
markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId, resp);
throw new CloudRuntimeException(err);
} else {
res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId);
StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed successfully=%s, snapshot uuid=%s, name=%s", res, snapshotVO.getUuid(), name);
@ -129,6 +131,16 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
return res;
}
private void markSnapshotAsDestroyedIfAlreadyRemoved(Long snapshotId, SpApiResponse resp) {
if (resp.getError().getName().equals("objectDoesNotExist")) {
SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findBySourceSnapshot(snapshotId, DataStoreRole.Primary);
if (snapshotOnPrimary != null) {
snapshotOnPrimary.setState(State.Destroyed);
_snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary);
}
}
}
@Override
public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) {
log.debug(String.format("StorpoolSnapshotStrategy.canHandle: snapshot=%s, uuid=%s, op=%s", snapshot.getName(), snapshot.getUuid(), op));
@ -166,7 +178,7 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
boolean resultIsSet = false;
try {
while (snapshot != null &&
(snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error)) {
(snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error || snapshot.getState() == Snapshot.State.BackedUp)) {
SnapshotInfo child = snapshot.getChild();
if (child != null) {
@ -330,9 +342,21 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy {
} else {
snapshotZoneDao.removeSnapshotFromZones(snapshotVO.getId());
}
if (CollectionUtils.isNotEmpty(retrieveSnapshotEntries(snapshotId, null))) {
return true;
}
updateSnapshotToDestroyed(snapshotVO);
return true;
}
private List<SnapshotInfo> retrieveSnapshotEntries(long snapshotId, Long zoneId) {
return snapshotDataFactory.getSnapshots(snapshotId, zoneId);
}
private void updateSnapshotToDestroyed(SnapshotVO snapshotVo) {
snapshotVo.setState(Snapshot.State.Destroyed);
_snapshotDao.update(snapshotVo.getId(), snapshotVo);
}
@Override
public SnapshotInfo takeSnapshot(SnapshotInfo snapshot) {