From 6c06e85c803a72b23149b09587823d05bce6b444 Mon Sep 17 00:00:00 2001 From: slavkap <51903378+slavkap@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:28:04 +0300 Subject: [PATCH] Temporarily backup StorPool volume before expunge (#8843) * Temporarily backup StorPool volume before expunge Sometimes the users delete the volumes by mistake. This enhancment provides a solution to backup the volume before it's deleted. The user will be able to see the snapshot in CloudStack UI/CLI and create only a volume from it. A task will check (by default on every 5mins) if the snapshots are deleted from StorPool Global settings to enable the delay delete option: `storpool.delete.after.interval` - The interval (in seconds) after the StorPool snapshot will be deleted `storpool.list.snapshots.delete.after.interval` - The interval (in seconds) to fetch the StorPool snapshots with deleteAfter flag Minor fix when deleting snapshots * added Apache licence * addressed comments --- .../cloud/storage/dao/SnapshotDetailsDao.java | 3 + .../storage/dao/SnapshotDetailsDaoImpl.java | 33 ++++++ .../storage/volume/VolumeServiceImpl.java | 4 +- .../datastore/api/StorPoolSnapshotDef.java | 98 ++++++++++++++++ .../StorPoolPrimaryDataStoreDriver.java | 87 ++++++++++++-- .../driver/StorPoolStatsCollector.java | 111 +++++++++++++++++- .../storage/datastore/util/StorPoolUtil.java | 41 +++++-- .../motion/StorPoolDataMotionStrategy.java | 9 +- .../StorPoolConfigurationManager.java | 12 +- .../snapshot/StorPoolSnapshotStrategy.java | 26 +++- 10 files changed, 393 insertions(+), 31 deletions(-) create mode 100644 plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java index 43bb5b3d4d5..02a0355d92d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDao.java @@ -18,9 +18,12 @@ */ package com.cloud.storage.dao; +import java.util.List; + import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; import com.cloud.utils.db.GenericDao; public interface SnapshotDetailsDao extends GenericDao, ResourceDetailsDao { + public List findDetailsByZoneAndKey(long dcId, String key); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java index e4ae22cd021..584a2481726 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDetailsDaoImpl.java @@ -18,11 +18,44 @@ */ package com.cloud.storage.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; + public class SnapshotDetailsDaoImpl extends ResourceDetailsDaoBase implements SnapshotDetailsDao { + private static final String GET_SNAPSHOT_DETAILS_ON_ZONE = "SELECT s.* FROM snapshot_details s LEFT JOIN snapshots ss ON ss.id=s.snapshot_id WHERE ss.data_center_id = ? AND s.name = ?"; + @Override public void addDetail(long resourceId, String key, String value, boolean display) { super.addDetail(new SnapshotDetailsVO(resourceId, key, value, display)); } + + public List findDetailsByZoneAndKey(long dcId, String key) { + StringBuilder sql = new StringBuilder(GET_SNAPSHOT_DETAILS_ON_ZONE); + TransactionLegacy txn = TransactionLegacy.currentTxn(); + List snapshotDetailsOnZone = new ArrayList(); + try (PreparedStatement pstmt = txn.prepareStatement(sql.toString());) { + if (pstmt != null) { + pstmt.setLong(1, dcId); + pstmt.setString(2, key); + try (ResultSet rs = pstmt.executeQuery();) { + while (rs.next()) { + snapshotDetailsOnZone.add(toEntityBean(rs, false)); + } + } catch (SQLException e) { + throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e); + } + } + return snapshotDetailsOnZone; + } catch (SQLException e) { + throw new CloudRuntimeException("Could not find details by given zone and key due to:" + e.getMessage(), e); + } + } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index a47cb41a323..63726b40093 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -504,7 +504,9 @@ public class VolumeServiceImpl implements VolumeService { _snapshotStoreDao.remove(snapStoreVo.getId()); } } else { - _snapshotStoreDao.remove(snapStoreVo.getId()); + if (!StoragePoolType.StorPool.equals(storagePoolVO.getPoolType())) { + _snapshotStoreDao.remove(snapStoreVo.getId()); + } } } snapshotApiService.markVolumeSnapshotsAsDestroyed(vo); diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java new file mode 100644 index 00000000000..26004205709 --- /dev/null +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/api/StorPoolSnapshotDef.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.datastore.api; + +import java.io.Serializable; +import java.util.Map; + +public class StorPoolSnapshotDef implements Serializable { + private static final long serialVersionUID = 1L; + private String name; + private Integer deleteAfter; + private Map tags; + private Boolean bind; + private Integer iops; + private String rename; + private transient String volumeName; + + public StorPoolSnapshotDef(String volumeName, Integer deleteAfter, Map tags) { + super(); + this.volumeName = volumeName; + this.deleteAfter = deleteAfter; + this.tags = tags; + } + + public StorPoolSnapshotDef(String name, Integer deleteAfter, Map tags, Boolean bind, Integer iops, + String rename) { + super(); + this.name = name; + this.deleteAfter = deleteAfter; + this.tags = tags; + this.bind = bind; + this.iops = iops; + this.rename = rename; + } + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + public Integer getDeleteAfter() { + return deleteAfter; + } + public void setDeleteAfter(Integer deleteAfter) { + this.deleteAfter = deleteAfter; + } + public Map getTags() { + return tags; + } + public void setTags(Map tags) { + this.tags = tags; + } + public Boolean getBind() { + return bind; + } + public void setBind(Boolean bind) { + this.bind = bind; + } + public Integer getIops() { + return iops; + } + public void setIops(Integer iops) { + this.iops = iops; + } + public String getRename() { + return rename; + } + public void setRename(String rename) { + this.rename = rename; + } + + public String getVolumeName() { + return volumeName; + } + + public void setVolumeName(String volumeName) { + this.volumeName = volumeName; + } + +} diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index 08a3252d869..f5fdc96cb48 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.storage.datastore.driver; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -31,6 +32,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; @@ -42,6 +44,7 @@ import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; @@ -87,6 +90,8 @@ import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ResizeVolumePayload; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; @@ -95,6 +100,7 @@ import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotDetailsDao; import com.cloud.storage.dao.SnapshotDetailsVO; import com.cloud.storage.dao.StoragePoolHostDao; @@ -132,9 +138,11 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { @Inject private HostDao hostDao; @Inject - private ResourceTagDao _resourceTagDao; + private ResourceTagDao resourceTagDao; @Inject - private SnapshotDetailsDao _snapshotDetailsDao; + private SnapshotDetailsDao snapshotDetailsDao; + @Inject + private SnapshotDao snapshotDao; @Inject private SnapshotDataStoreDao snapshotDataStoreDao; @Inject @@ -401,7 +409,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { } try { SpConnectionDesc conn = StorPoolUtil.getSpConnection(dataStore.getUuid(), dataStore.getId(), storagePoolDetailsDao, primaryStoreDao); - + tryToSnapshotVolumeBeforeDelete(vinfo, dataStore, name, conn); SpApiResponse resp = StorPoolUtil.volumeDelete(name, conn); if (resp.getError() == null) { updateStoragePool(dataStore.getId(), - vinfo.getSize()); @@ -431,6 +439,54 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { callback.complete(res); } + private void tryToSnapshotVolumeBeforeDelete(VolumeInfo vinfo, DataStore dataStore, String name, SpConnectionDesc conn) { + Integer deleteAfter = StorPoolConfigurationManager.DeleteAfterInterval.valueIn(dataStore.getId()); + if (deleteAfter != null && deleteAfter > 0 && vinfo.getPassphraseId() == null) { + createTemporarySnapshot(vinfo, name, deleteAfter, conn); + } else { + StorPoolUtil.spLog("The volume [%s] is not marked to be snapshot. Check the global setting `storpool.delete.after.interval` or the volume is encrypted [%s]", name, deleteAfter, vinfo.getPassphraseId() != null); + } + } + + private void createTemporarySnapshot(VolumeInfo vinfo, String name, Integer deleteAfter, SpConnectionDesc conn) { + Map tags = new HashMap<>(); + tags.put("cs", StorPoolUtil.DELAY_DELETE); + StorPoolSnapshotDef snapshot = new StorPoolSnapshotDef(name, deleteAfter, tags); + StorPoolUtil.spLog("Creating backup snapshot before delete the volume [%s]", vinfo.getName()); + SpApiResponse snapshotResponse = StorPoolUtil.volumeSnapshot(snapshot, conn); + if (snapshotResponse.getError() == null) { + String snapshotName = StorPoolUtil.getSnapshotNameFromResponse(snapshotResponse, false, StorPoolUtil.GLOBAL_ID); + String snapshotPath = StorPoolUtil.devPath(snapshotName); + SnapshotVO snapshotVo = createSnapshotVo(vinfo, snapshotName); + createSnapshotOnPrimaryVo(vinfo, snapshotVo, snapshotPath); + SnapshotDetailsVO snapshotDetails = new SnapshotDetailsVO(snapshotVo.getId(), StorPoolUtil.SP_DELAY_DELETE, "~" + snapshotName, true); + snapshotDetailsDao.persist(snapshotDetails); + } + } + + private void createSnapshotOnPrimaryVo(VolumeInfo vinfo, SnapshotVO snapshotVo, String snapshotPath) { + SnapshotDataStoreVO snapshotOnPrimaryVo = new SnapshotDataStoreVO(); + snapshotOnPrimaryVo.setSnapshotId(snapshotVo.getId()); + snapshotOnPrimaryVo.setDataStoreId(vinfo.getDataCenterId()); + snapshotOnPrimaryVo.setRole(vinfo.getDataStore().getRole()); + snapshotOnPrimaryVo.setVolumeId(vinfo.getId()); + snapshotOnPrimaryVo.setSize(vinfo.getSize()); + snapshotOnPrimaryVo.setPhysicalSize(vinfo.getSize()); + snapshotOnPrimaryVo.setInstallPath(snapshotPath); + snapshotOnPrimaryVo.setState(ObjectInDataStoreStateMachine.State.Ready); + snapshotDataStoreDao.persist(snapshotOnPrimaryVo); + } + + private SnapshotVO createSnapshotVo(VolumeInfo vinfo, String snapshotName) { + SnapshotVO snapshotVo = new SnapshotVO(vinfo.getDataCenterId(), vinfo.getAccountId(), vinfo.getDomainId(), vinfo.getId(), + vinfo.getDiskOfferingId(), snapshotName, + (short)Snapshot.Type.RECURRING.ordinal(), Snapshot.Type.RECURRING.name(), + vinfo.getSize(), vinfo.getMinIops(), vinfo.getMaxIops(), vinfo.getHypervisorType(), Snapshot.LocationType.PRIMARY); + snapshotVo.setState(com.cloud.storage.Snapshot.State.BackedUp); + snapshotVo = snapshotDao.persist(snapshotVo); + return snapshotVo; + } + private void logDataObject(final String pref, DataObject data) { final DataStore dstore = data.getDataStore(); String name = null; @@ -473,7 +529,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { try { if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.VOLUME) { SnapshotInfo sinfo = (SnapshotInfo)srcData; - final String snapshotName = StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), snapshotDataStoreDao, _snapshotDetailsDao); + final String snapshotName = StorPoolHelper.getSnapshotName(srcData.getId(), srcData.getUuid(), snapshotDataStoreDao, snapshotDetailsDao); VolumeInfo vinfo = (VolumeInfo)dstData; final String volumeName = vinfo.getUuid(); @@ -491,9 +547,12 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { StorPoolUtil.spLog("Created volume=%s with uuid=%s from snapshot=%s with uuid=%s", StorPoolUtil.getNameFromResponse(resp, false), to.getUuid(), snapshotName, sinfo.getUuid()); } else if (resp.getError().getName().equals("objectDoesNotExist")) { //check if snapshot is on secondary storage - StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snopshot on secondary storage", snapshotName); + StorPoolUtil.spLog("Snapshot %s does not exists on StorPool, will try to create a volume from a snapshot on secondary storage", snapshotName); SnapshotDataStoreVO snap = getSnapshotImageStoreRef(sinfo.getId(), vinfo.getDataCenterId()); - if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) { + SnapshotDetailsVO snapshotDetail = snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE); + if (snapshotDetail != null) { + err = String.format("Could not create volume from snapshot due to: %s", resp.getError()); + } else if (snap != null && StorPoolStorageAdaptor.getVolumeNameFromPath(snap.getInstallPath(), false) == null) { resp = StorPoolUtil.volumeCreate(srcData.getUuid(), null, size, null, "no", "snapshot", sinfo.getBaseVolume().getMaxIops(), conn); if (resp.getError() == null) { VolumeObjectTO dstTO = (VolumeObjectTO) dstData.getTO(); @@ -514,11 +573,11 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { err = String.format("Could not freeze Storpool volume %s. Error: %s", srcData.getUuid(), resp2.getError()); } else { String name = StorPoolUtil.getNameFromResponse(resp, false); - SnapshotDetailsVO snapshotDetails = _snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid()); + SnapshotDetailsVO snapshotDetails = snapshotDetailsDao.findDetail(sinfo.getId(), sinfo.getUuid()); if (snapshotDetails != null) { StorPoolHelper.updateSnapshotDetailsValue(snapshotDetails.getId(), StorPoolUtil.devPath(name), "snapshot"); }else { - StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), StorPoolUtil.devPath(name), _snapshotDetailsDao); + StorPoolHelper.addSnapshotDetails(sinfo.getId(), sinfo.getUuid(), StorPoolUtil.devPath(name), snapshotDetailsDao); } resp = StorPoolUtil.volumeCreate(volumeName, StorPoolUtil.getNameFromResponse(resp, true), size, null, null, "volume", sinfo.getBaseVolume().getMaxIops(), conn); if (resp.getError() == null) { @@ -548,8 +607,10 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { err = String.format("Could not create Storpool volume %s from snapshot %s. Error: %s", volumeName, snapshotName, resp.getError()); } } else if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.SNAPSHOT) { + SnapshotInfo sinfo = (SnapshotInfo)srcData; + SnapshotDetailsVO snapshotDetail = snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE); // bypass secondary storage - if (StorPoolConfigurationManager.BypassSecondaryStorage.value()) { + if (StorPoolConfigurationManager.BypassSecondaryStorage.value() || snapshotDetail != null) { SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData.getTO(); answer = new CopyCmdAnswer(snapshot); } else { @@ -986,9 +1047,9 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { SnapshotObjectTO snapTo = (SnapshotObjectTO)snapshot.getTO(); snapTo.setPath(StorPoolUtil.devPath(name.split("~")[1])); answer = new CreateObjectAnswer(snapTo); - StorPoolHelper.addSnapshotDetails(snapshot.getId(), snapshot.getUuid(), snapTo.getPath(), _snapshotDetailsDao); + StorPoolHelper.addSnapshotDetails(snapshot.getId(), snapshot.getUuid(), snapTo.getPath(), snapshotDetailsDao); //add primary storage of snapshot - StorPoolHelper.addSnapshotDetails(snapshot.getId(), StorPoolUtil.SP_STORAGE_POOL_ID, String.valueOf(snapshot.getDataStore().getId()), _snapshotDetailsDao); + StorPoolHelper.addSnapshotDetails(snapshot.getId(), StorPoolUtil.SP_STORAGE_POOL_ID, String.valueOf(snapshot.getDataStore().getId()), snapshotDetailsDao); StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.takeSnapshot: snapshot: name=%s, uuid=%s, volume: name=%s, uuid=%s", name, snapshot.getUuid(), volumeName, vinfo.getUuid()); } } catch (Exception e) { @@ -1003,7 +1064,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { @Override public void revertSnapshot(final SnapshotInfo snapshot, final SnapshotInfo snapshotOnPrimaryStore, final AsyncCompletionCallback callback) { final VolumeInfo vinfo = snapshot.getBaseVolume(); - final String snapshotName = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), snapshotDataStoreDao, _snapshotDetailsDao); + final String snapshotName = StorPoolHelper.getSnapshotName(snapshot.getId(), snapshot.getUuid(), snapshotDataStoreDao, snapshotDetailsDao); final String volumeName = StorPoolStorageAdaptor.getVolumeNameFromPath(vinfo.getPath(), true); StorPoolUtil.spLog("StorpoolPrimaryDataStoreDriverImpl.revertSnapshot: snapshot: name=%s, uuid=%s, volume: name=%s, uuid=%s", snapshotName, snapshot.getUuid(), volumeName, vinfo.getUuid()); String err = null; @@ -1058,7 +1119,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { } private String getVcPolicyTag(Long vmId) { - ResourceTag resourceTag = vmId != null ? _resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, StorPoolUtil.SP_VC_POLICY) : null; + ResourceTag resourceTag = vmId != null ? resourceTagDao.findByKey(vmId, ResourceObjectType.UserVm, StorPoolUtil.SP_VC_POLICY) : null; return resourceTag != null ? resourceTag.getValue() : ""; } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java index 359b11d491e..44acd1eab75 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolStatsCollector.java @@ -28,19 +28,29 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; import org.apache.cloudstack.storage.snapshot.StorPoolConfigurationManager; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; +import org.apache.commons.lang3.StringUtils; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotDetailsDao; +import com.cloud.storage.dao.SnapshotDetailsVO; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.exception.CloudRuntimeException; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -55,6 +65,12 @@ public class StorPoolStatsCollector extends ManagerBase { private StoragePoolDetailsDao storagePoolDetailsDao; @Inject private ConfigurationDao configurationDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private SnapshotDataStoreDao snapshotDataStoreDao; + @Inject + private SnapshotDetailsDao snapshotDetailsDao; private ScheduledExecutorService executor; @@ -70,7 +86,7 @@ public class StorPoolStatsCollector extends ManagerBase { public boolean start() { List spPools = storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME); if (CollectionUtils.isNotEmpty(spPools)) { - executor = Executors.newScheduledThreadPool(2,new NamedThreadFactory("StorPoolStatsCollector")); + executor = Executors.newScheduledThreadPool(3, new NamedThreadFactory("StorPoolStatsCollector")); long storageStatsInterval = NumbersUtil.parseLong(configurationDao.getValue("storage.stats.interval"), 60000L); long volumeStatsInterval = NumbersUtil.parseLong(configurationDao.getValue("volume.stats.interval"), 60000L); @@ -80,6 +96,13 @@ public class StorPoolStatsCollector extends ManagerBase { if (StorPoolConfigurationManager.StorageStatsInterval.value() > 0 && storageStatsInterval > 0) { executor.scheduleAtFixedRate(new StorPoolStorageStatsMonitorTask(), 120, StorPoolConfigurationManager.StorageStatsInterval.value(), TimeUnit.SECONDS); } + for (StoragePoolVO pool: spPools) { + Integer deleteAfter = StorPoolConfigurationManager.DeleteAfterInterval.valueIn(pool.getId()); + if (deleteAfter != null && deleteAfter > 0) { + executor.scheduleAtFixedRate(new StorPoolSnapshotsWithDelayDelete(), 120, StorPoolConfigurationManager.ListSnapshotsWithDeleteAfterInterval.value(), TimeUnit.SECONDS); + break; + } + } } return true; @@ -185,4 +208,90 @@ public class StorPoolStatsCollector extends ManagerBase { } } } + + class StorPoolSnapshotsWithDelayDelete implements Runnable { + + @Override + public void run() { + List spPools = storagePoolDao.findPoolsByProvider(StorPoolUtil.SP_PROVIDER_NAME); + if (CollectionUtils.isNotEmpty(spPools)) { + Map onePoolForZone = new HashMap<>(); + for (StoragePoolVO storagePoolVO : spPools) { + onePoolForZone.put(storagePoolVO.getDataCenterId(), storagePoolVO); + } + for (StoragePoolVO storagePool : onePoolForZone.values()) { + List snapshotsDetails = snapshotDetailsDao.findDetailsByZoneAndKey(storagePool.getDataCenterId(), StorPoolUtil.SP_DELAY_DELETE); + if (CollectionUtils.isEmpty(snapshotsDetails)) { + return; + } + Map snapshotsWithDelayDelete = new HashMap<>(); + + try { + log.debug(String.format("Collecting snapshots marked to be deleted for zone [%s]", storagePool.getDataCenterId())); + JsonArray arr = StorPoolUtil.snapshotsListAllClusters(StorPoolUtil.getSpConnection(storagePool.getUuid(), + storagePool.getId(), storagePoolDetailsDao, storagePoolDao)); + snapshotsWithDelayDelete.putAll(getSnapshotsMarkedForDeletion(arr)); + log.debug(String.format("Found snapshot details [%s] and snapshots on StorPool with delay delete flag [%s]", snapshotsDetails, snapshotsWithDelayDelete)); + syncSnapshots(snapshotsDetails, snapshotsWithDelayDelete); + } catch (Exception e) { + log.debug("Could not fetch the snapshots with delay delete flag " + e.getMessage()); + } + } + } + } + + private void syncSnapshots(List snapshotsDetails, + Map snapshotsWithDelayDelete) { + for (SnapshotDetailsVO snapshotDetailsVO : snapshotsDetails) { + if (!snapshotsWithDelayDelete.containsKey(snapshotDetailsVO.getValue())) { + StorPoolUtil.spLog("The snapshot [%s] with delayDelete flag is no longer on StorPool. Removing it from CloudStack", snapshotDetailsVO.getValue()); + SnapshotDataStoreVO ss = snapshotDataStoreDao + .findBySourceSnapshot(snapshotDetailsVO.getResourceId(), DataStoreRole.Primary); + if (ss != null) { + ss.setState(State.Destroyed); + snapshotDataStoreDao.update(ss.getId(), ss); + } + SnapshotVO snap = snapshotDao.findById(snapshotDetailsVO.getResourceId()); + if (snap != null) { + snap.setState(com.cloud.storage.Snapshot.State.Destroyed); + snapshotDao.update(snap.getId(), snap); + } + snapshotDetailsDao.remove(snapshotDetailsVO.getId()); + } + } + } + + private Map getSnapshotsMarkedForDeletion(JsonArray arr) { + for (JsonElement jsonElement : arr) { + JsonObject error = jsonElement.getAsJsonObject().getAsJsonObject("error"); + if (error != null) { + throw new CloudRuntimeException(String.format("Could not collect the snapshots marked for deletion from all storage nodes due to: [%s]", error)); + } + } + Map snapshotsWithDelayDelete = new HashMap<>(); + for (JsonElement jsonElement : arr) { + JsonObject response = jsonElement.getAsJsonObject().getAsJsonObject("response"); + if (response == null) { + return snapshotsWithDelayDelete; + } + collectSnapshots(snapshotsWithDelayDelete, response); + } + log.debug("Found snapshots on StorPool" + snapshotsWithDelayDelete); + return snapshotsWithDelayDelete; + } + + private void collectSnapshots(Map snapshotsWithDelayDelete, JsonObject response) { + JsonArray snapshots = response.getAsJsonObject().getAsJsonArray("data"); + for (JsonElement snapshot : snapshots) { + String name = snapshot.getAsJsonObject().get("name").getAsString(); + JsonObject tags = snapshot.getAsJsonObject().get("tags").getAsJsonObject(); + if (!StringUtils.startsWith(name, "*") && StringUtils.containsNone(name, "@") && tags != null && !tags.entrySet().isEmpty()) { + String tag = tags.getAsJsonPrimitive("cs").getAsString(); + if (tag != null && tag.equals(StorPoolUtil.DELAY_DELETE)) { + snapshotsWithDelayDelete.put(name, tag); + } + } + } + } + } } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java index 675dffbda5b..58e87b2fba6 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java @@ -28,6 +28,8 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; + +import org.apache.cloudstack.storage.datastore.api.StorPoolSnapshotDef; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; @@ -124,6 +126,15 @@ public class StorPoolUtil { public static final String SP_VOLUME_ON_CLUSTER = "SP_VOLUME_ON_CLUSTER"; + private static final String DATA = "data"; + + private static final String CLUSTERS = "clusters"; + + public static final String SP_DELAY_DELETE = "SP_DELAY_DELETE"; + + public static final String DELAY_DELETE = "delayDelete"; + + public static enum StorpoolRights { RO("ro"), RW("rw"), DETACH("detach"); @@ -416,27 +427,31 @@ public class StorPoolUtil { public static JsonArray snapshotsList(SpConnectionDesc conn) { SpApiResponse resp = GET("MultiCluster/SnapshotsList", conn); JsonObject obj = resp.fullJson.getAsJsonObject(); - JsonArray data = obj.getAsJsonArray("data"); - return data; + return obj.getAsJsonArray(DATA); + } + + public static JsonArray snapshotsListAllClusters(SpConnectionDesc conn) { + SpApiResponse resp = GET("MultiCluster/AllClusters/SnapshotsList", conn); + JsonObject obj = resp.fullJson.getAsJsonObject(); + return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS); } public static JsonArray volumesList(SpConnectionDesc conn) { SpApiResponse resp = GET("MultiCluster/VolumesList", conn); JsonObject obj = resp.fullJson.getAsJsonObject(); - JsonArray data = obj.getAsJsonArray("data"); - return data; + return obj.getAsJsonArray(DATA); } public static JsonArray volumesSpace(SpConnectionDesc conn) { SpApiResponse resp = GET("MultiCluster/AllClusters/VolumesSpace", conn); JsonObject obj = resp.fullJson.getAsJsonObject(); - return obj.getAsJsonObject("data").getAsJsonArray("clusters"); + return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS); } public static JsonArray templatesStats(SpConnectionDesc conn) { SpApiResponse resp = GET("MultiCluster/AllClusters/VolumeTemplatesStatus", conn); JsonObject obj = resp.fullJson.getAsJsonObject(); - return obj.getAsJsonObject("data").getAsJsonArray("clusters"); + return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS); } private static boolean objectExists(SpApiError err) { @@ -453,7 +468,7 @@ public class StorPoolUtil { if (resp.getError() != null && !objectExists(resp.getError())) { return null; } - JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject(); + JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject(); return data.getAsJsonPrimitive("size").getAsLong(); } @@ -461,7 +476,7 @@ public class StorPoolUtil { SpApiResponse resp = GET("MultiCluster/Snapshot/" + name, conn); JsonObject obj = resp.fullJson.getAsJsonObject(); - JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject(); + JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject(); JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId"); return clusterId != null ? clusterId.getAsString() : null; } @@ -470,7 +485,7 @@ public class StorPoolUtil { SpApiResponse resp = GET("MultiCluster/Volume/" + name, conn); JsonObject obj = resp.fullJson.getAsJsonObject(); - JsonObject data = obj.getAsJsonArray("data").get(0).getAsJsonObject(); + JsonObject data = obj.getAsJsonArray(DATA).get(0).getAsJsonObject(); JsonPrimitive clusterId = data.getAsJsonPrimitive("clusterId"); return clusterId != null ? clusterId.getAsString() : null; } @@ -579,6 +594,10 @@ public class StorPoolUtil { return POST("MultiCluster/VolumeSnapshot/" + volumeName, json, conn); } + public static SpApiResponse volumeSnapshot(StorPoolSnapshotDef snapshot, SpConnectionDesc conn) { + return POST("MultiCluster/VolumeSnapshot/" + snapshot.getVolumeName(), snapshot, conn); + } + public static SpApiResponse volumesGroupSnapshot(final List volumeTOs, final String vmUuid, final String snapshotName, String csTag, SpConnectionDesc conn) { Map json = new LinkedHashMap<>(); @@ -638,7 +657,7 @@ public class StorPoolUtil { public static String getSnapshotNameFromResponse(SpApiResponse resp, boolean tildeNeeded, String globalIdOrRemote) { JsonObject obj = resp.fullJson.getAsJsonObject(); - JsonPrimitive data = obj.getAsJsonObject("data").getAsJsonPrimitive(globalIdOrRemote); + JsonPrimitive data = obj.getAsJsonObject(DATA).getAsJsonPrimitive(globalIdOrRemote); String name = data != null ? data.getAsString() : null; name = name != null ? !tildeNeeded ? name : "~" + name : name; return name; @@ -646,7 +665,7 @@ public class StorPoolUtil { public static String getNameFromResponse(SpApiResponse resp, boolean tildeNeeded) { JsonObject obj = resp.fullJson.getAsJsonObject(); - JsonPrimitive data = obj.getAsJsonObject("data").getAsJsonPrimitive("name"); + JsonPrimitive data = obj.getAsJsonObject(DATA).getAsJsonPrimitive("name"); String name = data != null ? data.getAsString() : null; name = name != null ? name.startsWith("~") && !tildeNeeded ? name.split("~")[1] : name : name; return name; diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java index a735b0fe918..6cfa5ec14a2 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/motion/StorPoolDataMotionStrategy.java @@ -148,18 +148,21 @@ public class StorPoolDataMotionStrategy implements DataMotionStrategy { public StrategyPriority canHandle(DataObject srcData, DataObject destData) { DataObjectType srcType = srcData.getType(); DataObjectType dstType = destData.getType(); - if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.TEMPLATE - && StorPoolConfigurationManager.BypassSecondaryStorage.value()) { + if (srcType == DataObjectType.SNAPSHOT && dstType == DataObjectType.TEMPLATE) { SnapshotInfo sinfo = (SnapshotInfo) srcData; VolumeInfo volume = sinfo.getBaseVolume(); StoragePoolVO storagePool = _storagePool.findById(volume.getPoolId()); if (!storagePool.getStorageProviderName().equals(StorPoolUtil.SP_PROVIDER_NAME)) { return StrategyPriority.CANT_HANDLE; } + SnapshotDetailsVO snapshotDetail = _snapshotDetailsDao.findDetail(sinfo.getId(), StorPoolUtil.SP_DELAY_DELETE); + if (snapshotDetail != null) { + throw new CloudRuntimeException("Cannot create a template from the last snapshot of deleted volume. You can only restore the volume."); + } String snapshotName = StorPoolHelper.getSnapshotName(sinfo.getId(), sinfo.getUuid(), _snapshotStoreDao, _snapshotDetailsDao); StorPoolUtil.spLog("StorPoolDataMotionStrategy.canHandle snapshot name=%s", snapshotName); - if (snapshotName != null) { + if (snapshotName != null && StorPoolConfigurationManager.BypassSecondaryStorage.value()) { return StrategyPriority.HIGHEST; } } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java index dcb2b226467..e4e930c8dee 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolConfigurationManager.java @@ -44,6 +44,16 @@ public class StorPoolConfigurationManager implements Configurable { "The interval in seconds to get StorPool template statistics", false); + public static final ConfigKey DeleteAfterInterval = new ConfigKey<>("Advanced", Integer.class, + "storpool.delete.after.interval", "0", + "The interval (in seconds) after the StorPool snapshot will be deleted", + false, ConfigKey.Scope.StoragePool); + + public static final ConfigKey ListSnapshotsWithDeleteAfterInterval = new ConfigKey<>("Advanced", Integer.class, + "storpool.list.snapshots.delete.after.interval", "360", + "The interval (in seconds) to fetch the StorPool snapshots with deleteAfter flag", + false); + @Override public String getConfigComponentName() { return StorPoolConfigurationManager.class.getSimpleName(); @@ -51,6 +61,6 @@ public class StorPoolConfigurationManager implements Configurable { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval }; + return new ConfigKey[] { BypassSecondaryStorage, StorPoolClusterId, AlternativeEndPointEnabled, AlternativeEndpoint, VolumesStatsInterval, StorageStatsInterval, DeleteAfterInterval, ListSnapshotsWithDeleteAfterInterval }; } } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java index 55d691f33e0..23361bd1e5e 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java @@ -116,6 +116,8 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { if (resp.getError() != null) { final String err = String.format("Failed to clean-up Storpool snapshot %s. Error: %s", name, resp.getError()); StorPoolUtil.spLog(err); + markSnapshotAsDestroyedIfAlreadyRemoved(snapshotId, resp); + throw new CloudRuntimeException(err); } else { res = deleteSnapshotFromDbIfNeeded(snapshotVO, zoneId); StorPoolUtil.spLog("StorpoolSnapshotStrategy.deleteSnapshot: executed successfully=%s, snapshot uuid=%s, name=%s", res, snapshotVO.getUuid(), name); @@ -129,6 +131,16 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { return res; } + private void markSnapshotAsDestroyedIfAlreadyRemoved(Long snapshotId, SpApiResponse resp) { + if (resp.getError().getName().equals("objectDoesNotExist")) { + SnapshotDataStoreVO snapshotOnPrimary = _snapshotStoreDao.findBySourceSnapshot(snapshotId, DataStoreRole.Primary); + if (snapshotOnPrimary != null) { + snapshotOnPrimary.setState(State.Destroyed); + _snapshotStoreDao.update(snapshotOnPrimary.getId(), snapshotOnPrimary); + } + } + } + @Override public StrategyPriority canHandle(Snapshot snapshot, Long zoneId, SnapshotOperation op) { log.debug(String.format("StorpoolSnapshotStrategy.canHandle: snapshot=%s, uuid=%s, op=%s", snapshot.getName(), snapshot.getUuid(), op)); @@ -166,7 +178,7 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { boolean resultIsSet = false; try { while (snapshot != null && - (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error)) { + (snapshot.getState() == Snapshot.State.Destroying || snapshot.getState() == Snapshot.State.Destroyed || snapshot.getState() == Snapshot.State.Error || snapshot.getState() == Snapshot.State.BackedUp)) { SnapshotInfo child = snapshot.getChild(); if (child != null) { @@ -330,9 +342,21 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { } else { snapshotZoneDao.removeSnapshotFromZones(snapshotVO.getId()); } + if (CollectionUtils.isNotEmpty(retrieveSnapshotEntries(snapshotId, null))) { + return true; + } + updateSnapshotToDestroyed(snapshotVO); return true; } + private List retrieveSnapshotEntries(long snapshotId, Long zoneId) { + return snapshotDataFactory.getSnapshots(snapshotId, zoneId); + } + + private void updateSnapshotToDestroyed(SnapshotVO snapshotVo) { + snapshotVo.setState(Snapshot.State.Destroyed); + _snapshotDao.update(snapshotVo.getId(), snapshotVo); + } @Override public SnapshotInfo takeSnapshot(SnapshotInfo snapshot) {