From 86aef08865f3357d97407f82ce5d96c441cc92f2 Mon Sep 17 00:00:00 2001 From: anthony Date: Thu, 19 Jan 2012 20:44:44 -0800 Subject: [PATCH] cleanup snapshot in secondary storage in background Conflicts: core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java server/src/com/cloud/storage/StorageManager.java server/src/com/cloud/storage/StorageManagerImpl.java --- .../resource/NfsSecondaryStorageResource.java | 32 +++ .../src/com/cloud/storage/StorageManager.java | 1 + .../com/cloud/storage/StorageManagerImpl.java | 186 ++++++++++++++---- 3 files changed, 178 insertions(+), 41 deletions(-) diff --git a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java index cd507da0029..10240c1f56c 100755 --- a/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java +++ b/core/src/com/cloud/storage/resource/NfsSecondaryStorageResource.java @@ -39,6 +39,7 @@ import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckHealthAnswer; import com.cloud.agent.api.CheckHealthCommand; +import com.cloud.agent.api.CleanupSnapshotBackupCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.ComputeChecksumCommand; import com.cloud.agent.api.DeleteObjectFromSwiftCommand; @@ -165,6 +166,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return execute((uploadTemplateToSwiftFromSecondaryStorageCommand) cmd); } else if (cmd instanceof DeleteObjectFromSwiftCommand) { return execute((DeleteObjectFromSwiftCommand) cmd); + } else if (cmd instanceof CleanupSnapshotBackupCommand){ + return execute((CleanupSnapshotBackupCommand)cmd); } else { return Answer.createUnsupportedCommandAnswer(cmd); } @@ -837,6 +840,35 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new Answer(cmd, true, null); } + Answer execute(CleanupSnapshotBackupCommand cmd) { + String parent = getRootDir(cmd.getSecondaryStoragePoolURL()); + if (!parent.endsWith(File.separator)) { + parent += File.separator; + } + String absoluteSnapsthotDir = parent + File.separator + "snapshots" + File.separator + cmd.getAccountId() + File.separator + cmd.getVolumeId(); + File ssParent = new File(absoluteSnapsthotDir); + if (ssParent.exists() && ssParent.isDirectory()) { + File[] files = ssParent.listFiles(); + for (File file : files) { + boolean found = false; + String filename = file.getName(); + for (String uuid : cmd.getValidBackupUUIDs()) { + if (filename.startsWith(uuid)) { + found = true; + break; + } + } + if (!found) { + file.delete(); + String msg = "snapshot " + filename + " is not recorded in DB, remove it"; + s_logger.warn(msg); + } + } + } + return new Answer(cmd, true, null); + } + + synchronized public String getRootDir(String secUrl) { try { URI uri = new URI(secUrl); diff --git a/server/src/com/cloud/storage/StorageManager.java b/server/src/com/cloud/storage/StorageManager.java index 6700d0aceff..78bd520678a 100755 --- a/server/src/com/cloud/storage/StorageManager.java +++ b/server/src/com/cloud/storage/StorageManager.java @@ -224,4 +224,5 @@ public interface StorageManager extends Manager { List getUpHostsInPool(long poolId); + void cleanupSecondaryStorage(boolean recurring); } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 24910beae91..dfb750b998b 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -20,6 +20,8 @@ package com.cloud.storage; import java.net.URI; import java.net.URISyntaxException; import java.net.UnknownHostException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -45,6 +47,7 @@ import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.BackupSnapshotCommand; +import com.cloud.agent.api.CleanupSnapshotBackupCommand; import com.cloud.agent.api.Command; import com.cloud.agent.api.CreateStoragePoolCommand; import com.cloud.agent.api.CreateVolumeFromSnapshotAnswer; @@ -1102,7 +1105,9 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag @Override public boolean start() { if (_storageCleanupEnabled) { - _executor.scheduleWithFixedDelay(new StorageGarbageCollector(), _storageCleanupInterval, _storageCleanupInterval, TimeUnit.SECONDS); + Random generator = new Random(); + int initialDelay = generator.nextInt(_storageCleanupInterval); + _executor.scheduleWithFixedDelay(new StorageGarbageCollector(), initialDelay, _storageCleanupInterval, TimeUnit.SECONDS); } else { s_logger.debug("Storage cleanup is not enabled, so the storage cleanup thread is not being scheduled."); } @@ -1978,46 +1983,8 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } } } - - // Cleanup secondary storage hosts - List secondaryStorageHosts = _ssvmMgr.listSecondaryStorageHostsInAllZones(); - for (HostVO secondaryStorageHost : secondaryStorageHosts) { - try { - long hostId = secondaryStorageHost.getId(); - List destroyedTemplateHostVOs = _vmTemplateHostDao.listDestroyed(hostId); - s_logger.debug("Secondary storage garbage collector found " + destroyedTemplateHostVOs.size() + " templates to cleanup on secondary storage host: " - + secondaryStorageHost.getName()); - for (VMTemplateHostVO destroyedTemplateHostVO : destroyedTemplateHostVOs) { - if (!_tmpltMgr.templateIsDeleteable(destroyedTemplateHostVO)) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Not deleting template at: " + destroyedTemplateHostVO); - } - continue; - } - - if (s_logger.isDebugEnabled()) { - s_logger.debug("Deleting template host: " + destroyedTemplateHostVO); - } - - String installPath = destroyedTemplateHostVO.getInstallPath(); - - if (installPath != null) { - Answer answer = _agentMgr.sendToSecStorage(secondaryStorageHost, new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(),destroyedTemplateHostVO.getInstallPath())); - - if (answer == null || !answer.getResult()) { - s_logger.debug("Failed to delete " + destroyedTemplateHostVO + " due to " + ((answer == null) ? "answer is null" : answer.getDetails())); - } else { - _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId()); - s_logger.debug("Deleted template at: " + destroyedTemplateHostVO.getInstallPath()); - } - } else { - _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId()); - } - } - } catch (Exception e) { - s_logger.warn("problem cleaning up secondary storage " + secondaryStorageHost, e); - } - } + + cleanupSecondaryStorage(recurring); List vols = _volsDao.listVolumesToBeDestroyed(); for (VolumeVO vol : vols) { @@ -2035,6 +2002,142 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag scanLock.releaseRef(); } } + + @DB + List findAllVolumeIdInSnapshotTable(Long hostId) { + String sql = "SELECT volume_id from snapshots WHERE sechost_id=? GROUP BY volume_id"; + List list = new ArrayList(); + try { + Transaction txn = Transaction.currentTxn(); + ResultSet rs = null; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, hostId); + rs = pstmt.executeQuery(); + while (rs.next()) { + list.add(rs.getLong(1)); + } + return list; + } catch (Exception e) { + s_logger.debug("failed to get all volumes who has snapshots in secondary storage " + hostId + " due to " + e.getMessage()); + return null; + } + + } + + + List findAllSnapshotForVolume(Long volumeId) { + String sql = "SELECT backup_snap_id FROM snapshots WHERE volume_id=? and backup_snap_id is not NULL"; + try { + Transaction txn = Transaction.currentTxn(); + ResultSet rs = null; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setLong(1, volumeId); + rs = pstmt.executeQuery(); + List list = new ArrayList(); + while (rs.next()) { + list.add(rs.getString(1)); + } + return list; + } catch (Exception e) { + s_logger.debug("failed to get all snapshots for a volume " + volumeId + " due to " + e.getMessage()); + return null; + } + } + + @Override + @DB + public void cleanupSecondaryStorage(boolean recurring) { + try { + // Cleanup templates in secondary storage hosts + List secondaryStorageHosts = _hostDao.listSecondaryStorageHosts(); + for (HostVO secondaryStorageHost : secondaryStorageHosts) { + try { + long hostId = secondaryStorageHost.getId(); + List destroyedTemplateHostVOs = _vmTemplateHostDao.listDestroyed(hostId); + s_logger.debug("Secondary storage garbage collector found " + destroyedTemplateHostVOs.size() + " templates to cleanup on secondary storage host: " + + secondaryStorageHost.getName()); + for (VMTemplateHostVO destroyedTemplateHostVO : destroyedTemplateHostVOs) { + if (!_tmpltMgr.templateIsDeleteable(destroyedTemplateHostVO)) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Not deleting template at: " + destroyedTemplateHostVO); + } + continue; + } + + if (s_logger.isDebugEnabled()) { + s_logger.debug("Deleting template host: " + destroyedTemplateHostVO); + } + + String installPath = destroyedTemplateHostVO.getInstallPath(); + + if (installPath != null) { + Answer answer = _agentMgr.sendToSecStorage(secondaryStorageHost, new DeleteTemplateCommand(secondaryStorageHost.getStorageUrl(), destroyedTemplateHostVO.getInstallPath())); + + if (answer == null || !answer.getResult()) { + s_logger.debug("Failed to delete " + destroyedTemplateHostVO + " due to " + ((answer == null) ? "answer is null" : answer.getDetails())); + } else { + _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId()); + s_logger.debug("Deleted template at: " + destroyedTemplateHostVO.getInstallPath()); + } + } else { + _vmTemplateHostDao.remove(destroyedTemplateHostVO.getId()); + } + } + } catch (Exception e) { + s_logger.warn("problem cleaning up templates in secondary storage " + secondaryStorageHost, e); + } + } + + // Cleanup snapshot in secondary storage hosts + for (HostVO secondaryStorageHost : secondaryStorageHosts) { + try { + long hostId = secondaryStorageHost.getId(); + List vIDs = findAllVolumeIdInSnapshotTable(hostId); + if (vIDs == null) { + continue; + } + for (Long volumeId : vIDs) { + boolean lock = false; + try { + VolumeVO volume = _volsDao.findByIdIncludingRemoved(volumeId); + if (volume.getRemoved() == null) { + volume = _volsDao.acquireInLockTable(volumeId, 10); + if (volume == null) { + continue; + } + lock = true; + } + List snapshots = findAllSnapshotForVolume(volumeId); + if (snapshots == null) { + continue; + } + CleanupSnapshotBackupCommand cmd = new CleanupSnapshotBackupCommand(secondaryStorageHost.getStorageUrl(), secondaryStorageHost.getDataCenterId(), volume.getAccountId(), + volumeId, snapshots); + + Answer answer = _agentMgr.sendToSecStorage(secondaryStorageHost, cmd); + if ((answer == null) || !answer.getResult()) { + String details = "Failed to cleanup snapshots for volume " + volumeId + " due to " + (answer == null ? "null" : answer.getDetails()); + s_logger.warn(details); + } + } catch (Exception e1) { + s_logger.warn("problem cleaning up snapshots in secondary storage " + secondaryStorageHost, e1); + } finally { + if (lock) { + _volsDao.releaseFromLockTable(volumeId); + } + } + } + } catch (Exception e2) { + s_logger.warn("problem cleaning up snapshots in secondary storage " + secondaryStorageHost, e2); + } + } + + } catch (Exception e3) { + s_logger.warn("problem cleaning up secondary storage ", e3); + } + } @Override public String getPrimaryStorageNameLabel(VolumeVO volume) { @@ -3087,6 +3190,7 @@ public class StorageManagerImpl implements StorageManager, StorageService, Manag } } } + @Override public void onManagementNodeJoined(List nodeList, long selfNodeId) {