From c36cd2c26cb5d1a171ecae96ee317d9ccb234d13 Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 5 Feb 2026 09:12:50 +0530 Subject: [PATCH] Backup of stopped VMs --- .../cloudstack/backup/StartBackupCommand.java | 16 +- .../LibvirtStartBackupCommandWrapper.java | 101 ++++++-- .../LibvirtStartNBDServerCommandWrapper.java | 32 +-- .../backup/IncrementalBackupServiceImpl.java | 230 ++++++++++-------- 4 files changed, 239 insertions(+), 140 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java index d4ef6652b1e..b43c4661843 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/StartBackupCommand.java @@ -25,21 +25,25 @@ public class StartBackupCommand extends Command { private String vmName; private String toCheckpointId; private String fromCheckpointId; + private Long fromCheckpointCreateTime; private int nbdPort; private Map diskPathUuidMap; private String hostIpAddress; + private boolean stoppedVM; public StartBackupCommand() { } - public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, - int nbdPort, Map diskPathUuidMap, String hostIpAddress) { + public StartBackupCommand(String vmName, String toCheckpointId, String fromCheckpointId, Long fromCheckpointCreateTime, + int nbdPort, Map diskPathUuidMap, String hostIpAddress, boolean stoppedVM) { this.vmName = vmName; this.toCheckpointId = toCheckpointId; this.fromCheckpointId = fromCheckpointId; + this.fromCheckpointCreateTime = fromCheckpointCreateTime; this.nbdPort = nbdPort; this.diskPathUuidMap = diskPathUuidMap; this.hostIpAddress = hostIpAddress; + this.stoppedVM = stoppedVM; } public String getVmName() { @@ -54,6 +58,10 @@ public class StartBackupCommand extends Command { return fromCheckpointId; } + public Long getFromCheckpointCreateTime() { + return fromCheckpointCreateTime; + } + public int getNbdPort() { return nbdPort; } @@ -70,6 +78,10 @@ public class StartBackupCommand extends Command { return hostIpAddress; } + public boolean isStoppedVM() { + return stoppedVM; + } + @Override public boolean executeInSequence() { return true; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java index 1dfef22c17e..bc3faa04493 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartBackupCommandWrapper.java @@ -25,13 +25,8 @@ import org.apache.cloudstack.backup.StartBackupAnswer; import org.apache.cloudstack.backup.StartBackupCommand; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; -import org.libvirt.Connect; -import org.libvirt.Domain; -import org.libvirt.DomainInfo; - import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.StringUtils; @@ -43,22 +38,25 @@ public class LibvirtStartBackupCommandWrapper extends CommandWrapper\n"); + xml.append(" ").append(checkpointName).append("\n"); + xml.append(" ").append(createTime).append("\n"); + xml.append(""); + return xml.toString(); + } + private String createBackupXml(StartBackupCommand cmd, String fromCheckpointId, int nbdPort, LibvirtComputingResource resource) { StringBuilder xml = new StringBuilder(); xml.append("\n"); @@ -145,4 +184,30 @@ public class LibvirtStartBackupCommandWrapper extends CommandWrapper" + checkpointId + "\n" + ""; } + + private Answer handleStoppedVmBackup(StartBackupCommand cmd, LibvirtComputingResource resource, String toCheckpointId) { + String vmName = cmd.getVmName(); + Map diskPathUuidMap = cmd.getDiskPathUuidMap(); + for (Map.Entry entry : diskPathUuidMap.entrySet()) { + String diskPath = entry.getKey(); + Script script = new Script("sudo"); + script.add("qemu-img"); + script.add("bitmap"); + script.add("--add"); + script.add(diskPath); + script.add(toCheckpointId); + String result = script.execute(); + if (result != null) { + return new StartBackupAnswer(cmd, false, + "Failed to add bitmap " + toCheckpointId + " to disk " + diskPath + ": " + result); + } + } + long checkpointCreateTime = getCheckpointCreateTime(); + return new StartBackupAnswer(cmd, true, "Stopped VM backup: checkpoint bitmap added successfully", + checkpointCreateTime); + } + + private long getCheckpointCreateTime() { + return System.currentTimeMillis() / 1000; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java index c7f2e8d6d08..7a8588809df 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartNBDServerCommandWrapper.java @@ -32,7 +32,8 @@ import com.cloud.utils.script.Script; public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper { protected Logger logger = LogManager.getLogger(getClass()); - private StartNBDServerAnswer handleUpload(StartNBDServerCommand cmd) { + @Override + public Answer execute(StartNBDServerCommand cmd, LibvirtComputingResource resource) { String volumePath = cmd.getVolumePath(); int nbdPort = cmd.getNbdPort(); String hostIpAddress = cmd.getHostIpAddress(); @@ -60,8 +61,14 @@ public class LibvirtStartNBDServerCommandWrapper extends CommandWrapper volumes = volumeDao.findByInstance(vmId); Map diskPathUuidMap = new HashMap<>(); for (Volume vol : volumes) { - StoragePoolVO storagePool = primaryDataStoreDao.findById(vol.getPoolId()); - String volumePath = String.format("/mnt/%s/%s", storagePool.getUuid(), vol.getPath()); + String volumePath = getVolumePathForFileBasedBackend(vol); diskPathUuidMap.put(volumePath, vol.getUuid()); } - Host host = hostDao.findById(vm.getHostId()); + Host host = hostDao.findById(hostId); StartBackupCommand startCmd = new StartBackupCommand( vm.getInstanceName(), toCheckpointId, fromCheckpointId, + fromCheckpointCreateTime, nbdPort, diskPathUuidMap, - host.getPrivateIpAddress() + host.getPrivateIpAddress(), + vm.getState() == State.Stopped ); + StartBackupAnswer answer; try { - StartBackupAnswer answer; - if (dummyOffering) { answer = new StartBackupAnswer(startCmd, true, "Dummy answer", System.currentTimeMillis()); } else { - answer = (StartBackupAnswer) agentManager.send(vm.getHostId(), startCmd); + answer = (StartBackupAnswer) agentManager.send(hostId, startCmd); } - - if (!answer.getResult()) { - backupDao.remove(backup.getId()); - throw new CloudRuntimeException("Failed to start backup: " + answer.getDetails()); - } - - // Update backup with checkpoint creation time - backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); - if (Boolean.TRUE.equals(answer.getIncremental())) { - // todo: set it in the backend - backup.setType("Incremental"); - } - backupDao.update(backup.getId(), backup); - - BackupResponse response = new BackupResponse(); - response.setId(backup.getUuid()); - response.setVmId(vm.getUuid()); - response.setStatus(backup.getStatus()); - return response; - } catch (AgentUnavailableException | OperationTimedoutException e) { backupDao.remove(backup.getId()); throw new CloudRuntimeException("Failed to communicate with agent", e); } + + if (!answer.getResult()) { + backupDao.remove(backup.getId()); + throw new CloudRuntimeException("Failed to start backup: " + answer.getDetails()); + } + + // Update backup with checkpoint creation time + backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); + if (Boolean.TRUE.equals(answer.getIncremental())) { + // todo: set it in the backend + backup.setType("Incremental"); + } + backupDao.update(backup.getId(), backup); + + BackupResponse response = new BackupResponse(); + response.setId(backup.getUuid()); + response.setVmId(vm.getUuid()); + response.setStatus(backup.getStatus()); + return response; } @Override @@ -254,40 +255,43 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme throw new CloudRuntimeException("Image transfers not finalized for backup: " + backupId); } - StopBackupCommand stopCmd = new StopBackupCommand(vm.getInstanceName(), vmId, backupId); + if (vm.getState() == State.Running) { + StopBackupCommand stopCmd = new StopBackupCommand(vm.getInstanceName(), vmId, backupId); - try { StopBackupAnswer answer; - if (dummyOffering) { - answer = new StopBackupAnswer(stopCmd, true, "Dummy answer"); - } else { - answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); + try { + if (dummyOffering) { + answer = new StopBackupAnswer(stopCmd, true, "Dummy answer"); + } else { + answer = (StopBackupAnswer) agentManager.send(backup.getHostId(), stopCmd); + } + + } catch (AgentUnavailableException | OperationTimedoutException e) { + throw new CloudRuntimeException("Failed to communicate with agent", e); } if (!answer.getResult()) { throw new CloudRuntimeException("Failed to stop backup: " + answer.getDetails()); } - - // Update VM checkpoint tracking - String oldCheckpointId = vm.getActiveCheckpointId(); - vm.setActiveCheckpointId(backup.getToCheckpointId()); - vm.setActiveCheckpointCreateTime(backup.getCheckpointCreateTime()); - vmInstanceDao.update(vmId, vm); - - // Delete old checkpoint if exists (POC: skip actual libvirt call) - if (oldCheckpointId != null) { - // todo: In production: send command to delete oldCheckpointId via virsh checkpoint-delete - logger.debug("Would delete old checkpoint: " + oldCheckpointId); - } - - // Delete backup session record - backupDao.remove(backup.getId()); - - return true; - - } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent", e); } + + // Update VM checkpoint tracking + String oldCheckpointId = vm.getActiveCheckpointId(); + vm.setActiveCheckpointId(backup.getToCheckpointId()); + vm.setActiveCheckpointCreateTime(backup.getCheckpointCreateTime()); + vmInstanceDao.update(vmId, vm); + + // Delete old checkpoint if exists (POC: skip actual libvirt call) + if (oldCheckpointId != null) { + // todo: In production: send command to delete oldCheckpointId via virsh checkpoint-delete + logger.debug("Would delete old checkpoint: " + oldCheckpointId); + } + + // Delete backup session record + backupDao.remove(backup.getId()); + + return true; + } private ImageTransferVO createDownloadImageTransfer(Long backupId, VolumeVO volume) { @@ -300,6 +304,13 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme String transferId = UUID.randomUUID().toString(); Host host = hostDao.findById(backup.getHostId()); + + VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); + if (vm.getState() == State.Stopped) { + String volumePath = getVolumePathForFileBasedBackend(volume); + startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, backup.getNbdPort()); + } + CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( transferId, host.getPrivateIpAddress(), @@ -357,27 +368,16 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme return hosts.get(0); } - private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { - final String direction = ImageTransfer.Direction.upload.toString(); - String transferId = UUID.randomUUID().toString(); - - int nbdPort = allocateNbdPort(); - Long poolId = volume.getPoolId(); - StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); - Host host = getFirstHostFromStoragePool(storagePoolVO); - - // todo: This only works with file based storage (not ceph, linbit) - String volumePath = String.format("/mnt/%s/%s", storagePoolVO.getUuid(), volume.getPath()); + private void startNBDServer(String transferId, String direction, Host host, String exportName, String volumePath, int nbdPort) { StartNBDServerAnswer nbdServerAnswer; StartNBDServerCommand nbdServerCmd = new StartNBDServerCommand( transferId, host.getPrivateIpAddress(), - volume.getUuid(), + exportName, volumePath, nbdPort, direction ); - try { nbdServerAnswer = (StartNBDServerAnswer) agentManager.send(host.getId(), nbdServerCmd); } catch (AgentUnavailableException | OperationTimedoutException e) { @@ -386,6 +386,40 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme if (!nbdServerAnswer.getResult()) { throw new CloudRuntimeException("Failed to start the NBD server"); } + } + + private String getVolumePathForFileBasedBackend(Volume volume) { + Long poolId = volume.getPoolId(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); + // todo: This only works with file based storage (not ceph, linbit) + String volumePath = String.format("/mnt/%s/%s", storagePoolVO.getUuid(), volume.getPath()); + return volumePath; + } + + private ImageTransferVO createUploadImageTransfer(VolumeVO volume) { + final String direction = ImageTransfer.Direction.upload.toString(); + String transferId = UUID.randomUUID().toString(); + int nbdPort = allocateNbdPort(); + + Long poolId = volume.getPoolId(); + StoragePoolVO storagePoolVO = primaryDataStoreDao.findById(poolId); + Host host = getFirstHostFromStoragePool(storagePoolVO); + String volumePath = getVolumePathForFileBasedBackend(volume); + + startNBDServer(transferId, direction, host, volume.getUuid(), volumePath, nbdPort); + + ImageTransferVO imageTransfer = new ImageTransferVO( + transferId, + null, + volume.getId(), + host.getId(), + nbdPort, + ImageTransferVO.Phase.transferring, + ImageTransfer.Direction.upload, + volume.getAccountId(), + volume.getDomainId(), + volume.getDataCenterId() + ); CreateImageTransferAnswer transferAnswer; CreateImageTransferCommand transferCmd = new CreateImageTransferCommand( @@ -401,22 +435,10 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme transferAnswer = (CreateImageTransferAnswer) ssvm.sendMessage(transferCmd); if (!transferAnswer.getResult()) { - StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); + stopNbdServer(imageTransfer); throw new CloudRuntimeException("Failed to create image transfer: " + transferAnswer.getDetails()); } - ImageTransferVO imageTransfer = new ImageTransferVO( - transferId, - null, - volume.getId(), - host.getId(), - nbdPort, - ImageTransferVO.Phase.transferring, - ImageTransfer.Direction.upload, - volume.getAccountId(), - volume.getDomainId(), - volume.getDataCenterId() - ); imageTransfer.setTransferUrl(transferAnswer.getTransferUrl()); imageTransfer.setSignedTicketId(transferAnswer.getImageTransferId()); @@ -484,6 +506,29 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme } catch (AgentUnavailableException | OperationTimedoutException e) { throw new CloudRuntimeException("Failed to communicate with agent", e); } + + VMInstanceVO vm = vmInstanceDao.findById(backup.getVmId()); + if (vm.getState() == State.Stopped) { + boolean stopNbdServerResult = stopNbdServer(imageTransfer); + if (!stopNbdServerResult) { + throw new CloudRuntimeException("Failed to stop the nbd server"); + } + } + } + + private boolean stopNbdServer(ImageTransferVO imageTransfer) { + String transferId = imageTransfer.getUuid(); + int nbdPort = imageTransfer.getNbdPort(); + String direction = imageTransfer.getDirection().toString(); + StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); + Answer answer; + try { + answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); + } catch (AgentUnavailableException | OperationTimedoutException e) { + logger.error("Failed to stop NBD server on image transfer finalization", e); + return false; + } + return answer.getResult(); } private void finalizeUploadImageTransfer(ImageTransferVO imageTransfer) { @@ -491,20 +536,14 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme int nbdPort = imageTransfer.getNbdPort(); String direction = imageTransfer.getDirection().toString(); - StopNBDServerCommand stopNbdServerCommand = new StopNBDServerCommand(transferId, direction, nbdPort); - Answer answer; - try { - answer = agentManager.send(imageTransfer.getHostId(), stopNbdServerCommand); - } catch (AgentUnavailableException | OperationTimedoutException e) { - throw new CloudRuntimeException("Failed to communicate with agent", e); - } - if (!answer.getResult()) { + boolean stopNbdServerResult = stopNbdServer(imageTransfer); + if (!stopNbdServerResult) { throw new CloudRuntimeException("Failed to stop the nbd server"); } FinalizeImageTransferCommand finalizeCmd = new FinalizeImageTransferCommand(transferId, direction, nbdPort); EndPoint ssvm = _epSelector.findSsvm(imageTransfer.getDataCenterId()); - answer = ssvm.sendMessage(finalizeCmd); + Answer answer = ssvm.sendMessage(finalizeCmd); if (!answer.getResult()) { throw new CloudRuntimeException("Failed to finalize image transfer: " + answer.getDetails()); @@ -527,7 +566,6 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme } imageTransfer.setPhase(ImageTransferVO.Phase.finished); imageTransferDao.update(imageTransfer.getId(), imageTransfer); - imageTransferDao.remove(imageTransfer.getId()); return true; } @@ -688,15 +726,11 @@ public class IncrementalBackupServiceImpl extends ManagerBase implements Increme String transferId = transfer.getUuid(); transferIds.add(transferId); - String volumePath = volume.getPath(); - if (volumePath == null) { + if (volume.getPath() == null) { logger.warn("Volume path is null for image transfer: " + transfer.getUuid()); continue; } - - StoragePoolVO storagePool = primaryDataStoreDao.findById(volume.getPoolId()); - volumePath = String.format("/mnt/%s/%s", storagePool.getUuid(), volumePath); - + String volumePath = getVolumePathForFileBasedBackend(volume); volumePaths.put(transferId, volumePath); volumeSizes.put(transferId, volume.getSize()); }