From fa8faf519889fd4311d5388918d16c4dcc9c3cfe Mon Sep 17 00:00:00 2001 From: Pierre Charton Date: Wed, 12 Mar 2025 10:09:09 +0100 Subject: [PATCH] Update BackrollProvider from BackupProvider --- .../backup/BackrollBackupProvider.java | 65 ++++++++++-- .../backup/backroll/BackrollClient.java | 100 ++++++++++++------ .../backup/BackrollBackupProviderTest.java | 4 +- 3 files changed, 127 insertions(+), 42 deletions(-) diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java index 6a112895a77..69d50f7f5ef 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java @@ -29,7 +29,9 @@ import java.util.stream.Collectors; import javax.inject.Inject; + import org.apache.cloudstack.backup.Backup.Metric; +import org.apache.cloudstack.backup.Backup.RestorePoint; import org.apache.cloudstack.backup.backroll.BackrollClient; import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; import org.apache.cloudstack.backup.backroll.model.BackrollTaskStatus; @@ -153,6 +155,9 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide @Override public Map getBackupMetrics(Long zoneId, List vms) { + Long vmBackupSize=0L; + Long vmBackupProtectedSize=0L; + final Map metrics = new HashMap<>(); if (CollectionUtils.isEmpty(vms)) { logger.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); @@ -170,7 +175,13 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide Metric metric; try { - metric = client.getVirtualMachineMetrics(vm.getUuid()); + final List backups = client.getAllBackupsfromVirtualMachine(vm.getUuid()); + for (BackrollVmBackup backup : backups ) { + BackrollBackupMetrics backupMetric = client.getBackupMetrics(vm.getUuid(), backup.getId()); + vmBackupProtectedSize += backupMetric.getDeduplicated(); + vmBackupSize += backupMetric.getSize(); + } + metric = new Metric(vmBackupSize, vmBackupProtectedSize); } catch (BackrollApiException | IOException e) { throw new CloudRuntimeException("Failed to retrieve backup metrics"); } @@ -192,7 +203,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide } @Override - public boolean takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm) { logger.info("Starting backup for VM ID {} on backroll provider", vm.getUuid()); final BackrollClient client = getClient(vm.getDataCenterId()); @@ -215,18 +226,16 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide backup.setZoneId(vm.getDataCenterId()); Boolean result = backupDao.persist(backup) != null; client.triggerTaskStatus(urlToRequest); - syncBackups(vm, null); - return result; + return new Pair(result, backup); } } catch (ParseException | BackrollApiException | IOException e) { logger.debug(e.getMessage()); throw new CloudRuntimeException("Failed to take backup"); } - return false; + return new Pair(false, null); } - @Override public void syncBackups(VirtualMachine vm, Backup.Metric metric) { logger.info("Starting sync backup for VM ID " + vm.getUuid() + " on backroll provider"); @@ -401,6 +410,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide backupToUpdate.setStatus(Backup.Status.Removed); if (backupDao.persist(backupToUpdate) != null) { logger.debug("Backroll backup {} deleted in database.", backup.getUuid()); + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); return true; } return false; @@ -433,4 +443,47 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide logger.debug("Restoring volume {} from backup {} on the Backroll Backup Provider", volumeUuid, backup.getUuid()); throw new CloudRuntimeException("Backroll plugin does not support this feature"); } + + @Override + public List listRestorePoints(VirtualMachine vm) { + try { + final BackrollClient client = getClient(vm.getDataCenterId()); + return client.listRestorePoints(vm.getUuid()); + } catch (BackrollApiException | IOException e) { + logger.error(e); + throw new CloudRuntimeException("Failed to delete backup"); + } + } + + @Override + public Backup createNewBackupEntryForRestorePoint(RestorePoint restorePoint, VirtualMachine vm, Metric metric) { + final BackrollClient client = getClient(vm.getDataCenterId()); + BackupVO backupToInsert = new BackupVO(); + backupToInsert.setVmId(vm.getId()); + backupToInsert.setExternalId(restorePoint.getId()); + backupToInsert.setType("INCREMENTAL"); + backupToInsert.setDate(restorePoint.getCreated()); + backupToInsert.setStatus(Backup.Status.BackedUp); + backupToInsert.setBackupOfferingId(vm.getBackupOfferingId()); + backupToInsert.setAccountId(vm.getAccountId()); + backupToInsert.setDomainId(vm.getDomainId()); + backupToInsert.setZoneId(vm.getDataCenterId()); + if(metric == null || metric.getBackupSize().equals(0L)){ + try { + BackrollBackupMetrics backupMetrics = client.getBackupMetrics(vm.getUuid() , restorePoint.getId()); + if (backupMetrics != null) { + backupToInsert.setSize(backupMetrics.getDeduplicated()); // real size + backupToInsert.setProtectedSize(backupMetrics.getSize()); // total size + } + } catch (BackrollApiException | IOException e) { + logger.error(e); + throw new CloudRuntimeException("Failed to get backup metrics"); + } + }else{ + backupToInsert.setSize(metric.getBackupSize()); + backupToInsert.setProtectedSize(metric.getDataSize()); + } + backupDao.persist(backupToInsert); + return backupToInsert; + } } diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java index 55da37cbd2c..d5f919683f1 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java @@ -22,6 +22,8 @@ import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; + +import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; @@ -42,7 +44,6 @@ import org.apache.cloudstack.backup.backroll.model.response.policy.BackrollBacku import org.apache.cloudstack.backup.backroll.model.response.policy.BackupPoliciesResponse; import org.apache.cloudstack.backup.backroll.utils.BackrollApiException; import org.apache.cloudstack.backup.backroll.utils.BackrollHttpClientProvider; - import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -54,8 +55,6 @@ import org.json.JSONException; import org.json.JSONObject; import com.fasterxml.jackson.databind.ObjectMapper; - - public class BackrollClient { protected Logger logger = LogManager.getLogger(BackrollClient.class); @@ -71,16 +70,18 @@ public class BackrollClient { public String startBackupJob(final String jobId) throws IOException, BackrollApiException { logger.info("startBackupJob : Trying to start backup for Backroll job: {}", jobId); String backupJob = ""; - BackrollTaskRequestResponse requestResponse = httpProvider.post(String.format("/tasks/singlebackup/%s", jobId), null, BackrollTaskRequestResponse.class); + BackrollTaskRequestResponse requestResponse = httpProvider.post(String.format("/tasks/singlebackup/%s", jobId), + null, BackrollTaskRequestResponse.class); logger.info("startBackupJob : BackupJob status link: {}", requestResponse.location); backupJob = requestResponse.location.replace("/api/v1", ""); return StringUtils.isEmpty(backupJob) ? null : backupJob; } - public String getBackupOfferingUrl() throws IOException, BackrollApiException { + public String getBackupOfferingUrl() throws IOException, BackrollApiException { logger.info("Trying to get backroll backup policies url"); String url = ""; - BackrollTaskRequestResponse requestResponse = httpProvider.get("/backup_policies", BackrollTaskRequestResponse.class); + BackrollTaskRequestResponse requestResponse = httpProvider.get("/backup_policies", + BackrollTaskRequestResponse.class); logger.info("BackrollClient:getBackupOfferingUrl:Apres Parse: " + requestResponse.location); url = requestResponse.location.replace("/api/v1", ""); return StringUtils.isEmpty(url) ? null : url; @@ -90,7 +91,8 @@ public class BackrollClient { logger.info("Trying to list backroll backup policies"); final List policies = new ArrayList<>(); BackupPoliciesResponse backupPoliciesResponse = httpProvider.waitGet(idTask, BackupPoliciesResponse.class); - logger.info("BackrollClient:getBackupOfferings:Apres Parse: " + backupPoliciesResponse.backupPolicies.get(0).name); + logger.info( + "BackrollClient:getBackupOfferings:Apres Parse: " + backupPoliciesResponse.backupPolicies.get(0).name); for (final BackrollBackupPolicyResponse policy : backupPoliciesResponse.backupPolicies) { policies.add(new BackrollOffering(policy.name, policy.id)); } @@ -98,7 +100,8 @@ public class BackrollClient { return policies; } - public boolean restoreVMFromBackup(final String vmId, final String backupName) throws IOException, BackrollApiException { + public boolean restoreVMFromBackup(final String vmId, final String backupName) + throws IOException, BackrollApiException { logger.info("Start restore backup with backroll with backup {} for vm {}", backupName, vmId); boolean isRestoreOk = false; @@ -114,11 +117,12 @@ public class BackrollClient { logger.error("Backroll Error: {}", e.getMessage()); } - BackrollTaskRequestResponse requestResponse = httpProvider.post(String.format("/tasks/restore/%s", vmId), jsonBody, BackrollTaskRequestResponse.class); + BackrollTaskRequestResponse requestResponse = httpProvider.post(String.format("/tasks/restore/%s", vmId), + jsonBody, BackrollTaskRequestResponse.class); String urlToRequest = requestResponse.location.replace("/api/v1", ""); String result = httpProvider.waitGetWithoutParseResponse(urlToRequest); - if(result.contains("SUCCESS")) { + if (result.contains("SUCCESS")) { logger.debug("RESTORE SUCCESS content : " + result); logger.debug("RESTORE SUCCESS"); isRestoreOk = true; @@ -135,10 +139,12 @@ public class BackrollClient { String backupResponse = httpProvider.getWithoutParseResponse("/status/" + taskId); if (backupResponse.contains(TaskState.FAILURE) || backupResponse.contains(TaskState.PENDING)) { - BackrollBackupStatusResponse backupStatusRequestResponse = new ObjectMapper().readValue(backupResponse, BackrollBackupStatusResponse.class); + BackrollBackupStatusResponse backupStatusRequestResponse = new ObjectMapper().readValue(backupResponse, + BackrollBackupStatusResponse.class); status.setState(backupStatusRequestResponse.state); } else { - BackrollBackupStatusSuccessResponse backupStatusSuccessRequestResponse = new ObjectMapper().readValue(backupResponse, BackrollBackupStatusSuccessResponse.class); + BackrollBackupStatusSuccessResponse backupStatusSuccessRequestResponse = new ObjectMapper() + .readValue(backupResponse, BackrollBackupStatusSuccessResponse.class); status.setState(backupStatusSuccessRequestResponse.state); status.setInfo(backupStatusSuccessRequestResponse.info); } @@ -146,14 +152,16 @@ public class BackrollClient { return StringUtils.isEmpty(status.getState()) ? null : status; } - public boolean deleteBackup(final String vmId, final String backupName) throws IOException, BackrollApiException{ + public boolean deleteBackup(final String vmId, final String backupName) throws IOException, BackrollApiException { logger.info("Trying to delete backup {} for vm {} using Backroll", vmId, backupName); boolean isBackupDeleted = false; - BackrollTaskRequestResponse requestResponse = httpProvider.delete(String.format("/virtualmachines/%s/backups/%s", vmId, backupName), BackrollTaskRequestResponse.class); + BackrollTaskRequestResponse requestResponse = httpProvider.delete( + String.format("/virtualmachines/%s/backups/%s", vmId, backupName), BackrollTaskRequestResponse.class); String urlToRequest = requestResponse.location.replace("/api/v1", ""); - BackrollBackupsFromVMResponse backrollBackupsFromVMResponse = httpProvider.waitGet(urlToRequest, BackrollBackupsFromVMResponse.class); + BackrollBackupsFromVMResponse backrollBackupsFromVMResponse = httpProvider.waitGet(urlToRequest, + BackrollBackupsFromVMResponse.class); logger.debug(backrollBackupsFromVMResponse.state); isBackupDeleted = backrollBackupsFromVMResponse.state.equals(TaskState.SUCCESS); @@ -161,8 +169,8 @@ public class BackrollClient { } public void triggerTaskStatus(String urlToRequest) - throws IOException, BackrollApiException{ - httpProvider.waitGetWithoutParseResponse(urlToRequest); + throws IOException, BackrollApiException { + httpProvider.waitGetWithoutParseResponse(urlToRequest); } public Metric getVirtualMachineMetrics(final String vmId) throws IOException, BackrollApiException { @@ -170,11 +178,13 @@ public class BackrollClient { Metric metric = new Metric(0L, 0L); - BackrollTaskRequestResponse requestResponse = httpProvider.get(String.format("/virtualmachines/%s/repository", vmId), BackrollTaskRequestResponse.class); + BackrollTaskRequestResponse requestResponse = httpProvider + .get(String.format("/virtualmachines/%s/repository", vmId), BackrollTaskRequestResponse.class); String urlToRequest = requestResponse.location.replace("/api/v1", ""); - BackrollVmMetricsResponse vmMetricsResponse = httpProvider.waitGet(urlToRequest, BackrollVmMetricsResponse.class); + BackrollVmMetricsResponse vmMetricsResponse = httpProvider.waitGet(urlToRequest, + BackrollVmMetricsResponse.class); if (vmMetricsResponse != null && vmMetricsResponse.state.equals(TaskState.SUCCESS)) { logger.debug("SUCCESS ok"); @@ -192,18 +202,21 @@ public class BackrollClient { return metric; } - public BackrollBackupMetrics getBackupMetrics(String vmId, String backupId) throws IOException, BackrollApiException { + public BackrollBackupMetrics getBackupMetrics(String vmId, String backupId) + throws IOException, BackrollApiException { logger.info("Trying to get backup metrics for VM: {}, and backup: {}", vmId, backupId); BackrollBackupMetrics metrics = null; - BackrollTaskRequestResponse requestResponse = httpProvider.get(String.format("/virtualmachines/%s/backups/%s", vmId, backupId), BackrollTaskRequestResponse.class); + BackrollTaskRequestResponse requestResponse = httpProvider.get( + String.format("/virtualmachines/%s/backups/%s", vmId, backupId), BackrollTaskRequestResponse.class); String urlToRequest = requestResponse.location.replace("/api/v1", ""); logger.debug(urlToRequest); - BackrollBackupMetricsResponse metricsResponse = httpProvider.waitGet(urlToRequest, BackrollBackupMetricsResponse.class); + BackrollBackupMetricsResponse metricsResponse = httpProvider.waitGet(urlToRequest, + BackrollBackupMetricsResponse.class); if (metricsResponse.info != null) { metrics = new BackrollBackupMetrics(Long.parseLong(metricsResponse.info.originalSize), Long.parseLong(metricsResponse.info.deduplicatedSize)); @@ -211,26 +224,45 @@ public class BackrollClient { return metrics; } - public List getAllBackupsfromVirtualMachine(String vmId) throws BackrollApiException, IOException { - logger.info("Trying to retrieve all backups for vm {}", vmId); - + public List getAllBackupsfromVirtualMachine(String vmId) + throws BackrollApiException, IOException { List backups = new ArrayList(); + List backupInfos = getBackupInfosFromVm(vmId); + if (backupInfos != null && backupInfos.size() > 0) { + for (BackupInfos infos : backupInfos) { + var dateStart = new DateTime(infos.start); + backups.add(new BackrollVmBackup(infos.id, infos.name, dateStart.toDate())); + } + } + return backups; + } - BackrollTaskRequestResponse requestResponse = httpProvider.get(String.format("/virtualmachines/%s/backups", vmId), BackrollTaskRequestResponse.class); + private List getBackupInfosFromVm(String vmId) + throws BackrollApiException, IOException { + logger.info("Trying to retrieve all backups for vm {}", vmId); + BackrollTaskRequestResponse requestResponse = httpProvider + .get(String.format("/virtualmachines/%s/backups", vmId), BackrollTaskRequestResponse.class); String urlToRequest = requestResponse.location.replace("/api/v1", ""); logger.debug(urlToRequest); - VirtualMachineBackupsResponse virtualMachineBackupsResponse = httpProvider.waitGet(urlToRequest, VirtualMachineBackupsResponse.class); - + VirtualMachineBackupsResponse virtualMachineBackupsResponse = httpProvider.waitGet(urlToRequest, + VirtualMachineBackupsResponse.class); if (virtualMachineBackupsResponse.state.equals(TaskState.SUCCESS)) { - if (virtualMachineBackupsResponse.info.archives.size() > 0) { - for (BackupInfos infos : virtualMachineBackupsResponse.info.archives) { - var dateStart = new DateTime(infos.start); - backups.add(new BackrollVmBackup(infos.id, infos.name, dateStart.toDate())); - } + return virtualMachineBackupsResponse.info.archives; + } + return null; + + } + + public List listRestorePoints(String vmId) throws BackrollApiException, IOException { + List backups = new ArrayList(); + List backupInfos = getBackupInfosFromVm(vmId); + if (backupInfos != null && backupInfos.size() > 0) { + for (BackupInfos infos : backupInfos) { + var dateStart = new DateTime(infos.start); + backups.add(new Backup.RestorePoint(infos.id, dateStart.toDate(), "INCREMENTAL")); } } - return backups; } } diff --git a/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java index 9107b228392..66831b441bf 100644 --- a/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java +++ b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java @@ -110,8 +110,8 @@ public class BackrollBackupProviderTest { Mockito.doReturn(new BackupVO()).when(backupDao).persist(Mockito.any(BackupVO.class)); Mockito.doNothing().when(clientMock).triggerTaskStatus(Mockito.anyString()); syncBackups_Test(); - Boolean result = backupProvider.takeBackup(vmInstanceVO); - assertTrue(result); + Pair result = backupProvider.takeBackup(vmInstanceVO); + assertTrue(result.first()); } @Test