From 86f5066d9957dbe66ec82a38527083c3054fd292 Mon Sep 17 00:00:00 2001 From: Pierre Charton Date: Fri, 21 Mar 2025 14:06:54 +0100 Subject: [PATCH] Delete syncBackups in backroll provider, fix delete backup and fix restore --- .../backup/BackrollBackupProvider.java | 157 +----------------- .../backup/backroll/BackrollClient.java | 4 +- .../utils/BackrollHttpClientProvider.java | 23 ++- .../backup/BackrollBackupProviderTest.java | 19 +-- 4 files changed, 33 insertions(+), 170 deletions(-) diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java index 69d50f7f5ef..764ab0e2a5a 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java @@ -34,8 +34,6 @@ import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.Backup.RestorePoint; import org.apache.cloudstack.backup.backroll.BackrollClient; import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; -import org.apache.cloudstack.backup.backroll.model.BackrollTaskStatus; -import org.apache.cloudstack.backup.backroll.model.BackrollVmBackup; import org.apache.cloudstack.backup.backroll.utils.BackrollApiException; import org.apache.cloudstack.backup.backroll.utils.BackrollHttpClientProvider; import org.apache.cloudstack.backup.dao.BackupDao; @@ -155,9 +153,6 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide @Override public Map getBackupMetrics(Long zoneId, List vms) { - Long vmBackupSize=0L; - Long vmBackupProtectedSize=0L; - final Map metrics = new HashMap<>(); if (CollectionUtils.isEmpty(vms)) { logger.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); @@ -175,13 +170,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide Metric metric; try { - final List backups = client.getAllBackupsfromVirtualMachine(vm.getUuid()); - for (BackrollVmBackup backup : backups ) { - BackrollBackupMetrics backupMetric = client.getBackupMetrics(vm.getUuid(), backup.getId()); - vmBackupProtectedSize += backupMetric.getDeduplicated(); - vmBackupSize += backupMetric.getSize(); - } - metric = new Metric(vmBackupSize, vmBackupProtectedSize); + metric = client.getVirtualMachineMetrics(vm.getUuid()); } catch (BackrollApiException | IOException e) { throw new CloudRuntimeException("Failed to retrieve backup metrics"); } @@ -235,136 +224,6 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide return new Pair(false, null); } - - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - logger.info("Starting sync backup for VM ID " + vm.getUuid() + " on backroll provider"); - - final BackrollClient client = getClient(vm.getDataCenterId()); - List backupsInDb = backupDao.listByVmId(vm.getDataCenterId(), vm.getId()); - - for (Backup backup : backupsInDb) { - if (backup.getStatus().equals(Backup.Status.BackingUp)) { - BackrollTaskStatus response; - try { - response = client.checkBackupTaskStatus(backup.getExternalId()); - } catch (ParseException | BackrollApiException | IOException e) { - logger.error(e); - throw new CloudRuntimeException("Failed to sync backups"); - } - - if (response != null) { - logger.debug("backroll backup id: {}", backup.getExternalId()); - logger.debug("backroll backup status: {}", response.getState()); - - BackupVO backupToUpdate = ((BackupVO) backup); - - if (response.getState().equals("PENDING")) { - backupToUpdate.setStatus(Backup.Status.BackingUp); - } else if (response.getState().equals("FAILURE")) { - backupToUpdate.setStatus(Backup.Status.Failed); - } else if (response.getState().equals("SUCCESS")) { - backupToUpdate.setStatus(Backup.Status.BackedUp); - backupToUpdate.setExternalId(backup.getExternalId() + "," + response.getInfo()); - - BackrollBackupMetrics backupMetrics = null; - try { - backupMetrics = client.getBackupMetrics(vm.getUuid() , response.getInfo()); - if (backupMetrics != null) { - backupToUpdate.setSize(backupMetrics.getDeduplicated()); // real size - backupToUpdate.setProtectedSize(backupMetrics.getSize()); // total size - } - } catch (BackrollApiException | IOException e) { - logger.error(e); - throw new CloudRuntimeException("Failed to get backup metrics"); - } - } else { - backupToUpdate.setStatus(Backup.Status.BackingUp); - } - - if (backupDao.persist(backupToUpdate) != null) { - logger.info("Backroll mise à jour enregistrée"); - } - } - } else if (backup.getStatus().equals(Backup.Status.BackedUp) && backup.getSize().equals(0L)) { - String backupId = backup.getExternalId().contains(",") ? backup.getExternalId().split(",")[1] : backup.getExternalId(); - BackrollBackupMetrics backupMetrics; - try { - backupMetrics = client.getBackupMetrics(vm.getUuid() , backupId); - } catch (BackrollApiException | IOException e) { - logger.error(e); - throw new CloudRuntimeException("Failed to get backup metrics"); - } - - if (backupMetrics != null) { - BackupVO backupToUpdate = ((BackupVO) backup); - backupToUpdate.setSize(backupMetrics.getDeduplicated()); // real size - backupToUpdate.setProtectedSize(backupMetrics.getSize()); // total size - backupDao.persist(backupToUpdate); - } - } - } - - // Backups synchronisation between Backroll ad CS Db - List backupsFromBackroll; - try { - backupsFromBackroll = client.getAllBackupsfromVirtualMachine(vm.getUuid()); - - backupsInDb = backupDao.listByVmId(null, vm.getId()); - - // insert new backroll backup in CS - for (BackrollVmBackup backupInBackroll : backupsFromBackroll) { - Backup backupToFind = backupsInDb.stream() - .filter(backupInDb -> backupInDb.getExternalId().contains(backupInBackroll.getName())) - .findAny() - .orElse(null); - - if (backupToFind == null) { - BackupVO backupToInsert = new BackupVO(); - backupToInsert.setVmId(vm.getId()); - backupToInsert.setExternalId(backupInBackroll.getId() + "," + backupInBackroll.getName()); - backupToInsert.setType("INCREMENTAL"); - backupToInsert.setDate(backupInBackroll.getDate()); - backupToInsert.setSize(0L); - backupToInsert.setProtectedSize(0L); - backupToInsert.setStatus(Backup.Status.BackedUp); - backupToInsert.setBackupOfferingId(vm.getBackupOfferingId()); - backupToInsert.setAccountId(vm.getAccountId()); - backupToInsert.setDomainId(vm.getDomainId()); - backupToInsert.setZoneId(vm.getDataCenterId()); - backupDao.persist(backupToInsert); - } - if (backupToFind != null && backupToFind.getStatus() == Backup.Status.Removed) { - BackupVO backupToUpdate = ((BackupVO) backupToFind); - backupToUpdate.setStatus(Backup.Status.BackedUp); - if (backupDao.persist(backupToUpdate) != null) { - logger.info("Backroll update saved"); - backupDao.remove(backupToFind.getId()); - } - } - } - - // delete deleted backroll backup in CS - backupsInDb = backupDao.listByVmId(null, vm.getId()); - for (Backup backup : backupsInDb) { - String backupName = backup.getExternalId().contains(",") ? backup.getExternalId().split(",")[1] : backup.getExternalId(); - BackrollVmBackup backupToFind = backupsFromBackroll.stream() - .filter(backupInBackroll -> backupInBackroll.getName().contains(backupName)) - .findAny() - .orElse(null); - - if (backupToFind == null) { - BackupVO backupToUpdate = ((BackupVO) backup); - backupToUpdate.setStatus(Backup.Status.Removed); - if (backupDao.persist(backupToUpdate) != null) { - logger.debug("Backroll delete saved (sync)"); - } - } - } - } catch (BackrollApiException | IOException e) { - logger.error(e); - } - } - @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); @@ -381,11 +240,11 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide @Override public boolean deleteBackup(Backup backup, boolean forced) { - logger.info("backroll delete backup id: {}", backup.getExternalId()); + logger.info("BACKROLL: delete backup id: {}", backup.getExternalId()); if (backup.getStatus().equals(Backup.Status.BackingUp)) { throw new CloudRuntimeException("You can't delete a backup while it still BackingUp"); } else { - logger.debug("backroll - try delete backup"); + logger.debug("BACKROLL: try delete backup"); if (backup.getStatus().equals(Backup.Status.Removed) || backup.getStatus().equals(Backup.Status.Failed)){ return deleteBackupInDb(backup); @@ -393,12 +252,12 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); try { if (getClient(backup.getZoneId()).deleteBackup(vm.getUuid(), getBackupName(backup))) { - logger.debug("Backup deletion for backup {} complete on backroll side.", backup.getUuid()); + logger.debug("BACKROLL: Backup deletion for backup {} complete on backroll side.", backup.getUuid()); return deleteBackupInDb(backup); } } catch (BackrollApiException | IOException e) { logger.error(e); - throw new CloudRuntimeException("Failed to delete backup"); + throw new CloudRuntimeException("BACKROLL: Failed to delete backup"); } } } @@ -409,7 +268,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide BackupVO backupToUpdate = ((BackupVO) backup); backupToUpdate.setStatus(Backup.Status.Removed); if (backupDao.persist(backupToUpdate) != null) { - logger.debug("Backroll backup {} deleted in database.", backup.getUuid()); + logger.debug("BACKROLL: Backroll backup {} deleted in database.", backup.getUuid()); VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); return true; } @@ -421,7 +280,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide try { if (backrollClient == null) { logger.debug("backroll client null - instantiation of new one "); - BackrollHttpClientProvider provider = BackrollHttpClientProvider.createProvider(new BackrollHttpClientProvider(), BackrollUrlConfigKey.valueIn(zoneId), BackrollAppNameConfigKey.valueIn(zoneId), BackrollPasswordConfigKey.valueIn(zoneId), true, 300, 600); + BackrollHttpClientProvider provider = BackrollHttpClientProvider.createProvider(BackrollUrlConfigKey.valueIn(zoneId), BackrollAppNameConfigKey.valueIn(zoneId), BackrollPasswordConfigKey.valueIn(zoneId), true, 300, 600); backrollClient = new BackrollClient(provider); } return backrollClient; @@ -451,7 +310,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide return client.listRestorePoints(vm.getUuid()); } catch (BackrollApiException | IOException e) { logger.error(e); - throw new CloudRuntimeException("Failed to delete backup"); + throw new CloudRuntimeException("Error while listing restore points"); } } diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java index d5f919683f1..52712ac71f3 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java @@ -153,7 +153,7 @@ public class BackrollClient { } public boolean deleteBackup(final String vmId, final String backupName) throws IOException, BackrollApiException { - logger.info("Trying to delete backup {} for vm {} using Backroll", vmId, backupName); + logger.info("BACKROLL: Trying to delete backup {} for vm {} using Backroll", vmId, backupName); boolean isBackupDeleted = false; BackrollTaskRequestResponse requestResponse = httpProvider.delete( @@ -260,7 +260,7 @@ public class BackrollClient { if (backupInfos != null && backupInfos.size() > 0) { for (BackupInfos infos : backupInfos) { var dateStart = new DateTime(infos.start); - backups.add(new Backup.RestorePoint(infos.id, dateStart.toDate(), "INCREMENTAL")); + backups.add(new Backup.RestorePoint(infos.id + "," + infos.name, dateStart.toDate(), "INCREMENTAL")); } } return backups; diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollHttpClientProvider.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollHttpClientProvider.java index 6214cac4a6e..e46b8737547 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollHttpClientProvider.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollHttpClientProvider.java @@ -69,10 +69,10 @@ public class BackrollHttpClientProvider { private Logger logger = LogManager.getLogger(BackrollClient.class); - public static BackrollHttpClientProvider createProvider(BackrollHttpClientProvider backrollHttpClientProvider, final String url, final String appname, final String password, + public static BackrollHttpClientProvider createProvider(final String url, final String appname, final String password, final boolean validateCertificate, final int timeout, final int restoreTimeout) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { - + BackrollHttpClientProvider backrollHttpClientProvider = new BackrollHttpClientProvider(); backrollHttpClientProvider.apiURI = new URI(url); backrollHttpClientProvider.appname = appname; backrollHttpClientProvider.password = password; @@ -229,6 +229,12 @@ public class BackrollHttpClientProvider { } public class NotOkBodyException extends Exception { + public NotOkBodyException() { + super(); + } + public NotOkBodyException(String errorMessage) { + super(errorMessage); + } } public String okBody(final CloseableHttpResponse response) throws NotOkBodyException { @@ -251,11 +257,14 @@ public class BackrollHttpClientProvider { } default: try { + HttpEntity bodyEntity2 = response.getEntity(); + result = EntityUtils.toString(bodyEntity2); closeConnection(response); } catch (IOException e) { e.printStackTrace(); } - throw new NotOkBodyException(); + + throw new NotOkBodyException("Error ok body, code: " + String.valueOf(response.getStatusLine().getStatusCode()) + " / body response: " + result); } } @@ -339,11 +348,11 @@ public class BackrollHttpClientProvider { public void loginIfAuthenticationFailed() throws BackrollApiException, IOException { if (!isAuthenticated()) { - login(appname, password); + login(); } } - protected void login(final String appname, final String appsecret) throws BackrollApiException, IOException { + protected void login() throws BackrollApiException, IOException { logger.debug("Backroll client - start login"); CloseableHttpClient httpClient = createHttpClient(); @@ -357,7 +366,7 @@ public class BackrollHttpClientProvider { try { jsonBody.put("app_id", appname); - jsonBody.put("app_secret", appsecret); + jsonBody.put("app_secret", password); params = new StringEntity(jsonBody.toString()); request.setEntity(params); @@ -368,7 +377,7 @@ public class BackrollHttpClientProvider { logger.info("BACKROLL: " + response); LoginApiResponse loginResponse = objectMapper.readValue(response, LoginApiResponse.class); logger.info("ok"); - this.backrollToken = loginResponse.accessToken; + backrollToken = loginResponse.accessToken; logger.debug("Backroll client - Token : {}", backrollToken); if (StringUtils.isEmpty(loginResponse.accessToken)) { diff --git a/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java index f26004ca77d..e95d7186d0c 100644 --- a/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java +++ b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java @@ -36,6 +36,7 @@ import org.apache.cloudstack.backup.backroll.model.BackrollVmBackup; import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; import org.apache.cloudstack.backup.backroll.model.BackrollOffering; import org.apache.cloudstack.backup.backroll.model.response.TaskState; +import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.Backup.RestorePoint; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.logging.log4j.Logger; @@ -145,7 +146,7 @@ public class BackrollBackupProviderTest { } @Test - public void getBackupMetrics_Test() throws BackrollApiException, IOException { + public void getBackupMetrics_Test() throws BackrollApiException, IOException{ VMInstanceVO vmInstanceVO = new VMInstanceVO(); vmInstanceVO.setInstanceName("test"); vmInstanceVO.setDataCenterId(1l); @@ -156,24 +157,18 @@ public class BackrollBackupProviderTest { vmInstanceVO2.setDataCenterId(2l); vmInstanceVO2.setBackupOfferingId(2l); + VMInstanceVO vmInstanceVO3 = new VMInstanceVO(); vmInstanceVO3.setInstanceName("test3"); vmInstanceVO3.setDataCenterId(3l); vmInstanceVO3.setBackupOfferingId(3l); - List backupsFromBackroll = Arrays.asList( - new BackrollVmBackup("OK", "OK", new Date())); + Metric metric = new Metric(2L, 3L); - BackrollBackupMetrics metrics = new BackrollBackupMetrics(2L, 3L); + Mockito.doReturn(metric).when(clientMock).getVirtualMachineMetrics(Mockito.anyString()); + assertEquals(backupProvider.getBackupMetrics(2L, Arrays.asList(vmInstanceVO, vmInstanceVO2, vmInstanceVO3)).size(), 1); - Mockito.doReturn(metrics).when(clientMock).getBackupMetrics(Mockito.anyString(), Mockito.anyString()); - Mockito.doReturn(backupsFromBackroll).when(clientMock).getAllBackupsfromVirtualMachine(Mockito.anyString()); - assertEquals( - backupProvider.getBackupMetrics(2L, Arrays.asList(vmInstanceVO, vmInstanceVO2, vmInstanceVO3)).size(), - 1); - - Mockito.verify(clientMock, times(3)).getAllBackupsfromVirtualMachine(Mockito.anyString()); - Mockito.verify(clientMock, times(3)).getBackupMetrics(Mockito.anyString(), Mockito.anyString()); + Mockito.verify(clientMock, times(3)).getVirtualMachineMetrics(Mockito.anyString()); } @Test