From 354355437185d2557ae02c9a91841252617effa4 Mon Sep 17 00:00:00 2001 From: Pierre Charton Date: Fri, 13 Dec 2024 09:35:37 +0100 Subject: [PATCH] Sync Backroll dev before merge backroll --- build_backroll_plugin.sh | 4 + plugins/backup/backroll/pom.xml | 12 +- .../backup/BackrollBackupProvider.java | 53 ++- .../backup/backroll/BackrollClient.java | 331 ++++---------- .../backup/backroll/BackrollService.java | 188 ++++++++ .../backup/BackrollBackupProviderTest.java | 419 ++++++++++++++++++ .../backup/backroll/BackrollClientTest.java | 162 +++++++ 7 files changed, 905 insertions(+), 264 deletions(-) create mode 100755 build_backroll_plugin.sh create mode 100644 plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollService.java create mode 100644 plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java create mode 100644 plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/backroll/BackrollClientTest.java diff --git a/build_backroll_plugin.sh b/build_backroll_plugin.sh new file mode 100755 index 00000000000..bc314fbf7e5 --- /dev/null +++ b/build_backroll_plugin.sh @@ -0,0 +1,4 @@ +mvn install -f "/opt/cloudstack/plugins/backup/backroll/pom.xml" -DskipTests +mvn install -f "/opt/cloudstack/plugins/pom.xml" -DskipTests +mvn install -f "/opt/cloudstack/client/pom.xml" -DskipTests +mvn -pl :cloud-client-ui jetty:run \ No newline at end of file diff --git a/plugins/backup/backroll/pom.xml b/plugins/backup/backroll/pom.xml index 5f7718c129f..37b2c1572ed 100644 --- a/plugins/backup/backroll/pom.xml +++ b/plugins/backup/backroll/pom.xml @@ -16,7 +16,9 @@ specific language governing permissions and limitations under the License. --> - + 4.0.0 cloud-plugin-backup-backroll Apache CloudStack Plugin - BackRoll Backup and Recovery Plugin by DIMSI @@ -41,5 +43,11 @@ org.json json + + com.github.tomakehurst + wiremock-standalone + ${cs.wiremock.version} + test + - + \ No newline at end of file diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java index 6b5cce53449..97af7e848df 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.backroll.BackrollClient; +import org.apache.cloudstack.backup.backroll.BackrollService; import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; import org.apache.cloudstack.backup.backroll.model.BackrollTaskStatus; import org.apache.cloudstack.backup.backroll.model.BackrollVmBackup; @@ -208,30 +209,35 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide logger.info("Starting backup for VM ID {} on backroll provider", vm.getUuid()); final BackrollClient client = getClient(vm.getDataCenterId()); - String idBackupTask; try { - idBackupTask = client.startBackupJob(vm.getUuid()); - } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + String urlToRequest = client.startBackupJob(vm.getUuid()); + + String backupJob = urlToRequest.replace("/status/", ""); + if (!StringUtils.isEmpty(backupJob)) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(backupJob); + backup.setType("INCREMENTAL"); + backup.setDate(new DateTime().toDate()); + backup.setSize(0L); + backup.setProtectedSize(0L); + backup.setStatus(Backup.Status.BackingUp); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + Boolean result = backupDao.persist(backup) != null; + client.triggerTaskStatus(urlToRequest); + syncBackups(vm, null); + return result; + } + } catch (KeyManagementException | NoSuchAlgorithmException | ParseException | InterruptedException | IOException e) { throw new CloudRuntimeException("Failed to take backup"); } - if (!StringUtils.isEmpty(idBackupTask)) { - BackupVO backup = new BackupVO(); - backup.setVmId(vm.getId()); - backup.setExternalId(idBackupTask); - backup.setType("INCREMENTAL"); - backup.setDate(new DateTime().toDate()); - backup.setSize(0L); - backup.setProtectedSize(0L); - backup.setStatus(Backup.Status.BackingUp); - backup.setBackupOfferingId(vm.getBackupOfferingId()); - backup.setAccountId(vm.getAccountId()); - backup.setDomainId(vm.getDomainId()); - backup.setZoneId(vm.getDataCenterId()); - return backupDao.persist(backup) != null; - } return false; } + @Override public void syncBackups(VirtualMachine vm, Backup.Metric metric) { logger.info("Starting sync backup for VM ID " + vm.getUuid() + " on backroll provider"); @@ -275,14 +281,13 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide BackrollBackupMetrics backupMetrics = null; try { backupMetrics = client.getBackupMetrics(vm.getUuid() , response.getInfo()); + if (backupMetrics != null) { + backupToUpdate.setSize(backupMetrics.getDeduplicated()); // real size + backupToUpdate.setProtectedSize(backupMetrics.getSize()); // total size + } } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { throw new CloudRuntimeException("Failed to get backup metrics"); } - - if (backupMetrics != null) { - backupToUpdate.setSize(backupMetrics.getDeduplicated()); // real size - backupToUpdate.setProtectedSize(backupMetrics.getSize()); // total size - } } else { backupToUpdate.setStatus(Backup.Status.BackingUp); } @@ -419,7 +424,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide try { if (backrollClient == null) { logger.debug("backroll client null - instanciation of new one "); - backrollClient = new BackrollClient(BackrollUrlConfigKey.valueIn(zoneId), BackrollAppNameConfigKey.valueIn(zoneId), BackrollPasswordConfigKey.valueIn(zoneId), true, 300, 600); + backrollClient = new BackrollClient(BackrollUrlConfigKey.valueIn(zoneId), BackrollAppNameConfigKey.valueIn(zoneId), BackrollPasswordConfigKey.valueIn(zoneId), true, 300, 600, new BackrollService()); } return backrollClient; } catch (URISyntaxException e) { diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java index 469a354a2f2..891927736d0 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.Backup.Metric; +import org.apache.cloudstack.backup.backroll.BackrollService.NotOkBodyException; import org.apache.cloudstack.backup.backroll.model.BackrollBackup; import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; import org.apache.cloudstack.backup.backroll.model.BackrollOffering; @@ -89,9 +90,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; + + public class BackrollClient { - private Logger logger = LogManager.getLogger(BackrollClient.class); + protected Logger logger = LogManager.getLogger(BackrollClient.class); private final URI apiURI; @@ -100,17 +103,19 @@ public class BackrollClient { private String password = null; private boolean validateCertificate = false; private RequestConfig config = null; + private BackrollService backrollService; private int restoreTimeout; public BackrollClient(final String url, final String appname, final String password, final boolean validateCertificate, final int timeout, - final int restoreTimeout) + final int restoreTimeout, final BackrollService backrollService) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { this.apiURI = new URI(url); this.restoreTimeout = restoreTimeout; this.appname = appname; this.password = password; + this.backrollService = backrollService; this.config = RequestConfig.custom() .setConnectTimeout(timeout * 1000) @@ -139,63 +144,13 @@ public class BackrollClient { closeableHttpResponse.close(); } - public String startBackupJob(final String jobId) throws KeyManagementException, NoSuchAlgorithmException, IOException { - logger.info("Trying to start backup for Backroll job: {}", jobId); - String backupJob = ""; - - loginIfAuthenticationFailed(); - - try { - CloseableHttpResponse response = post(String.format("/tasks/singlebackup/%s", jobId), null); - String result = okBody(response); + private String triggerBackrollTaskForVM(final String vmId, JSONObject jsonBody, final String task) throws InterruptedException, KeyManagementException, ParseException, NoSuchAlgorithmException, IOException, NotOkBodyException { + CloseableHttpResponse response = backrollService.post(apiURI,String.format("/tasks/%s/%s",task, vmId) ,jsonBody); + String result = backrollService.okBody(response); BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); response.close(); - backupJob = requestResponse.location.replace("/api/v1/status/", ""); - } catch (final Exception e) { - logger.error("Failed to start Backroll backup job due to: {}", e.getMessage()); - } - return StringUtils.isEmpty(backupJob) ? null : backupJob; - } - - public String getBackupOfferingUrl() throws KeyManagementException, NoSuchAlgorithmException, IOException { - logger.info("Trying to list backroll backup policies"); - - loginIfAuthenticationFailed(); - String url = ""; - - try { - CloseableHttpResponse response = get("/backup_policies"); - String result = okBody(response); - logger.info("BackrollClient:getBackupOfferingUrl:result: " + result); - //BackrollTaskRequestResponse requestResponse = parse(result); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - logger.info("BackrollClient:getBackupOfferingUrl:Apres PArse: " + requestResponse.location); - response.close(); - url = requestResponse.location.replace("/api/v1", ""); - } catch (final Exception e) { - logger.info("Failed to list Backroll jobs due to: {}", e.getMessage()); - logger.error("Failed to list Backroll jobs due to: {}", e.getMessage()); - } - return StringUtils.isEmpty(url) ? null : url; - } - - public List getBackupOfferings(String idTask) throws KeyManagementException, ParseException, NoSuchAlgorithmException, IOException { - logger.info("Trying to list backroll backup policies"); - - loginIfAuthenticationFailed(); - final List policies = new ArrayList<>(); - - try { - String results = waitGet(idTask); - BackupPoliciesResponse backupPoliciesResponse = new ObjectMapper().readValue(results, BackupPoliciesResponse.class); - - for (final BackrollBackupPolicyResponse policy : backupPoliciesResponse.backupPolicies) { - policies.add(new BackrollOffering(policy.name, policy.id)); - } - } catch (final IOException | InterruptedException e) { - logger.error("Failed to list Backroll jobs due to: {}", e.getMessage()); - } - return policies; + String urlToRequest = requestResponse.location.replace("/api/v1", ""); + return urlToRequest; } public boolean restoreVMFromBackup(final String vmId, final String backupName) throws KeyManagementException, ParseException, NoSuchAlgorithmException, IOException { @@ -216,12 +171,8 @@ public class BackrollClient { logger.error("Backroll Error: {}", e.getMessage()); } - CloseableHttpResponse response = post(String.format("/tasks/restore/%s", vmId), jsonBody); - String result = okBody(response); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - response.close(); - String urlToRequest = requestResponse.location.replace("/api/v1", ""); - result = waitGet(urlToRequest); + String urlToRequest = triggerBackrollTaskForVM(vmId, jsonBody, "restore"); + String result = backrollService.waitGet(apiURI,urlToRequest); if(result.contains("SUCCESS")) { logger.debug("RESTORE SUCCESS"); @@ -237,6 +188,65 @@ public class BackrollClient { return isRestoreOk; } + public void triggerTaskStatus(String urlToRequest) + throws IOException, InterruptedException, KeyManagementException, ParseException, NoSuchAlgorithmException{ + backrollService.waitGet(apiURI,urlToRequest); + } + + public String startBackupJob(final String vmId) throws KeyManagementException, NoSuchAlgorithmException, IOException { + logger.info("Trying to start backup for Backroll job: {}", vmId); + loginIfAuthenticationFailed(); + + try { + String urlToRequest = triggerBackrollTaskForVM(vmId, null, "singlebackup"); + return urlToRequest; + } catch (final Exception e) { + logger.error("Failed to start Backroll backup job due to: {}", e.getMessage()); + } + return null; + } + + public String getBackupOfferingUrl() throws KeyManagementException, NoSuchAlgorithmException, IOException { + logger.info("Trying to list backroll backup policies"); + + loginIfAuthenticationFailed(); + String url = ""; + + try { + CloseableHttpResponse response = backrollService.get(apiURI,"/backup_policies"); + String result = backrollService.okBody(response); + logger.info("BackrollClient:getBackupOfferingUrl:result: " + result); + //BackrollTaskRequestResponse requestResponse = parse(result); + BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); + logger.info("BackrollClient:getBackupOfferingUrl:Apres PArse: " + requestResponse.location); + response.close(); + url = requestResponse.location.replace("/api/v1", ""); + } catch (final Exception e) { + logger.info("Failed to list Backroll jobs due to: {}", e.getMessage()); + logger.error("Failed to list Backroll jobs due to: {}", e.getMessage()); + } + return StringUtils.isEmpty(url) ? null : url; + } + + public List getBackupOfferings(String idTask) throws KeyManagementException, ParseException, NoSuchAlgorithmException, IOException { + logger.info("Trying to list backroll backup policies"); + + loginIfAuthenticationFailed(); + final List policies = new ArrayList<>(); + + try { + String results = backrollService.waitGet(apiURI,idTask); + BackupPoliciesResponse backupPoliciesResponse = new ObjectMapper().readValue(results, BackupPoliciesResponse.class); + + for (final BackrollBackupPolicyResponse policy : backupPoliciesResponse.backupPolicies) { + policies.add(new BackrollOffering(policy.name, policy.id)); + } + } catch (final IOException | InterruptedException e) { + logger.error("Failed to list Backroll jobs due to: {}", e.getMessage()); + } + return policies; + } + public BackrollTaskStatus checkBackupTaskStatus(String taskId) throws KeyManagementException, ParseException, NoSuchAlgorithmException, IOException { logger.info("Trying to get backup status for Backroll task: {}", taskId); @@ -246,7 +256,7 @@ public class BackrollClient { try { - String body = okBody(get("/status/" + taskId)); + String body = backrollService.okBody(backrollService.get(apiURI,"/status/" + taskId)); if (body.contains(TaskState.FAILURE) || body.contains(TaskState.PENDING)) { BackrollBackupStatusResponse backupStatusRequestResponse = new ObjectMapper().readValue(body, BackrollBackupStatusResponse.class); @@ -277,13 +287,13 @@ public class BackrollClient { try { - CloseableHttpResponse response = delete(String.format("/virtualmachines/%s/backups/%s", vmId, backupName)); - String result = okBody(response); + CloseableHttpResponse response = backrollService.delete(apiURI,String.format("/virtualmachines/%s/backups/%s", vmId, backupName)); + String result = backrollService.okBody(response); BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); response.close(); String urlToRequest = requestResponse.location.replace("/api/v1", ""); - result = waitGet(urlToRequest); + result = backrollService.waitGet(apiURI,urlToRequest); BackrollBackupsFromVMResponse backrollBackupsFromVMResponse = new ObjectMapper().readValue(result, BackrollBackupsFromVMResponse.class); logger.debug(backrollBackupsFromVMResponse.state); isBackupDeleted = backrollBackupsFromVMResponse.state.equals(TaskState.SUCCESS); @@ -305,13 +315,13 @@ public class BackrollClient { try { - CloseableHttpResponse response = get(String.format("/virtualmachines/%s/repository", vmId)); - String result = okBody(response); + CloseableHttpResponse response = backrollService.get(apiURI,String.format("/virtualmachines/%s/repository", vmId)); + String result = backrollService.okBody(response); BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); response.close(); String urlToRequest = requestResponse.location.replace("/api/v1", ""); - result = waitGet(urlToRequest); + result = backrollService.waitGet(apiURI,urlToRequest); BackrollVmMetricsResponse vmMetricsResponse =new ObjectMapper().readValue(result, BackrollVmMetricsResponse.class); if (vmMetricsResponse != null && vmMetricsResponse.state.equals(TaskState.SUCCESS)) { @@ -342,15 +352,15 @@ public class BackrollClient { try { - CloseableHttpResponse response = get(String.format("/virtualmachines/%s/backups/%s", vmId, backupId)); - String result = okBody(response); + CloseableHttpResponse response = backrollService.get(apiURI,String.format("/virtualmachines/%s/backups/%s", vmId, backupId)); + String result = backrollService.okBody(response); BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); response.close(); String urlToRequest = requestResponse.location.replace("/api/v1", ""); logger.debug(urlToRequest); - result = waitGet(urlToRequest); + result = backrollService.waitGet(apiURI,urlToRequest); BackrollBackupMetricsResponse metricsResponse = new ObjectMapper().readValue(result, BackrollBackupMetricsResponse.class); if (metricsResponse.info != null) { metrics = new BackrollBackupMetrics(Long.parseLong(metricsResponse.info.originalSize), @@ -365,21 +375,21 @@ public class BackrollClient { } public List getAllBackupsfromVirtualMachine(String vmId) { - logger.info("Trying to retrieve all backups for vm {}", vmId); + //logger.info("Trying to retrieve all backups for vm {}", vmId); List backups = new ArrayList(); try { - CloseableHttpResponse response = get(String.format("/virtualmachines/%s/backups", vmId)); - String result = okBody(response); + CloseableHttpResponse response = backrollService.get(apiURI, String.format("/virtualmachines/%s/backups", vmId)); + String result = backrollService.okBody(response); BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); response.close(); String urlToRequest = requestResponse.location.replace("/api/v1", ""); - logger.debug(urlToRequest); + //logger.debug(urlToRequest); - result = waitGet(urlToRequest); + result = backrollService.waitGet(apiURI,urlToRequest); VirtualMachineBackupsResponse virtualMachineBackupsResponse = new ObjectMapper().readValue(result, VirtualMachineBackupsResponse.class); if (virtualMachineBackupsResponse.state.equals(TaskState.SUCCESS)) { @@ -397,114 +407,11 @@ public class BackrollClient { return backups; } - private CloseableHttpResponse post(final String path, final JSONObject json) throws IOException, KeyManagementException, NoSuchAlgorithmException { - CloseableHttpClient httpClient = createHttpClient(); - - String xml = null; - StringEntity params = null; - if (json != null) { - logger.debug("JSON {}", json.toString()); - params = new StringEntity(json.toString(), ContentType.APPLICATION_JSON); - } - - String url = apiURI.toString() + path; - final HttpPost request = new HttpPost(url); - - if (params != null) { - request.setEntity(params); - } - - request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); - request.setHeader("Content-type", "application/json"); - - final CloseableHttpResponse response = httpClient.execute(request); - - logger.debug("Response received in POST request with body {} is: {} for URL {}.", xml, response.toString(), url); - - return response; - } - - protected CloseableHttpResponse get(String path) throws IOException, KeyManagementException, NoSuchAlgorithmException { - CloseableHttpClient httpClient = createHttpClient(); - - String url = apiURI.toString() + path; - logger.debug("Backroll URL {}", url); - - HttpGet request = new HttpGet(url); - request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); - request.setHeader("Content-type", "application/json"); - CloseableHttpResponse response = httpClient.execute(request); - logger.debug("Response received in GET request is: {} for URL: {}.", response.toString(), url); - return response; - } - - protected CloseableHttpResponse delete(String path) throws IOException, KeyManagementException, NoSuchAlgorithmException { - CloseableHttpClient httpClient = createHttpClient(); - - String url = apiURI.toString() + path; - final HttpDelete request = new HttpDelete(url); - request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); - request.setHeader("Content-type", "application/json"); - final CloseableHttpResponse response = httpClient.execute(request); - logger.debug("Response received in GET request is: {} for URL: {}.", response.toString(), url); - return response; - } - private boolean isResponseAuthorized(final HttpResponse response) { return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK || response.getStatusLine().getStatusCode() == HttpStatus.SC_ACCEPTED; } - private class NotOkBodyException extends Exception { - } - - private String okBody(final CloseableHttpResponse response) - throws ParseException, IOException, NotOkBodyException { - String result = ""; - switch (response.getStatusLine().getStatusCode()) { - case HttpStatus.SC_OK: - case HttpStatus.SC_ACCEPTED: - HttpEntity bodyEntity = response.getEntity(); - try { - result = EntityUtils.toString(bodyEntity); - EntityUtils.consumeQuietly(bodyEntity); - closeConnection(response); - return result; - } finally { - EntityUtils.consumeQuietly(bodyEntity); - } - default: - closeConnection(response); - throw new NotOkBodyException(); - } - } - - private T parse(final String json) - throws JsonMappingException, JsonProcessingException { - return new ObjectMapper().readValue(json, new TypeReference() { - }); - } - - private String waitGet(String url) - throws IOException, InterruptedException, KeyManagementException, ParseException, NoSuchAlgorithmException { - // int threshold = 30; // 5 minutes - int maxAttempts = 12; // 2 minutes - - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - try { - String body = okBody(get(url)); - if (!body.contains(TaskState.PENDING)) { - return body; - } - } catch (final NotOkBodyException e) { - throw new CloudRuntimeException("An error occured with Backroll"); - } - TimeUnit.SECONDS.sleep(10); - } - - return null; - } - private boolean isAuthenticated() throws KeyManagementException, NoSuchAlgorithmException { boolean result = false; @@ -513,7 +420,7 @@ public class BackrollClient { } try { - CloseableHttpResponse response = post("/auth", null); + CloseableHttpResponse response = backrollService.post(apiURI,"/auth", null); result = isResponseAuthorized(response); EntityUtils.consumeQuietly(response.getEntity()); closeConnection(response); @@ -529,58 +436,6 @@ public class BackrollClient { } } - // private void login(final String appname, final String appsecret) throws IOException, KeyManagementException, NoSuchAlgorithmException { - // logger.info("Backroll client - start login"); - // CloseableHttpClient httpClient = createHttpClient(); - // CloseableHttpResponse response = null; - // final HttpPost request = new HttpPost(apiURI.toString() + "/login"); - // request.addHeader("content-type", "application/json"); - - // JSONObject jsonBody = new JSONObject(); - // StringEntity params; - - // try { - // jsonBody.put("app_id", appname); - // jsonBody.put("app_secret", appsecret); - // params = new StringEntity(jsonBody.toString()); - // request.setEntity(params); - - // response = httpClient.execute(request); - - // try { - // HttpEntity body = response.getEntity(); - // String bodyStr = EntityUtils.toString(body); - - // LoginApiResponse loginResponse = new ObjectMapper().readValue(bodyStr, LoginApiResponse.class); - // backrollToken = loginResponse.accessToken; - // logger.debug(String.format("Backroll client - Token : %s", backrollToken)); - - // EntityUtils.consumeQuietly(response.getEntity()); - - // if (StringUtils.isEmpty(loginResponse.accessToken)) { - // throw new CloudRuntimeException("Backroll token is not available to perform API requests"); - // } - // } catch (final Exception e) { - // EntityUtils.consumeQuietly(response.getEntity()); - // if (response.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { - // throw new CloudRuntimeException( - // "Failed to create and authenticate Backroll client, please check the settings."); - // } else { - // throw new ServerApiException(ApiErrorCode.UNAUTHORIZED, - // "Backroll API call unauthorized, please ask your administrator to fix integration issues."); - // } - // } - - // } catch (final IOException e) { - // throw new CloudRuntimeException("Failed to authenticate Backroll API service due to:" + e.getMessage()); - // } catch (JSONException e) { - // e.printStackTrace(); - // } - // finally { - // closeConnection(response); - // } - // logger.info("Backroll client - end login"); - // } private void login(final String appname, final String appsecret) throws IOException, KeyManagementException, NoSuchAlgorithmException { logger.debug("Backroll client - start login"); @@ -602,7 +457,7 @@ public class BackrollClient { response = httpClient.execute(request); try { - String toto = okBody(response); + String toto = backrollService.okBody(response); ObjectMapper objectMapper = new ObjectMapper(); logger.info("BACKROLL: " + toto); LoginApiResponse loginResponse = objectMapper.readValue(toto, LoginApiResponse.class); @@ -637,14 +492,14 @@ public class BackrollClient { try { logger.info("start to list Backroll backups for vm {}", vmId); - CloseableHttpResponse response = get("/virtualmachines/" + vmId + "/backups"); - String result = okBody(response); + CloseableHttpResponse response = backrollService.get(apiURI,"/virtualmachines/" + vmId + "/backups"); + String result = backrollService.okBody(response); BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); response.close(); String urlToRequest = requestResponse.location.replace("/api/v1", ""); logger.debug(urlToRequest); - result = waitGet(urlToRequest); + result = backrollService.waitGet(apiURI,urlToRequest); BackrollBackupsFromVMResponse backrollBackupsFromVMResponse = new ObjectMapper().readValue(result, BackrollBackupsFromVMResponse.class); final List backups = new ArrayList<>(); diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollService.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollService.java new file mode 100644 index 00000000000..67a3c40ac83 --- /dev/null +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollService.java @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.backroll; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509TrustManager; + +import org.apache.cloudstack.backup.backroll.model.response.TaskState; +import org.apache.cloudstack.utils.security.SSLUtils; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.ParseException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.json.JSONObject; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.nio.TrustAllManager; + +public class BackrollService { + + protected Logger logger = LogManager.getLogger(BackrollService.class); + + private String backrollToken = null; + private boolean validateCertificate = false; + private RequestConfig config = null; + + protected CloseableHttpClient createHttpClient() throws KeyManagementException, NoSuchAlgorithmException { + if(!validateCertificate) { + SSLContext sslContext = SSLUtils.getSSLContext(); + sslContext.init(null, new X509TrustManager[] { new TrustAllManager() }, new SecureRandom()); + final SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + return HttpClientBuilder.create() + .setDefaultRequestConfig(config) + .setSSLSocketFactory(factory) + .build(); + } else { + return HttpClientBuilder.create() + .setDefaultRequestConfig(config) + .build(); + } + } + + public void closeConnection(CloseableHttpResponse closeableHttpResponse) throws IOException { + closeableHttpResponse.close(); + } + + protected CloseableHttpResponse post(final URI apiURI, final String path, final JSONObject json) throws IOException, KeyManagementException, NoSuchAlgorithmException { + CloseableHttpClient httpClient = createHttpClient(); + + String xml = null; + StringEntity params = null; + if (json != null) { + logger.debug("JSON {}", json.toString()); + params = new StringEntity(json.toString(), ContentType.APPLICATION_JSON); + } + + String url = apiURI.toString() + path; + final HttpPost request = new HttpPost(url); + + if (params != null) { + request.setEntity(params); + } + + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); + request.setHeader("Content-type", "application/json"); + + final CloseableHttpResponse response = httpClient.execute(request); + + logger.debug("Response received in POST request with body {} is: {} for URL {}.", xml, response.toString(), url); + + return response; + } + + protected CloseableHttpResponse get(final URI apiURI, String path) throws IOException, KeyManagementException, NoSuchAlgorithmException { + CloseableHttpClient httpClient = createHttpClient(); + + String url = apiURI.toString() + path; + logger.debug("Backroll URL {}", url); + + HttpGet request = new HttpGet(url); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); + request.setHeader("Content-type", "application/json"); + CloseableHttpResponse response = httpClient.execute(request); + logger.debug("Response received in GET request is: {} for URL: {}.", response.toString(), url); + return response; + } + + protected CloseableHttpResponse delete(final URI apiURI, String path) throws IOException, KeyManagementException, NoSuchAlgorithmException { + CloseableHttpClient httpClient = createHttpClient(); + + String url = apiURI.toString() + path; + final HttpDelete request = new HttpDelete(url); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); + request.setHeader("Content-type", "application/json"); + final CloseableHttpResponse response = httpClient.execute(request); + logger.debug("Response received in GET request is: {} for URL: {}.", response.toString(), url); + return response; + } + + protected boolean isResponseAuthorized(final HttpResponse response) { + return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK + || response.getStatusLine().getStatusCode() == HttpStatus.SC_ACCEPTED; + } + + protected class NotOkBodyException extends Exception { + } + + protected String okBody(final CloseableHttpResponse response) + throws ParseException, IOException, NotOkBodyException { + String result = ""; + switch (response.getStatusLine().getStatusCode()) { + case HttpStatus.SC_OK: + case HttpStatus.SC_ACCEPTED: + HttpEntity bodyEntity = response.getEntity(); + try { + result = EntityUtils.toString(bodyEntity); + EntityUtils.consumeQuietly(bodyEntity); + closeConnection(response); + return result; + } finally { + EntityUtils.consumeQuietly(bodyEntity); + } + default: + closeConnection(response); + throw new NotOkBodyException(); + } + } + + protected String waitGet(final URI apiURI, String url) + throws IOException, InterruptedException, KeyManagementException, ParseException, NoSuchAlgorithmException { + // int threshold = 30; // 5 minutes + int maxAttempts = 12; // 2 minutes + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + String body = okBody(get(apiURI, url)); + if (!body.contains(TaskState.PENDING)) { + return body; + } + } catch (final NotOkBodyException e) { + throw new CloudRuntimeException("An error occured with Backroll"); + } + TimeUnit.SECONDS.sleep(10); + } + + return null; + } +} \ No newline at end of file diff --git a/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java new file mode 100644 index 00000000000..1dde11d8479 --- /dev/null +++ b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/BackrollBackupProviderTest.java @@ -0,0 +1,419 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.backup.Backup.Metric; +import org.apache.cloudstack.backup.backroll.BackrollClientTest; +import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; +import org.apache.cloudstack.backup.backroll.model.BackrollTaskStatus; +import org.apache.cloudstack.backup.backroll.model.BackrollVmBackup; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.ParseException; +import org.joda.time.DateTime; + +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; + +public class BackrollBackupProviderTest extends AdapterBase implements BackupProvider, Configurable { + + public static final String BACKUP_IDENTIFIER = "-CSBKP-"; + + public ConfigKey BackrollUrlConfigKey = new ConfigKey<>("Advanced", String.class, + "backup.plugin.backroll.config.url", + "http://api.backup.demo.ccc:5050/api/v1", + "Url for backroll plugin.", true, ConfigKey.Scope.Zone); + + public ConfigKey BackrollAppNameConfigKey = new ConfigKey<>("Advanced", String.class, + "backup.plugin.backroll.config.appname", + "backroll-api", + "App Name for backroll plugin.", true, ConfigKey.Scope.Zone); + + public ConfigKey BackrollPasswordConfigKey = new ConfigKey<>("Advanced", String.class, + "backup.plugin.backroll.config.password", + "VviX8dALauSyYJMqVYJqf3UyZOpO3joS", + "Password for backroll plugin.", true, ConfigKey.Scope.Zone); + + private BackrollClientTest backrollClient; + + @Inject + private BackupDao backupDao; + @Inject + private VMInstanceDao vmInstanceDao; + + @Override + public String getName() { + return "backroll"; + } + + @Override + public String getDescription() { + return "Backroll Backup Plugin"; + } + + @Override + public List listBackupOfferings(Long zoneId) { + logger.debug("Listing backup policies on backroll B&R Plugin"); + logger.info("Listing backup policies on backroll B&R Plugin"); + BackrollClientTest client = getClient(zoneId); + try{ + String urlToRequest = client.getBackupOfferingUrl(); + logger.info("BackrollProvider: urlToRequest: " + urlToRequest); + if (!StringUtils.isEmpty(urlToRequest)){ + List results = new ArrayList(); + // return client.getBackupOfferings(urlToRequest); + results = client.getBackupOfferings(urlToRequest); + if(results.size()>0) { + logger.info("BackrollProvider: results > 0"); + } else { + logger.info("BackrollProvider: results <= 0"); + } + return results; + } + } catch (KeyManagementException | ParseException | NoSuchAlgorithmException | IOException e) { + logger.info("BackrollProvider: catch erreur!!!!!!!!!!!!!!!!!!!!!!!!"); + throw new CloudRuntimeException("Failed to load backup offerings"); + } + return new ArrayList(); + } + + @Override + public boolean isValidProviderOffering(Long zoneId, String uuid) { + logger.info("Checking if backup offering exists on the Backroll Backup Provider"); + return true; + } + + @Override + public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { + logger.info("Creating VM backup for VM {} from backup offering {}", vm.getInstanceName(), backupOffering.getName()); + ((VMInstanceVO) vm).setBackupExternalId(backupOffering.getUuid()); + return true; + } + + @Override + public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { + logger.debug("Restoring vm {} from backup {} on the backroll Backup Provider", vm.getUuid(), backup.getUuid()); + boolean isSuccess; + try { + isSuccess = getClient(vm.getDataCenterId()).restoreVMFromBackup(vm.getUuid(), getBackupName(backup)); + } catch (KeyManagementException | ParseException | NoSuchAlgorithmException | IOException e) { + throw new CloudRuntimeException("Failed to restore VM from Backrup"); + } + return isSuccess; + } + + @Override + public Map getBackupMetrics(Long zoneId, List vms) { + final Map metrics = new HashMap<>(); + if (CollectionUtils.isEmpty(vms)) { + logger.warn("Unable to get VM Backup Metrics because the list of VMs is empty."); + return metrics; + } + + List vmUuids = vms.stream().filter(Objects::nonNull).map(VirtualMachine::getUuid).collect(Collectors.toList()); + logger.debug("Get Backup Metrics for VMs: {}.", String.join(", ", vmUuids)); + + for (final VirtualMachine vm : vms) { + if (vm == null) { + continue; + } + + Metric metric; + try { + metric = getClient(zoneId).getVirtualMachineMetrics(vm.getUuid()); + } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + throw new CloudRuntimeException("Failed to retrieve backup metrics"); + } + logger.debug("Metrics for VM [uuid: {}, name: {}] is [backup size: {}, data size: {}].", vm.getUuid(), + vm.getInstanceName(), metric.getBackupSize(), metric.getDataSize()); + metrics.put(vm, metric); + } + return metrics; + } + + @Override + public boolean removeVMFromBackupOffering(VirtualMachine vm) { + return true; + } + + @Override + public boolean willDeleteBackupsOnOfferingRemoval() { + return false; + } + + @Override + public boolean takeBackup(VirtualMachine vm) { + logger.info("Starting backup for VM ID {} on backroll provider", vm.getUuid()); + final BackrollClientTest client = getClient(vm.getDataCenterId()); + + try { + String urlToRequest = client.startBackupJob(vm.getUuid()); + + String backupJob = urlToRequest.replace("/status/", ""); + if (!StringUtils.isEmpty(backupJob)) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(backupJob); + backup.setType("INCREMENTAL"); + backup.setDate(new DateTime().toDate()); + backup.setSize(0L); + backup.setProtectedSize(0L); + backup.setStatus(Backup.Status.BackingUp); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + Boolean result = backupDao.persist(backup) != null; + client.triggerTaskStatus(urlToRequest); + syncBackups(vm, null); + return result; + } + } catch (KeyManagementException | NoSuchAlgorithmException | ParseException | InterruptedException | IOException e) { + throw new CloudRuntimeException("Failed to take backup"); + } + return false; + } + + + @Override + public void syncBackups(VirtualMachine vm, Backup.Metric metric) { + logger.info("Starting sync backup for VM ID " + vm.getUuid() + " on backroll provider"); + + final BackrollClientTest client = getClient(vm.getDataCenterId()); + List backupsInDb = backupDao.listByVmId(null, vm.getId()); + + for (Backup backup : backupsInDb) { + if (backup.getStatus().equals(Backup.Status.BackingUp)) { + BackrollTaskStatus response; + try { + response = client.checkBackupTaskStatus(backup.getExternalId()); + } catch (KeyManagementException | ParseException | NoSuchAlgorithmException | IOException e) { + throw new CloudRuntimeException("Failed to sync backups"); + } + + if (response != null) { + logger.debug("backroll backup id: {}", backup.getExternalId()); + logger.debug("backroll backup status: {}", response.getState()); + + BackupVO backupToUpdate = new BackupVO(); + backupToUpdate.setVmId(backup.getVmId()); + backupToUpdate.setExternalId(backup.getExternalId()); + backupToUpdate.setType(backup.getType()); + backupToUpdate.setDate(backup.getDate()); + backupToUpdate.setSize(backup.getSize()); + backupToUpdate.setProtectedSize(backup.getProtectedSize()); + backupToUpdate.setBackupOfferingId(vm.getBackupOfferingId()); + backupToUpdate.setAccountId(backup.getAccountId()); + backupToUpdate.setDomainId(backup.getDomainId()); + backupToUpdate.setZoneId(backup.getZoneId()); + + if (response.getState().equals("PENDING")) { + backupToUpdate.setStatus(Backup.Status.BackingUp); + } else if (response.getState().equals("FAILURE")) { + backupToUpdate.setStatus(Backup.Status.Failed); + } else if (response.getState().equals("SUCCESS")) { + backupToUpdate.setStatus(Backup.Status.BackedUp); + backupToUpdate.setExternalId(backup.getExternalId() + "," + response.getInfo()); + + BackrollBackupMetrics backupMetrics = null; + try { + backupMetrics = client.getBackupMetrics(vm.getUuid() , response.getInfo()); + if (backupMetrics != null) { + backupToUpdate.setSize(backupMetrics.getDeduplicated()); // real size + backupToUpdate.setProtectedSize(backupMetrics.getSize()); // total size + } + } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + throw new CloudRuntimeException("Failed to get backup metrics"); + } + } else { + backupToUpdate.setStatus(Backup.Status.BackingUp); + } + + if (backupDao.persist(backupToUpdate) != null) { + logger.info("Backroll mise à jour enregistrée"); + backupDao.remove(backup.getId()); + } + } + } else if (backup.getStatus().equals(Backup.Status.BackedUp) && backup.getSize().equals(0L)) { + String backupId = backup.getExternalId().contains(",") ? backup.getExternalId().split(",")[1] : backup.getExternalId(); + BackrollBackupMetrics backupMetrics; + try { + backupMetrics = client.getBackupMetrics(vm.getUuid() , backupId); + } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + throw new CloudRuntimeException("Failed to get backup metrics"); + } + + if (backupMetrics != null) { + BackupVO backupToUpdate = ((BackupVO) backup); + backupToUpdate.setSize(backupMetrics.getDeduplicated()); // real size + backupToUpdate.setProtectedSize(backupMetrics.getSize()); // total size + backupDao.persist(backupToUpdate); + } + } + } + + // Backups synchronisation between Backroll ad CS Db + List backupsFromBackroll = client.getAllBackupsfromVirtualMachine(vm.getUuid()); + backupsInDb = backupDao.listByVmId(null, vm.getId()); + + // insert new backroll backup in CS + for (BackrollVmBackup backupInBackroll : backupsFromBackroll) { + Backup backupToFind = backupsInDb.stream() + .filter(backupInDb -> backupInDb.getExternalId().contains(backupInBackroll.getName())) + .findAny() + .orElse(null); + + if (backupToFind == null) { + BackupVO backupToInsert = new BackupVO(); + backupToInsert.setVmId(vm.getId()); + backupToInsert.setExternalId(backupInBackroll.getId() + "," + backupInBackroll.getName()); + backupToInsert.setType("INCREMENTAL"); + backupToInsert.setDate(backupInBackroll.getDate()); + backupToInsert.setSize(0L); + backupToInsert.setProtectedSize(0L); + backupToInsert.setStatus(Backup.Status.BackedUp); + backupToInsert.setBackupOfferingId(vm.getBackupOfferingId()); + backupToInsert.setAccountId(vm.getAccountId()); + backupToInsert.setDomainId(vm.getDomainId()); + backupToInsert.setZoneId(vm.getDataCenterId()); + backupDao.persist(backupToInsert); + } + if (backupToFind != null && backupToFind.getStatus() == Backup.Status.Removed) { + BackupVO backupToUpdate = ((BackupVO) backupToFind); + backupToUpdate.setStatus(Backup.Status.BackedUp); + if (backupDao.persist(backupToUpdate) != null) { + logger.info("Backroll update saved"); + backupDao.remove(backupToFind.getId()); + } + } + } + + // delete deleted backroll backup in CS + backupsInDb = backupDao.listByVmId(null, vm.getId()); + for (Backup backup : backupsInDb) { + String backupName = backup.getExternalId().contains(",") ? backup.getExternalId().split(",")[1] : backup.getExternalId(); + BackrollVmBackup backupToFind = backupsFromBackroll.stream() + .filter(backupInBackroll -> backupInBackroll.getName().contains(backupName)) + .findAny() + .orElse(null); + + if (backupToFind == null) { + BackupVO backupToUpdate = ((BackupVO) backup); + backupToUpdate.setStatus(Backup.Status.Removed); + if (backupDao.persist(backupToUpdate) != null) { + logger.debug("Backroll delete saved (sync)"); + } + } + } + } + + @Override + public String getConfigComponentName() { + return BackupService.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + BackrollUrlConfigKey, + BackrollAppNameConfigKey, + BackrollPasswordConfigKey + }; + } + + @Override + public boolean deleteBackup(Backup backup, boolean forced) { + logger.info("backroll delete backup id: {}", backup.getExternalId()); + if (backup.getStatus().equals(Backup.Status.BackingUp)) { + throw new CloudRuntimeException("You can't delete a backup while it still BackingUp"); + } else { + logger.debug("backroll - try delete backup"); + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); + + if (backup.getStatus().equals(Backup.Status.Removed) || backup.getStatus().equals(Backup.Status.Failed)){ + return deleteBackupInDb(backup); + } else { + try { + if (getClient(backup.getZoneId()).deleteBackup(vm.getUuid(), getBackupName(backup))) { + logger.debug("Backup deletion for backup {} complete on backroll side.", backup.getUuid()); + return deleteBackupInDb(backup); + } + } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + throw new CloudRuntimeException("Failed to delete backup"); + } + } + } + return false; + } + + private boolean deleteBackupInDb(Backup backup) { + BackupVO backupToUpdate = ((BackupVO) backup); + backupToUpdate.setStatus(Backup.Status.Removed); + if (backupDao.persist(backupToUpdate) != null) { + logger.debug("Backroll backup {} deleted in database.", backup.getUuid()); + return true; + } + return false; + } + + protected BackrollClientTest getClient(final Long zoneId) { + logger.debug("Backroll Provider GetClient with zone id {}", zoneId); + try { + if (backrollClient == null) { + logger.debug("backroll client null - instanciation of new one "); + backrollClient = new BackrollClientTest(BackrollUrlConfigKey.valueIn(zoneId), BackrollAppNameConfigKey.valueIn(zoneId), BackrollPasswordConfigKey.valueIn(zoneId), true, 300, 600); + } + return backrollClient; + } catch (URISyntaxException e) { + throw new CloudRuntimeException("Failed to parse Backroll API URL: " + e.getMessage()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + logger.info("Failed to build Backroll API client due to: ", e); + } + throw new CloudRuntimeException("Failed to build Backroll API client"); + } + + private String getBackupName(Backup backup) { + return backup.getExternalId().substring(backup.getExternalId().indexOf(",") + 1); + } + + @Override + public Pair restoreBackedUpVolume(Backup backup, String volumeUuid, String hostIp, String dataStoreUuid, Pair vmNameAndState) { + logger.debug("Restoring volume {} from backup {} on the Backroll Backup Provider", volumeUuid, backup.getUuid()); + throw new CloudRuntimeException("Backroll plugin does not support this feature"); + } +} diff --git a/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/backroll/BackrollClientTest.java b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/backroll/BackrollClientTest.java new file mode 100644 index 00000000000..437277adffe --- /dev/null +++ b/plugins/backup/backroll/src/test/java/org/apache/cloudstack/backup/backroll/BackrollClientTest.java @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.backroll; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +import org.apache.cloudstack.backup.BackupService; +import org.apache.cloudstack.backup.backroll.model.BackrollVmBackup; + +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.util.EntityUtils; + +import org.apache.logging.log4j.Logger; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.github.tomakehurst.wiremock.junit.WireMockRule; + +public class BackrollClientTest { + private BackrollClient client; + + @Mock + private BackrollService mockBackrollService; + + + @Rule + public WireMockRule wireMockRule = new WireMockRule(9399); + + @Before + public void setUp() throws Exception { + mockBackrollService = mock(BackrollService.class); + mockBackrollService.logger = Mockito.mock(Logger.class); + client = new BackrollClient("http://localhost:5050/api/v1/", "backroll", "password", true, 300, 600, mockBackrollService); + } + + @Test + public void getAllBackupsfromVirtualMachine_test() throws Exception { + String vmId = "TEST-vm_uuid"; + CloseableHttpResponse mockHttpResponse = mock(CloseableHttpResponse.class); + CloseableHttpResponse mockHttpResponse2 = mock(CloseableHttpResponse.class); + + // Define mock behavior for first HTTP response + String responseContent1 = "{\"Location\":\"/api/v1/status/f32092e4-3e8a-461b-8733-ed93e23fa782\"}"; + + // Define mock behavior for second HTTP response + String responseContent2 = "{ \"state\": \"SUCCESS\", \"info\": { \"archives\": [ { \"archive\": \"ROOT-00000\", \"barchive\": \"ROOT-00000\", \"id\": \"25d55ad283aa400af464c76d713c07ad7d163abdd3b8fbcdbdc46b827e5e0457\", \"name\": \"ROOT-00000\", \"start\": \"2024-11-08T18:24:48.000000\", \"time\": \"2024-11-08T18:24:48.000000\" } ], \"encryption\": { \"mode\": \"none\" }, \"repository\": { \"id\": \"36a11ebc0775a097c927735cc7015d19be7309be69fc15b896c5b1fd87fcbd79\", \"last_modified\": \"2024-11-29T09:53:09.000000\", \"location\": \"/mnt/backup/backup1\" } } }"; + + // Mocking client responses + doReturn(mockHttpResponse).when(mockBackrollService).get(Mockito.any(URI.class), Mockito.matches(".*/virtualmachines/.*")); + doReturn(responseContent1).when(mockBackrollService).okBody(mockHttpResponse); + doReturn(responseContent2).when(mockBackrollService).waitGet(Mockito.any(URI.class), Mockito.anyString()); + doReturn(mockHttpResponse2).when(mockBackrollService).get(Mockito.any(URI.class), Mockito.matches(".*/status/.*")); + + // Assuming getAllBackupsfromVirtualMachine belongs to a class named BackupService + + // Run the method under test + List backupsTestList = client.getAllBackupsfromVirtualMachine(vmId); + + // Check results + assertEquals(1, backupsTestList.size()); // Should be 1 based on provided mock data + + // Optional: Verifications + verify(mockHttpResponse).close(); + } + + // @Test + // public void getAllBackupsfromVirtualMachine_test2() throws Exception { + // String vmId = "TEST-vm_uuid"; + // CloseableHttpResponse mockHttpResponse = mock(CloseableHttpResponse.class); + // CloseableHttpResponse mockHttpResponse2 = mock(CloseableHttpResponse.class); + // StatusLine mockStatusLine = mock(StatusLine.class); + // HttpEntity mockEntity = mock(HttpEntity.class); + // HttpEntity mockEntity2 = mock(HttpEntity.class); + // //InputStream mockInputStream = new ByteArrayInputStream("{\"Location\":\"/api/v1/status/f32092e4-3e8a-461b-8733-ed93e23fa782\"}".getBytes()); + + // // Define mock behavior + // when(mockHttpResponse.getStatusLine()).thenReturn(mockStatusLine); + // when(mockStatusLine.getStatusCode()).thenReturn(200); + // when(mockHttpResponse.getEntity()).thenReturn(mockEntity); + // when(mockEntity.getContentLength()).thenReturn(400L); + + // String responseContent = "{\"Location\":\"/api/v1/status/f32092e4-3e8a-461b-8733-ed93e23fa782\"}"; + // InputStream stream = new ByteArrayInputStream(responseContent.getBytes()); + // when(mockEntity.getContent()).thenReturn(stream); + + // when(mockHttpResponse2.getStatusLine()).thenReturn(mockStatusLine); + // when(mockStatusLine.getStatusCode()).thenReturn(200); + // when(mockHttpResponse2.getEntity()).thenReturn(mockEntity2); + // when(mockEntity2.getContentLength()).thenReturn(400L); + + // String responseContent2 = "{\r\n" + // + // " \"state\": \"SUCCESS\",\r\n" + // + // " \"info\": {\r\n" + // + // " \"archives\": [\r\n" + // + // " {\r\n" + // + // " \"archive\": \"ROOT-00000\",\r\n" + // + // " \"barchive\": \"ROOT-00000\",\r\n" + // + // " \"id\": \"25d55ad283aa400af464c76d713c07ad7d163abdd3b8fbcdbdc46b827e5e0457\",\r\n" + // + // " \"name\": \"ROOT-00000\",\r\n" + // + // " \"start\": \"2024-11-08T18:24:48.000000\",\r\n" + // + // " \"time\": \"2024-11-08T18:24:48.000000\"\r\n" + // + // " }\r\n" + // + // " ],\r\n" + // + // " \"encryption\": {\r\n" + // + // " \"mode\": \"none\"\r\n" + // + // " },\r\n" + // + // " \"repository\": {\r\n" + // + // " \"id\": \"36a11ebc0775a097c927735cc7015d19be7309be69fc15b896c5b1fd87fcbd79\",\r\n" + // + // " \"last_modified\": \"2024-11-29T09:53:09.000000\",\r\n" + // + // " \"location\": \"/mnt/backup/backup1\"\r\n" + // + // " }\r\n" + // + // " }\r\n" + // + // "}"; + // InputStream stream2 = new ByteArrayInputStream(responseContent2.getBytes()); + // when(mockEntity2.getContent()).thenReturn(stream2); + + // try { + // Mockito.doReturn(mockHttpResponse).when(mockClient).get(Mockito.matches(".*/virtualmachines/.*")); + // Mockito.doReturn(mockHttpResponse2).when(mockClient).get(Mockito.matches(".*/api/v1.*")); + // List backupsTesList = mockClient.getAllBackupsfromVirtualMachine(vmId); + // Assert.assertTrue(backupsTesList.size() > 0); + // fail(); + // } catch (Exception e) { + // Assert.assertEquals(CloudRuntimeException.class, e.getClass()); + // Assert.assertEquals("Failed to get Repository Name from Job [name: TEST-BACKUP].", e.getMessage()); + // } + // } +} \ No newline at end of file