From c3f0d14d31583fdf2570de2633db5c87a0f08fa4 Mon Sep 17 00:00:00 2001 From: Wido den Hollander Date: Thu, 5 Sep 2024 14:28:39 +0200 Subject: [PATCH] storage/object: Add support for Ceph RGW Object Store (#8389) This feature adds support for Ceph's RADOS Gateway (RGW) support for the Object Store feature of CloudStack. The RGW of Ceph is Amazon S3 compliant and is therefor an easy and straigforward implementation of basic S3 features. Existing Ceph environments can have the RGW added as an additional feature to a cluster already providing RBD (Block Device) to a CloudStack environment. Introduce the BucketTO to pass to the drivers. This replaces just passing the bucket's name. Some upcoming drivers require more information then just the bucket name to perform their actions, for example they require the access and secret key which belong to the account of this bucket. This is leftover code from a long time ago and this validation test has nu influence on the end result on how a URL will be used afterwards. We should support hosts pointing to an IPv6(-only) address out of the box. For the code it does not matter if it's IPv4 or IPv6. This is the admin's choice. Signed-off-by: Rohit Yadav Co-authored-by: Rohit Yadav --- .../java/com/cloud/agent/api/to/BucketTO.java | 50 +++ client/pom.xml | 5 + .../storage/object/ObjectStoreEntity.java | 15 +- .../main/java/com/cloud/storage/BucketVO.java | 14 +- .../storage/object/store/ObjectStoreImpl.java | 29 +- .../storage/object/ObjectStoreDriver.java | 23 +- plugins/pom.xml | 1 + plugins/storage/object/ceph/pom.xml | 52 +++ .../driver/CephObjectStoreDriverImpl.java | 362 ++++++++++++++++++ .../CephObjectStoreLifeCycleImpl.java | 133 +++++++ .../provider/CephObjectStoreProviderImpl.java | 85 ++++ .../storage-object-ceph/module.properties | 18 + .../spring-storage-object-ceph-context.xml | 31 ++ .../driver/CephObjectStoreDriverImplTest.java | 112 ++++++ .../CephObjectStoreProviderImplTest.java | 50 +++ .../driver/MinIOObjectStoreDriverImpl.java | 35 +- .../MinIOObjectStoreDriverImplTest.java | 4 +- .../SimulatorObjectStoreDriverImpl.java | 23 +- .../com/cloud/storage/StorageManagerImpl.java | 5 +- .../storage/object/BucketApiServiceImpl.java | 30 +- ui/src/views/infra/AddObjectStorage.vue | 2 +- .../main/java/com/cloud/utils/UriUtils.java | 2 +- .../java/com/cloud/utils/UriUtilsTest.java | 9 + 23 files changed, 1008 insertions(+), 82 deletions(-) create mode 100644 api/src/main/java/com/cloud/agent/api/to/BucketTO.java create mode 100644 plugins/storage/object/ceph/pom.xml create mode 100644 plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java create mode 100644 plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java create mode 100644 plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImpl.java create mode 100644 plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/module.properties create mode 100644 plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/spring-storage-object-ceph-context.xml create mode 100644 plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java create mode 100644 plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImplTest.java diff --git a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java new file mode 100644 index 00000000000..f7e4bfea80f --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +import org.apache.cloudstack.storage.object.Bucket; + +public final class BucketTO { + + private String name; + + private String accessKey; + + private String secretKey; + + public BucketTO(Bucket bucket) { + this.name = bucket.getName(); + this.accessKey = bucket.getAccessKey(); + this.secretKey = bucket.getSecretKey(); + } + + public BucketTO(String name) { + this.name = name; + } + + public String getName() { + return this.name; + } + + public String getAccessKey() { + return this.accessKey; + } + + public String getSecretKey() { + return this.secretKey; + } +} diff --git a/client/pom.xml b/client/pom.xml index adf25dcbd99..473b711b9e3 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -648,6 +648,11 @@ cloud-plugin-storage-object-minio ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-object-ceph + ${project.version} + org.apache.cloudstack cloud-plugin-storage-object-simulator diff --git a/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java index 9ee94b083cf..7efb72d23b2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java +++ b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.storage.object; +import com.cloud.agent.api.to.BucketTO; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import java.util.List; @@ -30,19 +31,19 @@ public interface ObjectStoreEntity extends DataStore, ObjectStore { boolean createUser(long accountId); - boolean deleteBucket(String name); + boolean deleteBucket(BucketTO bucket); - boolean setBucketEncryption(String name); + boolean setBucketEncryption(BucketTO bucket); - boolean deleteBucketEncryption(String name); + boolean deleteBucketEncryption(BucketTO bucket); - boolean setBucketVersioning(String name); + boolean setBucketVersioning(BucketTO bucket); - boolean deleteBucketVersioning(String name); + boolean deleteBucketVersioning(BucketTO bucket); - void setBucketPolicy(String name, String policy); + void setBucketPolicy(BucketTO bucket, String policy); - void setQuota(String name, int quota); + void setQuota(BucketTO bucket, int quota); Map getAllBucketsUsage(); } diff --git a/engine/schema/src/main/java/com/cloud/storage/BucketVO.java b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java index 181b02e5a1b..53017447c07 100644 --- a/engine/schema/src/main/java/com/cloud/storage/BucketVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java @@ -97,17 +97,23 @@ public class BucketVO implements Bucket { String uuid; public BucketVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public BucketVO(String name) { + this.uuid = UUID.randomUUID().toString(); + this.name = name; + this.state = State.Allocated; } public BucketVO(long accountId, long domainId, long objectStoreId, String name, Integer quota, boolean versioning, - boolean encryption, boolean objectLock, String policy) - { + boolean encryption, boolean objectLock, String policy) { this.accountId = accountId; this.domainId = domainId; this.objectStoreId = objectStoreId; this.name = name; - state = State.Allocated; - uuid = UUID.randomUUID().toString(); + this.state = State.Allocated; + this.uuid = UUID.randomUUID().toString(); this.quota = quota; this.versioning = versioning; this.encryption = encryption; diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java index 3c525ba9364..f1c27526f52 100644 --- a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.storage.object.store; +import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.storage.DataStoreRole; @@ -107,38 +108,38 @@ public class ObjectStoreImpl implements ObjectStoreEntity { } @Override - public boolean deleteBucket(String bucketName) { - return driver.deleteBucket(bucketName, objectStoreVO.getId()); + public boolean deleteBucket(BucketTO bucket) { + return driver.deleteBucket(bucket, objectStoreVO.getId()); } @Override - public boolean setBucketEncryption(String bucketName) { - return driver.setBucketEncryption(bucketName, objectStoreVO.getId()); + public boolean setBucketEncryption(BucketTO bucket) { + return driver.setBucketEncryption(bucket, objectStoreVO.getId()); } @Override - public boolean deleteBucketEncryption(String bucketName) { - return driver.deleteBucketEncryption(bucketName, objectStoreVO.getId()); + public boolean deleteBucketEncryption(BucketTO bucket) { + return driver.deleteBucketEncryption(bucket, objectStoreVO.getId()); } @Override - public boolean setBucketVersioning(String bucketName) { - return driver.setBucketVersioning(bucketName, objectStoreVO.getId()); + public boolean setBucketVersioning(BucketTO bucket) { + return driver.setBucketVersioning(bucket, objectStoreVO.getId()); } @Override - public boolean deleteBucketVersioning(String bucketName) { - return driver.deleteBucketVersioning(bucketName, objectStoreVO.getId()); + public boolean deleteBucketVersioning(BucketTO bucket) { + return driver.deleteBucketVersioning(bucket, objectStoreVO.getId()); } @Override - public void setBucketPolicy(String bucketName, String policy) { - driver.setBucketPolicy(bucketName, policy, objectStoreVO.getId()); + public void setBucketPolicy(BucketTO bucket, String policy) { + driver.setBucketPolicy(bucket, policy, objectStoreVO.getId()); } @Override - public void setQuota(String bucketName, int quota) { - driver.setBucketQuota(bucketName, objectStoreVO.getId(), quota); + public void setQuota(BucketTO bucket, int quota) { + driver.setBucketQuota(bucket, objectStoreVO.getId(), quota); } @Override diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java index 4953b9b0cdf..13aaf7c002e 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.object; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.BucketTO; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; import java.util.List; @@ -30,30 +31,30 @@ public interface ObjectStoreDriver extends DataStoreDriver { List listBuckets(long storeId); - boolean deleteBucket(String bucketName, long storeId); + boolean deleteBucket(BucketTO bucket, long storeId); - AccessControlList getBucketAcl(String bucketName, long storeId); + AccessControlList getBucketAcl(BucketTO bucket, long storeId); - void setBucketAcl(String bucketName, AccessControlList acl, long storeId); + void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId); - void setBucketPolicy(String bucketName, String policyType, long storeId); + void setBucketPolicy(BucketTO bucket, String policyType, long storeId); - BucketPolicy getBucketPolicy(String bucketName, long storeId); + BucketPolicy getBucketPolicy(BucketTO bucket, long storeId); - void deleteBucketPolicy(String bucketName, long storeId); + void deleteBucketPolicy(BucketTO bucket, long storeId); boolean createUser(long accountId, long storeId); - boolean setBucketEncryption(String bucketName, long storeId); + boolean setBucketEncryption(BucketTO bucket, long storeId); - boolean deleteBucketEncryption(String bucketName, long storeId); + boolean deleteBucketEncryption(BucketTO bucket, long storeId); - boolean setBucketVersioning(String bucketName, long storeId); + boolean setBucketVersioning(BucketTO bucket, long storeId); - boolean deleteBucketVersioning(String bucketName, long storeId); + boolean deleteBucketVersioning(BucketTO bucket, long storeId); - void setBucketQuota(String bucketName, long storeId, long size); + void setBucketQuota(BucketTO bucket, long storeId, long size); Map getAllBucketsUsage(long storeId); } diff --git a/plugins/pom.xml b/plugins/pom.xml index 233fd5ad26d..92fe7951649 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -138,6 +138,7 @@ storage/volume/flasharray storage/volume/primera storage/object/minio + storage/object/ceph storage/object/simulator diff --git a/plugins/storage/object/ceph/pom.xml b/plugins/storage/object/ceph/pom.xml new file mode 100644 index 00000000000..43b3a15731d --- /dev/null +++ b/plugins/storage/object/ceph/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + cloud-plugin-storage-object-ceph + Apache CloudStack Plugin - Ceph RGW object storage provider + + org.apache.cloudstack + cloudstack-plugins + 4.20.0.0-SNAPSHOT + ../../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + io.github.twonote + radosgw-admin4j + 2.0.9 + + + diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java new file mode 100644 index 00000000000..6fece40e6ac --- /dev/null +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -0,0 +1,362 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.amazonaws.services.s3.model.BucketVersioningConfiguration; +import com.amazonaws.services.s3.model.DeleteBucketPolicyRequest; +import com.amazonaws.services.s3.model.SetBucketPolicyRequest; +import com.amazonaws.services.s3.model.GetBucketPolicyRequest; +import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest; +import com.cloud.agent.api.to.BucketTO; +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.cloudstack.storage.object.BucketObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.twonote.rgwadmin4j.RgwAdmin; +import org.twonote.rgwadmin4j.RgwAdminBuilder; +import org.twonote.rgwadmin4j.model.BucketInfo; +import org.twonote.rgwadmin4j.model.S3Credential; +import org.twonote.rgwadmin4j.model.User; + +import javax.inject.Inject; +import java.util.List; +import java.util.ArrayList; +import java.util.Optional; +import java.util.Map; +import java.util.HashMap; + +public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = LogManager.getLogger(CephObjectStoreDriverImpl.class); + + @Inject + AccountDao _accountDao; + + @Inject + AccountDetailsDao _accountDetailsDao; + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + @Inject + ObjectStoreDetailsDao _storeDetailsDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + private static final String CEPH_ACCESS_KEY = "ceph-rgw-accesskey"; + private static final String CEPH_SECRET_KEY = "ceph-rgw-secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + long accountId = bucket.getAccountId(); + AmazonS3 s3client = getS3Client(storeId, accountId); + + try { + if (s3client.getBucketAcl(bucketName) != null) { + throw new CloudRuntimeException("Bucket already exists with name " + bucketName); + } + } catch (AmazonS3Exception e) { + if (e.getStatusCode() != 404) { + throw new CloudRuntimeException(e); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + try { + s3client.createBucket(bucketName); + String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(accessKey); + bucketVO.setSecretKey(secretKey); + bucketVO.setBucketURL(store.getUrl() + "/" + bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucketVO; + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public List listBuckets(long storeId) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + List bucketsList = new ArrayList<>(); + try { + List buckets = rgwAdmin.listBucket(); + for(String name : buckets) { + Bucket bucket = new BucketObject(); + bucket.setName(name); + bucketsList.add(bucket); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return bucketsList; + } + + @Override + public boolean deleteBucket(BucketTO bucket, long storeId) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + + try { + rgwAdmin.removeBucket(bucket.getName()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public AccessControlList getBucketAcl(BucketTO bucket, long storeId) { + return null; + } + + @Override + public void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { + String policyConfig; + + if (policy.equalsIgnoreCase("public")) { + s_logger.debug("Setting public policy on bucket " + bucket.getName()); + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + builder.append(" \"Statement\": [\n"); + builder.append(" {\n"); + builder.append(" \"Action\": [\n"); + builder.append(" \"s3:GetBucketLocation\",\n"); + builder.append(" \"s3:ListBucket\"\n"); + builder.append(" ],\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::" + bucket.getName() + "\"\n"); + builder.append(" },\n"); + builder.append(" {\n"); + builder.append(" \"Action\": \"s3:GetObject\",\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::" + bucket.getName() + "/*\"\n"); + builder.append(" }\n"); + builder.append(" ],\n"); + builder.append(" \"Version\": \"2012-10-17\"\n"); + builder.append("}\n"); + policyConfig = builder.toString(); + } else { + s_logger.debug("Setting private policy on bucket " + bucket.getName()); + policyConfig = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + } + + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + client.setBucketPolicy(new SetBucketPolicyRequest(bucket.getName(), policyConfig)); + } + + @Override + public BucketPolicy getBucketPolicy(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + return client.getBucketPolicy(new GetBucketPolicyRequest(bucket.getName())); + } + + @Override + public void deleteBucketPolicy(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + client.deleteBucketPolicy(new DeleteBucketPolicyRequest(bucket.getName())); + } + + @Override + public boolean createUser(long accountId, long storeId) { + Account account = _accountDao.findById(accountId); + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + String username = account.getUuid(); + + s_logger.debug("Attempting to create Ceph RGW user for account " + account.getAccountName() + " with UUID " + username); + try { + Optional user = rgwAdmin.getUserInfo(username); + if (user.isPresent()) { + s_logger.info("User already exists in Ceph RGW: " + username); + return true; + } + } catch (Exception e) { + s_logger.debug("User does not exist. Creating user in Ceph RGW: " + username); + } + + try { + rgwAdmin.createUser(username); + User newUser = rgwAdmin.getUserInfo(username).get(); + S3Credential credentials = newUser.getS3Credentials().get(0); + + Map details = new HashMap<>(); + details.put(CEPH_ACCESS_KEY, credentials.getAccessKey()); + details.put(CEPH_SECRET_KEY, credentials.getSecretKey()); + _accountDetailsDao.persist(accountId, details); + return true; + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public boolean setBucketEncryption(BucketTO bucket, long storeId) { + return false; + } + + @Override + public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { + return false; + } + + @Override + public boolean setBucketVersioning(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + try { + BucketVersioningConfiguration configuration = + new BucketVersioningConfiguration().withStatus("Enabled"); + + SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = + new SetBucketVersioningConfigurationRequest(bucket.getName(), configuration); + + client.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest); + return true; + } catch (AmazonS3Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public boolean deleteBucketVersioning(BucketTO bucket, long storeId) { + AmazonS3 client = getS3Client(getStoreURL(storeId), bucket.getAccessKey(), bucket.getAccessKey()); + try { + BucketVersioningConfiguration configuration = + new BucketVersioningConfiguration().withStatus("Suspended"); + + SetBucketVersioningConfigurationRequest setBucketVersioningConfigurationRequest = + new SetBucketVersioningConfigurationRequest(bucket.getName(), configuration); + + client.setBucketVersioningConfiguration(setBucketVersioningConfigurationRequest); + return true; + } catch (AmazonS3Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public void setBucketQuota(BucketTO bucket, long storeId, long size) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + + try { + rgwAdmin.setBucketQuota(bucket.getName(), -1, size); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public Map getAllBucketsUsage(long storeId) { + RgwAdmin rgwAdmin = getRgwAdminClient(storeId); + try { + List bucketinfo = rgwAdmin.listBucketInfo(); + Map bucketsusage = new HashMap(); + for (BucketInfo bucket: bucketinfo) { + BucketInfo.Usage usage = bucket.getUsage(); + bucketsusage.put(bucket.getBucket(), usage.getRgwMain().getSize_kb()); + } + return bucketsusage; + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + protected RgwAdmin getRgwAdminClient(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String url = store.getUrl(); + String accessKey = storeDetails.get(ACCESS_KEY); + String secretKey = storeDetails.get(SECRET_KEY); + RgwAdmin admin = new RgwAdminBuilder() + .accessKey(accessKey) + .secretKey(secretKey) + .endpoint(url + "/admin") + .build(); + if (admin == null) { + throw new CloudRuntimeException("Error while creating Ceph RGW client"); + } + return admin; + } + + private String getStoreURL(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + String url = store.getUrl(); + return url; + } + + protected AmazonS3 getS3Client(long storeId, long accountId) { + String url = getStoreURL(storeId); + String accessKey = _accountDetailsDao.findDetail(accountId, CEPH_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, CEPH_SECRET_KEY).getValue(); + return this.getS3Client(url, accessKey, secretKey); + } + protected AmazonS3 getS3Client(String url, String accessKey, String secretKey) { + AmazonS3 client = AmazonS3ClientBuilder.standard() + .enablePathStyleAccess() + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials(accessKey, secretKey))) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(url, "auto")) + .build(); + + if (client == null) { + throw new CloudRuntimeException("Error while creating Ceph RGW S3 client"); + } + return client; + } +} diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java new file mode 100644 index 00000000000..a9b13bf338e --- /dev/null +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CephObjectStoreLifeCycleImpl.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.lifecycle; + +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.twonote.rgwadmin4j.RgwAdmin; +import org.twonote.rgwadmin4j.RgwAdminBuilder; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CephObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { + + private static final Logger s_logger = LogManager.getLogger(CephObjectStoreLifeCycleImpl.class); + + @Inject + ObjectStoreHelper objectStoreHelper; + @Inject + ObjectStoreProviderManager objectStoreMgr; + + public CephObjectStoreLifeCycleImpl() { + } + + @SuppressWarnings("unchecked") + @Override + public DataStore initialize(Map dsInfos) { + String url = (String)dsInfos.get("url"); + String name = (String)dsInfos.get("name"); + String providerName = (String)dsInfos.get("providerName"); + Map details = (Map)dsInfos.get("details"); + if (details == null) { + throw new CloudRuntimeException("Ceph RGW Admin credentials are missing"); + } + + String accessKey = details.get("accesskey"); + String secretKey = details.get("secretkey"); + + + Map objectStoreParameters = new HashMap(); + objectStoreParameters.put("name", name); + objectStoreParameters.put("url", url); + + objectStoreParameters.put("providerName", providerName); + objectStoreParameters.put("accesskey", accessKey); + objectStoreParameters.put("secretkey", secretKey); + + s_logger.info("Attempting to connect to Ceph RGW at " + url + " with access key " + accessKey); + + RgwAdmin rgwAdmin = new RgwAdminBuilder() + .accessKey(accessKey) + .secretKey(secretKey) + .endpoint(url + "/admin") + .build(); + try { + List buckets = rgwAdmin.listBucket(); + s_logger.debug("Found " + buckets + " buckets at Ceph RGW: " + url); + s_logger.info("Successfully connected to Ceph RGW: " + url); + } catch (Exception e) { + s_logger.debug("Error while initializing Ceph RGW Object Store: " + e.getMessage()); + throw new RuntimeException("Error while initializing Ceph RGW Object Store. Invalid credentials or URL"); + } + + ObjectStoreVO objectStore = objectStoreHelper.createObjectStore(objectStoreParameters, details); + return objectStoreMgr.getObjectStore(objectStore.getId()); + } + + @Override + public boolean attachCluster(DataStore store, ClusterScope scope) { + return false; + } + + @Override + public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { + return false; + } + + @Override + public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { + return false; + } + + @Override + public boolean maintain(DataStore store) { + return false; + } + + @Override + public boolean cancelMaintain(DataStore store) { + return false; + } + + @Override + public boolean deleteDataStore(DataStore store) { + return false; + } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + +} diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImpl.java new file mode 100644 index 00000000000..e4b0eda42e8 --- /dev/null +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.driver.CephObjectStoreDriverImpl; +import org.apache.cloudstack.storage.datastore.lifecycle.CephObjectStoreLifeCycleImpl; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Component +public class CephObjectStoreProviderImpl implements ObjectStoreProvider { + + @Inject + ObjectStoreProviderManager storeMgr; + @Inject + ObjectStoreHelper helper; + + private final String providerName = "Ceph"; + protected ObjectStoreLifeCycle lifeCycle; + protected ObjectStoreDriver driver; + + @Override + public DataStoreLifeCycle getDataStoreLifeCycle() { + return lifeCycle; + } + + @Override + public String getName() { + return this.providerName; + } + + @Override + public boolean configure(Map params) { + lifeCycle = ComponentContext.inject(CephObjectStoreLifeCycleImpl.class); + driver = ComponentContext.inject(CephObjectStoreDriverImpl.class); + storeMgr.registerDriver(this.getName(), driver); + return true; + } + + @Override + public DataStoreDriver getDataStoreDriver() { + return this.driver; + } + + @Override + public HypervisorHostListener getHostListener() { + return null; + } + + @Override + public Set getTypes() { + Set types = new HashSet(); + types.add(DataStoreProviderType.OBJECT); + return types; + } +} diff --git a/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/module.properties b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/module.properties new file mode 100644 index 00000000000..2aa3f3e2fa2 --- /dev/null +++ b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=storage-object-ceph +parent=storage diff --git a/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/spring-storage-object-ceph-context.xml b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/spring-storage-object-ceph-context.xml new file mode 100644 index 00000000000..c31e652758c --- /dev/null +++ b/plugins/storage/object/ceph/src/main/resources/META-INF/cloudstack/storage-object-ceph/spring-storage-object-ceph-context.xml @@ -0,0 +1,31 @@ + + + + diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java new file mode 100644 index 00000000000..d0cd2e86a22 --- /dev/null +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImplTest.java @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.AmazonS3; +import com.cloud.agent.api.to.BucketTO; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.AccountDetailVO; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.Bucket; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.twonote.rgwadmin4j.RgwAdmin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class CephObjectStoreDriverImplTest { + + @Spy + CephObjectStoreDriverImpl cephObjectStoreDriverImpl = new CephObjectStoreDriverImpl(); + + @Mock + AmazonS3 rgwClient; + @Mock + RgwAdmin rgwAdmin; + @Mock + ObjectStoreDao objectStoreDao; + @Mock + ObjectStoreVO objectStoreVO; + @Mock + ObjectStoreDetailsDao objectStoreDetailsDao; + @Mock + AccountDao accountDao; + @Mock + BucketDao bucketDao; + @Mock + AccountVO account; + @Mock + AccountDetailsDao accountDetailsDao; + + Bucket bucket; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + cephObjectStoreDriverImpl._storeDao = objectStoreDao; + cephObjectStoreDriverImpl._storeDetailsDao = objectStoreDetailsDao; + cephObjectStoreDriverImpl._accountDao = accountDao; + cephObjectStoreDriverImpl._bucketDao = bucketDao; + cephObjectStoreDriverImpl._accountDetailsDao = accountDetailsDao; + bucket = new BucketVO(); + bucket.setName("test-bucket"); + when(objectStoreVO.getUrl()).thenReturn("http://localhost:8000"); + when(objectStoreDao.findById(any())).thenReturn(objectStoreVO); + } + + @Test + public void testCreateBucket() throws Exception { + doReturn(rgwClient).when(cephObjectStoreDriverImpl).getS3Client(anyLong(), anyLong()); + when(accountDetailsDao.findDetail(anyLong(),anyString())). + thenReturn(new AccountDetailVO(1L, "abc","def")); + when(bucketDao.findById(anyLong())).thenReturn(new BucketVO(bucket.getName())); + Bucket bucketRet = cephObjectStoreDriverImpl.createBucket(bucket, false); + assertEquals(bucketRet.getName(), bucket.getName()); + verify(rgwClient, times(1)).getBucketAcl(anyString()); + verify(rgwClient, times(1)).createBucket(anyString()); + } + + @Test + public void testDeleteBucket() throws Exception { + String bucketName = "test-bucket"; + BucketTO bucket = new BucketTO(bucketName); + doReturn(rgwAdmin).when(cephObjectStoreDriverImpl).getRgwAdminClient(anyLong()); + boolean success = cephObjectStoreDriverImpl.deleteBucket(bucket, 1L); + assertTrue(success); + verify(rgwAdmin, times(1)).removeBucket(anyString()); + } +} diff --git a/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImplTest.java b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImplTest.java new file mode 100644 index 00000000000..8b17f52668b --- /dev/null +++ b/plugins/storage/object/ceph/src/test/java/org/apache/cloudstack/storage/datastore/provider/CephObjectStoreProviderImplTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.provider; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider.DataStoreProviderType; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class CephObjectStoreProviderImplTest { + + private CephObjectStoreProviderImpl cephObjectStoreProviderImpl; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + cephObjectStoreProviderImpl = new CephObjectStoreProviderImpl(); + } + + @Test + public void testGetName() { + String name = cephObjectStoreProviderImpl.getName(); + assertEquals("Ceph", name); + } + + @Test + public void testGetTypes() { + Set types = cephObjectStoreProviderImpl.getTypes(); + assertEquals(1, types.size()); + assertEquals("OBJECT", types.toArray()[0].toString()); + } +} diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java index 7effcb78314..9dc4b30414e 100644 --- a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java @@ -41,6 +41,7 @@ import org.apache.commons.lang3.StringUtils; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.storage.BucketVO; import com.cloud.storage.dao.BucketDao; @@ -180,7 +181,8 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucket(String bucketName, long storeId) { + public boolean deleteBucket(BucketTO bucket, long storeId) { + String bucketName = bucket.getName(); MinioClient minioClient = getMinIOClient(storeId); try { if(!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { @@ -199,17 +201,18 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public AccessControlList getBucketAcl(String bucketName, long storeId) { + public AccessControlList getBucketAcl(BucketTO bucket, long storeId) { return null; } @Override - public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + public void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId) { } @Override - public void setBucketPolicy(String bucketName, String policy, long storeId) { + public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { + String bucketName = bucket.getName(); String privatePolicy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; StringBuilder builder = new StringBuilder(); @@ -249,12 +252,12 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + public BucketPolicy getBucketPolicy(BucketTO bucket, long storeId) { return null; } @Override - public void deleteBucketPolicy(String bucketName, long storeId) { + public void deleteBucketPolicy(BucketTO bucket, long storeId) { } @@ -324,11 +327,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean setBucketEncryption(String bucketName, long storeId) { + public boolean setBucketEncryption(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.setBucketEncryption(SetBucketEncryptionArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .config(SseConfiguration.newConfigWithSseS3Rule()) .build() ); @@ -339,11 +342,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucketEncryption(String bucketName, long storeId) { + public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.deleteBucketEncryption(DeleteBucketEncryptionArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .build() ); } catch (Exception e) { @@ -353,11 +356,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean setBucketVersioning(String bucketName, long storeId) { + public boolean setBucketVersioning(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .config(new VersioningConfiguration(VersioningConfiguration.Status.ENABLED, null)) .build() ); @@ -368,11 +371,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucketVersioning(String bucketName, long storeId) { + public boolean deleteBucketVersioning(BucketTO bucket, long storeId) { MinioClient minioClient = getMinIOClient(storeId); try { minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() - .bucket(bucketName) + .bucket(bucket.getName()) .config(new VersioningConfiguration(VersioningConfiguration.Status.SUSPENDED, null)) .build() ); @@ -383,11 +386,11 @@ public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public void setBucketQuota(String bucketName, long storeId, long size) { + public void setBucketQuota(BucketTO bucket, long storeId, long size) { MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); try { - minioAdminClient.setBucketQuota(bucketName, size, QuotaUnit.GB); + minioAdminClient.setBucketQuota(bucket.getName(), size, QuotaUnit.GB); } catch (Exception e) { throw new CloudRuntimeException(e); } diff --git a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java index 5b2faa8d2b5..1a8b3d9663a 100644 --- a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java +++ b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.agent.api.to.BucketTO; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -128,10 +129,11 @@ public class MinIOObjectStoreDriverImplTest { @Test public void testDeleteBucket() throws Exception { String bucketName = "test-bucket"; + BucketTO bucket = new BucketTO(bucketName); doReturn(minioClient).when(minioObjectStoreDriverImpl).getMinIOClient(anyLong()); when(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())).thenReturn(true); doNothing().when(minioClient).removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); - boolean success = minioObjectStoreDriverImpl.deleteBucket(bucketName, 1L); + boolean success = minioObjectStoreDriverImpl.deleteBucket(bucket, 1L); assertTrue(success); verify(minioClient, times(1)).bucketExists(any()); verify(minioClient, times(1)).removeBucket(any()); diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java index b6912483caa..7b9ac59d5b1 100644 --- a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.storage.datastore.driver; import com.amazonaws.services.s3.model.AccessControlList; import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.BucketTO; import com.cloud.agent.api.to.DataStoreTO; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.storage.BucketVO; @@ -71,32 +72,32 @@ public class SimulatorObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean deleteBucket(String bucketName, long storeId) { + public boolean deleteBucket(BucketTO bucket, long storeId) { return true; } @Override - public AccessControlList getBucketAcl(String bucketName, long storeId) { + public AccessControlList getBucketAcl(BucketTO bucket, long storeId) { return null; } @Override - public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + public void setBucketAcl(BucketTO bucket, AccessControlList acl, long storeId) { } @Override - public void setBucketPolicy(String bucketName, String policy, long storeId) { + public void setBucketPolicy(BucketTO bucket, String policy, long storeId) { } @Override - public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + public BucketPolicy getBucketPolicy(BucketTO bucket, long storeId) { return null; } @Override - public void deleteBucketPolicy(String bucketName, long storeId) { + public void deleteBucketPolicy(BucketTO bucket, long storeId) { } @@ -106,27 +107,27 @@ public class SimulatorObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { } @Override - public boolean setBucketEncryption(String bucketName, long storeId) { + public boolean setBucketEncryption(BucketTO bucket, long storeId) { return true; } @Override - public boolean deleteBucketEncryption(String bucketName, long storeId) { + public boolean deleteBucketEncryption(BucketTO bucket, long storeId) { return true; } @Override - public boolean setBucketVersioning(String bucketName, long storeId) { + public boolean setBucketVersioning(BucketTO bucket, long storeId) { return true; } @Override - public boolean deleteBucketVersioning(String bucketName, long storeId) { + public boolean deleteBucketVersioning(BucketTO bucket, long storeId) { return true; } @Override - public void setBucketQuota(String bucketName, long storeId, long size) { + public void setBucketQuota(BucketTO bucket, long storeId, long size) { } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 5e37fa2cb7b..5a2f4e690fa 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -4110,10 +4110,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } try { - // Check URL UriUtils.validateUrl(url); - } catch (final Exception e) { - throw new InvalidParameterValueException(url + " is not a valid URL"); + } catch (InvalidParameterValueException e) { + throw new InvalidParameterValueException(url + " is not a valid URL:" + e.getMessage()); } // Check Unique object store url diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java index e6acd180f16..58b41d6a55d 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.storage.object; import com.amazonaws.services.s3.internal.BucketNameUtils; import com.amazonaws.services.s3.model.IllegalBucketNameException; +import com.cloud.agent.api.to.BucketTO; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; @@ -136,29 +137,30 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic ObjectStoreVO objectStoreVO = _objectStoreDao.findById(cmd.getObjectStoragePoolId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); BucketVO bucket = _bucketDao.findById(cmd.getEntityId()); + BucketTO bucketTO = new BucketTO(bucket); boolean objectLock = false; boolean bucketCreated = false; if(cmd.isObjectLocking()) { objectLock = true; } try { - objectStore.createBucket(bucket, objectLock); + bucketTO = new BucketTO(objectStore.createBucket(bucket, objectLock)); bucketCreated = true; if (cmd.isVersioning()) { - objectStore.setBucketVersioning(bucket.getName()); + objectStore.setBucketVersioning(bucketTO); } if (cmd.isEncryption()) { - objectStore.setBucketEncryption(bucket.getName()); + objectStore.setBucketEncryption(bucketTO); } if (cmd.getQuota() != null) { - objectStore.setQuota(bucket.getName(), cmd.getQuota()); + objectStore.setQuota(bucketTO, cmd.getQuota()); } if (cmd.getPolicy() != null) { - objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + objectStore.setBucketPolicy(bucketTO, cmd.getPolicy()); } bucket.setState(Bucket.State.Created); @@ -166,7 +168,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic } catch (Exception e) { logger.debug("Failed to create bucket with name: "+bucket.getName(), e); if(bucketCreated) { - objectStore.deleteBucket(bucket.getName()); + objectStore.deleteBucket(bucketTO); } _bucketDao.remove(bucket.getId()); throw new CloudRuntimeException("Failed to create bucket with name: "+bucket.getName()+". "+e.getMessage()); @@ -178,13 +180,14 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @ActionEvent(eventType = EventTypes.EVENT_BUCKET_DELETE, eventDescription = "deleting bucket") public boolean deleteBucket(long bucketId, Account caller) { Bucket bucket = _bucketDao.findById(bucketId); + BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { throw new InvalidParameterValueException("Unable to find bucket with ID: " + bucketId); } _accountMgr.checkAccess(caller, null, true, bucket); ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); - if (objectStore.deleteBucket(bucket.getName())) { + if (objectStore.deleteBucket(bucketTO)) { return _bucketDao.remove(bucketId); } return false; @@ -194,6 +197,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @ActionEvent(eventType = EventTypes.EVENT_BUCKET_UPDATE, eventDescription = "updating bucket") public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { BucketVO bucket = _bucketDao.findById(cmd.getId()); + BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { throw new InvalidParameterValueException("Unable to find bucket with ID: " + cmd.getId()); } @@ -203,29 +207,29 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic try { if (cmd.getEncryption() != null) { if (cmd.getEncryption()) { - objectStore.setBucketEncryption(bucket.getName()); + objectStore.setBucketEncryption(bucketTO); } else { - objectStore.deleteBucketEncryption(bucket.getName()); + objectStore.deleteBucketEncryption(bucketTO); } bucket.setEncryption(cmd.getEncryption()); } if (cmd.getVersioning() != null) { if (cmd.getVersioning()) { - objectStore.setBucketVersioning(bucket.getName()); + objectStore.setBucketVersioning(bucketTO); } else { - objectStore.deleteBucketVersioning(bucket.getName()); + objectStore.deleteBucketVersioning(bucketTO); } bucket.setVersioning(cmd.getVersioning()); } if (cmd.getPolicy() != null) { - objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + objectStore.setBucketPolicy(bucketTO, cmd.getPolicy()); bucket.setPolicy(cmd.getPolicy()); } if (cmd.getQuota() != null) { - objectStore.setQuota(bucket.getName(), cmd.getQuota()); + objectStore.setQuota(bucketTO, cmd.getQuota()); bucket.setQuota(cmd.getQuota()); } _bucketDao.update(bucket.getId(), bucket); diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue index 4aacd6adc0f..94c4ef0adb9 100644 --- a/ui/src/views/infra/AddObjectStorage.vue +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -82,7 +82,7 @@ export default { inject: ['parentFetchData'], data () { return { - providers: ['MinIO', 'Simulator'], + providers: ['MinIO', 'Ceph', 'Simulator'], zones: [], loading: false } diff --git a/utils/src/main/java/com/cloud/utils/UriUtils.java b/utils/src/main/java/com/cloud/utils/UriUtils.java index 4964215020d..961c121597f 100644 --- a/utils/src/main/java/com/cloud/utils/UriUtils.java +++ b/utils/src/main/java/com/cloud/utils/UriUtils.java @@ -23,8 +23,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; -import java.net.Inet6Address; import java.net.InetAddress; +import java.net.Inet6Address; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; diff --git a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java index 4ec1f9a9bd9..04a74289122 100644 --- a/utils/src/test/java/com/cloud/utils/UriUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/UriUtilsTest.java @@ -273,4 +273,13 @@ public class UriUtilsTest { Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz")); Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2")); } + + @Test + public void validateUrl() { + Pair url1 = UriUtils.validateUrl("https://www.cloudstack.org"); + Assert.assertEquals(url1.first(), "www.cloudstack.org"); + + Pair url2 = UriUtils.validateUrl("https://www.apache.org"); + Assert.assertEquals(url2.first(), "www.apache.org"); + } }