diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 457dd2e1af4..1be892f4577 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0422cd4bd0..c6edc7bdb20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: if: github.repository == 'apache/cloudstack' @@ -90,6 +93,7 @@ jobs: smoke/test_nic smoke/test_nic_adapter_type smoke/test_non_contigiousvlan + smoke/test_object_stores smoke/test_outofbandmanagement smoke/test_outofbandmanagement_nestedplugin smoke/test_over_provisioning @@ -108,7 +112,8 @@ jobs: smoke/test_reset_configuration_settings smoke/test_reset_vm_on_reboot smoke/test_resource_accounting - smoke/test_resource_detail", + smoke/test_resource_detail + smoke/test_global_acls", "smoke/test_router_dhcphosts smoke/test_router_dns smoke/test_router_dnsservice diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index d243fa863fe..64fa4c3da0c 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 280024b5a91..4d89977adf9 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 7fe648a39fd..8293a22973a 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -10,10 +10,10 @@ This PR... - - + + - + ### Types of changes diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index b88da3621cd..3f07ba16237 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -407,3 +407,15 @@ iscsi.session.cleanup.enabled=false # The path of an executable file/script for host health check for CloudStack to Auto Disable/Enable the host # depending on the return value of the file/script # agent.health.check.script.path= + +# Time interval (in milliseconds) between KVM heartbeats. +# kvm.heartbeat.update.frequency=60000 + +# Number of maximum tries to KVM heartbeats. +# kvm.heartbeat.update.max.tries=5 + +# Time amount (in milliseconds) for the KVM heartbeat retry sleep. +# kvm.heartbeat.update.retry.sleep=10000 + +# Timeout (in milliseconds) of the KVM heartbeat checker. +# kvm.heartbeat.checker.timeout=360000 diff --git a/agent/src/main/java/com/cloud/agent/dao/impl/PropertiesStorage.java b/agent/src/main/java/com/cloud/agent/dao/impl/PropertiesStorage.java index a1db88c86c4..87610c29f34 100644 --- a/agent/src/main/java/com/cloud/agent/dao/impl/PropertiesStorage.java +++ b/agent/src/main/java/com/cloud/agent/dao/impl/PropertiesStorage.java @@ -92,11 +92,14 @@ public class PropertiesStorage implements StorageComponent { file = new File(path); try { if (!file.createNewFile()) { - s_logger.error("Unable to create _file: " + file.getAbsolutePath()); + s_logger.error(String.format("Unable to create _file: %s", file.getAbsolutePath())); return false; } } catch (IOException e) { - s_logger.error("Unable to create _file: " + file.getAbsolutePath(), e); + s_logger.error(String.format("Unable to create file: %s", file.getAbsolutePath())); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("IOException while trying to create file: %s", file.getAbsolutePath()), e); + } return false; } } diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 610c5be759f..8f51d470f07 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -539,10 +539,10 @@ public class AgentProperties{ /** * Heartbeat update timeout (in ms).
* Depending on the use case, this timeout might need increasing/decreasing.
- * Data type: Integer.
- * Default value: 60000 + * Data type: Long.
+ * Default value: 60000L */ - public static final Property HEARTBEAT_UPDATE_TIMEOUT = new Property<>("heartbeat.update.timeout", 60000); + public static final Property HEARTBEAT_UPDATE_TIMEOUT = new Property<>("heartbeat.update.timeout", 60000L); /** * The timeout (in seconds) to retrieve the target's domain ID when migrating a VM with KVM.
@@ -740,6 +740,38 @@ public class AgentProperties{ */ public static final Property CONTROL_CIDR = new Property<>("control.cidr", "169.254.0.0/16"); + /** + * Time interval (in milliseconds) between KVM heartbeats.
+ * This property is for KVM only. + * Data type: Long.
+ * Default value: 60000l + */ + public static final Property KVM_HEARTBEAT_UPDATE_FREQUENCY = new Property<>("kvm.heartbeat.update.frequency", 60000L); + + /** + * Number of maximum tries to KVM heartbeats.
+ * This property is for KVM only. + * Data type: Long.
+ * Default value: 5l + */ + public static final Property KVM_HEARTBEAT_UPDATE_MAX_TRIES = new Property<>("kvm.heartbeat.update.max.tries", 5L); + + /** + * Time amount (in milliseconds) for the KVM heartbeat retry sleep.
+ * This property is for KVM only. + * Data type: Long.
+ * Default value: 10000l + */ + public static final Property KVM_HEARTBEAT_UPDATE_RETRY_SLEEP = new Property<>("kvm.heartbeat.update.retry.sleep", 10000L); + + /** + * Timeout (in milliseconds) of the KVM heartbeat checker.
+ * This property is for KVM only. + * Data type: Long.
+ * Default value: 360000l + */ + public static final Property KVM_HEARTBEAT_CHECKER_TIMEOUT = new Property<>("kvm.heartbeat.checker.timeout", 360000L); + public static class Property { private String name; private T defaultValue; diff --git a/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java b/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java index d923694a854..eb7e8813ecd 100644 --- a/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java +++ b/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java @@ -52,6 +52,13 @@ public class StoragePoolInfo { this.details = details; } + public StoragePoolInfo(String uuid, String host, String hostPath, String localPath, StoragePoolType poolType, long capacityBytes, long availableBytes, + Map details, String name) { + this(uuid, host, hostPath, localPath, poolType, capacityBytes, availableBytes); + this.details = details; + this.name = name; + } + public long getCapacityBytes() { return capacityBytes; } diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 1ac8d79b9d5..5fce169ffed 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -29,6 +29,8 @@ import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.ha.HAConfig; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.vm.schedule.VMSchedule; @@ -714,6 +716,16 @@ public class EventTypes { // SystemVM public static final String EVENT_LIVE_PATCH_SYSTEMVM = "LIVE.PATCH.SYSTEM.VM"; + // OBJECT STORE + public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE"; + public static final String EVENT_OBJECT_STORE_DELETE = "OBJECT.STORE.DELETE"; + public static final String EVENT_OBJECT_STORE_UPDATE = "OBJECT.STORE.UPDATE"; + + // BUCKETS + public static final String EVENT_BUCKET_CREATE = "BUCKET.CREATE"; + public static final String EVENT_BUCKET_DELETE = "BUCKET.DELETE"; + public static final String EVENT_BUCKET_UPDATE = "BUCKET.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1151,6 +1163,16 @@ public class EventTypes { entityEventDetails.put(EVENT_IMAGE_STORE_DATA_MIGRATE, ImageStore.class); entityEventDetails.put(EVENT_IMAGE_STORE_OBJECT_DOWNLOAD, ImageStore.class); entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs"); + + //Object Store + entityEventDetails.put(EVENT_OBJECT_STORE_CREATE, ObjectStore.class); + entityEventDetails.put(EVENT_OBJECT_STORE_UPDATE, ObjectStore.class); + entityEventDetails.put(EVENT_OBJECT_STORE_DELETE, ObjectStore.class); + + //Buckets + entityEventDetails.put(EVENT_BUCKET_CREATE, Bucket.class); + entityEventDetails.put(EVENT_BUCKET_UPDATE, Bucket.class); + entityEventDetails.put(EVENT_BUCKET_DELETE, Bucket.class); } public static String getEntityForEvent(String eventName) { diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index fedde27835e..3efbc315e1a 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -25,6 +25,8 @@ import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd; import org.apache.cloudstack.api.command.admin.network.ListGuestVlansCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd; +import org.apache.cloudstack.api.command.user.address.RemoveQuarantinedIpCmd; +import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkPermissionsCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkPermissionsCmd; @@ -115,6 +117,8 @@ public interface NetworkService { IpAddress getIp(long id); + IpAddress getIp(String ipAddress); + Network updateGuestNetwork(final UpdateNetworkCmd cmd); /** @@ -249,4 +253,8 @@ public interface NetworkService { boolean resetNetworkPermissions(ResetNetworkPermissionsCmd resetNetworkPermissionsCmd); void validateIfServiceOfferingIsActiveAndSystemVmTypeIsDomainRouter(final Long serviceOfferingId) throws InvalidParameterValueException; + + PublicIpQuarantine updatePublicIpAddressInQuarantine(UpdateQuarantinedIpCmd cmd); + + void removePublicIpAddressFromQuarantine(RemoveQuarantinedIpCmd cmd); } diff --git a/api/src/main/java/com/cloud/network/PublicIpQuarantine.java b/api/src/main/java/com/cloud/network/PublicIpQuarantine.java new file mode 100644 index 00000000000..d1ec98afe46 --- /dev/null +++ b/api/src/main/java/com/cloud/network/PublicIpQuarantine.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface PublicIpQuarantine extends InternalIdentity, Identity { + Long getPublicIpAddressId(); + + Long getPreviousOwnerId(); + + Date getEndDate(); + + String getRemovalReason(); + + Date getRemoved(); + + Date getCreated(); +} diff --git a/api/src/main/java/com/cloud/network/VNF.java b/api/src/main/java/com/cloud/network/VNF.java index ebc11c1f39b..e7a7fb01cb9 100644 --- a/api/src/main/java/com/cloud/network/VNF.java +++ b/api/src/main/java/com/cloud/network/VNF.java @@ -68,7 +68,6 @@ public class VNF { } public enum VnfDetail { - ICON, VERSION, VENDOR, MAINTAINER diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index 89ec5b905c9..9bbb5d43eae 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -30,7 +30,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit public enum ResourceObjectType { UserVm(true, true, true), Template(true, true, true), - VnfTemplate(true, true, true), + VnfTemplate(false, false, true), ISO(true, false, true), Volume(true, true), Snapshot(true, false), @@ -69,7 +69,8 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit GuestOs(false, true), NetworkOffering(false, true), VpcOffering(true, false), - Domain(false, false, true); + Domain(false, false, true), + ObjectStore(false, false, true); ResourceObjectType(boolean resourceTagsSupport, boolean resourceMetadataSupport) { diff --git a/api/src/main/java/com/cloud/storage/DataStoreRole.java b/api/src/main/java/com/cloud/storage/DataStoreRole.java index cc20cc0ce96..185e370159c 100644 --- a/api/src/main/java/com/cloud/storage/DataStoreRole.java +++ b/api/src/main/java/com/cloud/storage/DataStoreRole.java @@ -21,7 +21,7 @@ package com.cloud.storage; import com.cloud.utils.exception.CloudRuntimeException; public enum DataStoreRole { - Primary("primary"), Image("image"), ImageCache("imagecache"), Backup("backup"); + Primary("primary"), Image("image"), ImageCache("imagecache"), Backup("backup"), Object("object"); public boolean isImageStore() { return (role.equalsIgnoreCase("image") || role.equalsIgnoreCase("imagecache")) ? true : false; @@ -45,6 +45,8 @@ public enum DataStoreRole { return ImageCache; } else if (role.equalsIgnoreCase("backup")) { return Backup; + } else if (role.equalsIgnoreCase("object")) { + return Object; } else { throw new CloudRuntimeException("can't identify the role"); } diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index bb086ad05cb..c3609cfd8ee 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -24,9 +24,11 @@ import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaint import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import com.cloud.exception.DiscoveryException; @@ -34,6 +36,11 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.storage.object.ObjectStore; public interface StorageService { /** @@ -109,4 +116,15 @@ public interface StorageService { StoragePool syncStoragePool(SyncStoragePoolCmd cmd); + Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd); + + Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd); + + void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd); + + ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; + + boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd); + + ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd); } diff --git a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java index 51c6286a9d5..64bd96d9a9b 100644 --- a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java +++ b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java @@ -45,7 +45,7 @@ public interface AnnotationService { SERVICE_OFFERING(false), DISK_OFFERING(false), NETWORK_OFFERING(false), ZONE(false), POD(false), CLUSTER(false), HOST(false), DOMAIN(false), PRIMARY_STORAGE(false), SECONDARY_STORAGE(false), VR(false), SYSTEM_VM(false), - AUTOSCALE_VM_GROUP(true), MANAGEMENT_SERVER(false),; + AUTOSCALE_VM_GROUP(true), MANAGEMENT_SERVER(false), OBJECT_STORAGE(false); private final boolean usersAllowed; @@ -78,6 +78,7 @@ public interface AnnotationService { list.add(EntityType.VR); list.add(EntityType.SYSTEM_VM); list.add(EntityType.MANAGEMENT_SERVER); + list.add(EntityType.OBJECT_STORAGE); if (roleType != RoleType.DomainAdmin) { list.add(EntityType.DOMAIN); list.add(EntityType.SERVICE_OFFERING); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 9267ca6fa96..affceb4e3f9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -78,7 +78,9 @@ public enum ApiCommandResourceType { VmSnapshot(com.cloud.vm.snapshot.VMSnapshot.class), Role(org.apache.cloudstack.acl.Role.class), VpnCustomerGateway(com.cloud.network.Site2SiteCustomerGateway.class), - ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class); + ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class), + ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class), + Bucket(org.apache.cloudstack.storage.object.Bucket.class); private final Class clazz; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index ee64caabed0..82273cfdd0c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -224,6 +224,8 @@ public class ApiConstants { public static final String INSTANCES_STATS_USER_ONLY = "instancesstatsuseronly"; public static final String PREFIX = "prefix"; public static final String PREVIOUS_ACL_RULE_ID = "previousaclruleid"; + public static final String PREVIOUS_OWNER_ID = "previousownerid"; + public static final String PREVIOUS_OWNER_NAME = "previousownername"; public static final String NEXT_ACL_RULE_ID = "nextaclruleid"; public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash"; public static final String IMAGE_PATH = "imagepath"; @@ -404,6 +406,7 @@ public class ApiConstants { public static final String SHOW_CAPACITIES = "showcapacities"; public static final String SHOW_REMOVED = "showremoved"; public static final String SHOW_RESOURCE_ICON = "showicon"; + public static final String SHOW_INACTIVE = "showinactive"; public static final String SHOW_UNIQUE = "showunique"; public static final String SIGNATURE = "signature"; public static final String SIGNATURE_VERSION = "signatureversion"; @@ -802,6 +805,7 @@ public class ApiConstants { public static final String IPSEC_PSK = "ipsecpsk"; public static final String GUEST_IP = "guestip"; public static final String REMOVED = "removed"; + public static final String REMOVAL_REASON = "removalreason"; public static final String COMPLETED = "completed"; public static final String IKE_VERSION = "ikeversion"; public static final String IKE_POLICY = "ikepolicy"; @@ -1059,11 +1063,20 @@ public class ApiConstants { public static final String MTU = "mtu"; public static final String AUTO_ENABLE_KVM_HOST = "autoenablekvmhost"; public static final String LIST_APIS = "listApis"; + public static final String OBJECT_STORAGE_ID = "objectstorageid"; + public static final String VERSIONING = "versioning"; + public static final String OBJECT_LOCKING = "objectlocking"; + public static final String ENCRYPTION = "encryption"; + public static final String QUOTA = "quota"; + public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; public static final String SOURCE_NAT_IP_ID = "sourcenatipaddressid"; public static final String HAS_RULES = "hasrules"; public static final String NSX_DETAIL_KEY = "forNsx"; + public static final String OBJECT_STORAGE = "objectstore"; + public static final String HEURISTIC_RULE = "heuristicrule"; + public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME."; public static final String MANAGEMENT = "management"; public static final String IS_VNF = "isvnf"; public static final String VNF_NICS = "vnfnics"; @@ -1076,6 +1089,10 @@ public class ApiConstants { public static final String CLIENT_ID = "clientid"; public static final String REDIRECT_URI = "redirecturi"; + public static final String IS_TAG_A_RULE = "istagarule"; + + public static final String PARAMETER_DESCRIPTION_IS_TAG_A_RULE = "Whether the informed tag is a JS interpretable rule or not."; + /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index 0b80cfc8229..f32922819b0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.network.element.InternalLoadBalancerElementService; import org.apache.cloudstack.network.lb.ApplicationLoadBalancerService; import org.apache.cloudstack.network.lb.InternalLoadBalancerVMService; import org.apache.cloudstack.query.QueryService; +import org.apache.cloudstack.storage.object.BucketApiService; import org.apache.cloudstack.storage.ImageStoreService; import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.usage.UsageService; @@ -216,6 +217,9 @@ public abstract class BaseCmd { public Ipv6Service ipv6Service; @Inject public VnfTemplateManager vnfTemplateManager; + @Inject + public BucketApiService _bucketService; + public abstract void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 1a0df486298..0bddf6d2994 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ApiConstants.HostDetails; @@ -37,6 +38,7 @@ import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.CapacityResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConditionResponse; @@ -64,6 +66,7 @@ import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.HypervisorCapabilitiesResponse; import org.apache.cloudstack.api.response.HypervisorGuestOsNamesResponse; import org.apache.cloudstack.api.response.IPAddressResponse; +import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; import org.apache.cloudstack.api.response.InternalLoadBalancerElementResponse; @@ -81,6 +84,7 @@ import org.apache.cloudstack.api.response.NetworkPermissionsResponse; import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.NicResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.OvsProviderResponse; import org.apache.cloudstack.api.response.PhysicalNetworkResponse; import org.apache.cloudstack.api.response.PodResponse; @@ -100,6 +104,7 @@ import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RollingMaintenanceResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SSHKeyPairResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; @@ -144,6 +149,8 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import com.cloud.capacity.Capacity; @@ -169,6 +176,7 @@ import com.cloud.network.OvsProvider; import com.cloud.network.PhysicalNetwork; import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.PhysicalNetworkTrafficType; +import com.cloud.network.PublicIpQuarantine; import com.cloud.network.RemoteAccessVpn; import com.cloud.network.RouterHealthCheckResult; import com.cloud.network.Site2SiteCustomerGateway; @@ -529,4 +537,12 @@ public interface ResponseGenerator { DirectDownloadCertificateHostStatusResponse createDirectDownloadCertificateProvisionResponse(Long certificateId, Long hostId, Pair result); FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl); + + SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic); + + IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp); + + ObjectStoreResponse createObjectStoreResponse(ObjectStore os); + + BucketResponse createBucketResponse(Bucket bucket); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index 75fe339b710..b8668f61ca4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -119,6 +119,10 @@ public class ListHostsCmd extends BaseListCmd { return id; } + public void setId(Long id) { + this.id = id; + } + public String getHostName() { return hostName; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java index e3ff130e2d4..9cf47a9c4b9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java @@ -60,6 +60,9 @@ public class UpdateHostCmd extends BaseCmd { @Parameter(name = ApiConstants.HOST_TAGS, type = CommandType.LIST, collectionType = CommandType.STRING, description = "list of tags to be added to the host") private List hostTags; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the new uri for the secondary storage: nfs://host/path") private String url; @@ -90,6 +93,10 @@ public class UpdateHostCmd extends BaseCmd { return hostTags; } + public Boolean getIsTagARule() { + return isTagARule; + } + public String getUrl() { return url; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java new file mode 100644 index 00000000000..a538962e076 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.storage.object.ObjectStore; +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.log4j.Logger; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +@APICommand(name = "addObjectStoragePool", description = "Adds a object storage pool", responseObject = ObjectStoreResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class AddObjectStoragePoolCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(AddObjectStoragePoolCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "the name for the object store") + private String name; + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, length = 2048, required = true, description = "the URL for the object store") + private String url; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, required = true, description = "the object store provider name") + private String providerName; + + @Parameter(name = ApiConstants.DETAILS, + type = CommandType.MAP, + description = "the details for the object store. Example: details[0].key=accesskey&details[0].value=s389ddssaa&details[1].key=secretkey&details[1].value=8dshfsss") + private Map details; + + @Parameter(name = ApiConstants.TAGS, type = CommandType.STRING, description = "the tags for the storage pool") + private String tags; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getUrl() { + return url; + } + + public String getName() { + return name; + } + + public Map getDetails() { + Map detailsMap = null; + if (details != null && !details.isEmpty()) { + detailsMap = new HashMap(); + Collection props = details.values(); + Iterator iter = props.iterator(); + while (iter.hasNext()) { + HashMap detail = (HashMap)iter.next(); + String key = detail.get(ApiConstants.KEY); + String value = detail.get(ApiConstants.VALUE); + detailsMap.put(key, value); + } + } + return detailsMap; + } + + public String getProviderName() { + return providerName; + } + + public void setUrl(String url) { + this.url = url; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public void setDetails(Map details) { + this.details = details; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute(){ + try{ + ObjectStore result = _storageService.discoverObjectStore(getName(), getUrl(), getProviderName(), getDetails()); + ObjectStoreResponse storeResponse = null; + if (result != null) { + storeResponse = _responseGenerator.createObjectStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("objectstore"); + setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add object storage"); + } + } catch (Exception ex) { + s_logger.error("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java index 0eacc5cda6b..477d7570dfa 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java @@ -90,6 +90,9 @@ public class CreateStoragePoolCmd extends BaseCmd { description = "hypervisor type of the hosts in zone that will be attached to this storage pool. KVM, VMware supported as of now.") private String hypervisor; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -146,6 +149,10 @@ public class CreateStoragePoolCmd extends BaseCmd { return hypervisor; } + public Boolean isTagARule() { + return this.isTagARule; + } + @Override public long getEntityOwnerId() { return Account.ACCOUNT_ID_SYSTEM; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java new file mode 100644 index 00000000000..ed305d9689d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "deleteObjectStoragePool", description = "Deletes an Object Storage Pool", responseObject = SuccessResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class DeleteObjectStoragePoolCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(DeleteObjectStoragePoolCmd.class.getName()); + + // /////////////////////////////////////////////////// + // ////////////// API parameters ///////////////////// + // /////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, required = true, description = "The Object Storage ID.") + private Long id; + + // /////////////////////////////////////////////////// + // ///////////////// Accessors /////////////////////// + // /////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + // /////////////////////////////////////////////////// + // ///////////// API Implementation/////////////////// + // /////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + boolean result = _storageService.deleteObjectStore(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete object store"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java new file mode 100644 index 00000000000..9d8d8eccc3c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "listObjectStoragePools", description = "Lists object storage pools.", responseObject = ObjectStoreResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListObjectStoragePoolsCmd extends BaseListCmd { + public static final Logger s_logger = Logger.getLogger(ListObjectStoragePoolsCmd.class.getName()); + + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the object store") + private String storeName; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, description = "the object store provider") + private String provider; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, description = "the ID of the storage pool") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public String getStoreName() { + return storeName; + } + + public Long getId() { + return id; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ListResponse response = _queryService.searchForObjectStores(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java index 209aaac279c..6923353b3bf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.ClusterResponse; +import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; @@ -63,16 +64,25 @@ public class ListStoragePoolsCmd extends BaseListCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "the ID of the storage pool") private Long id; - @Parameter(name = ApiConstants.SCOPE, type = CommandType.STRING, entityType = StoragePoolResponse.class, description = "the ID of the storage pool") + @Parameter(name = ApiConstants.SCOPE, type = CommandType.STRING, entityType = StoragePoolResponse.class, description = "the scope of the storage pool") private String scope; + @Parameter(name = ApiConstants.STATUS, type = CommandType.STRING, description = "the status of the storage pool") private String status; + @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "host ID of the storage pools") + private Long hostId; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// + public Long getHostId() { + return hostId; + } + public Long getClusterId() { return clusterId; } @@ -81,6 +91,10 @@ public class ListStoragePoolsCmd extends BaseListCmd { return ipAddress; } + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + public String getStoragePoolName() { return storagePoolName; } @@ -108,6 +122,15 @@ public class ListStoragePoolsCmd extends BaseListCmd { public void setId(Long id) { this.id = id; } + + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -123,8 +146,4 @@ public class ListStoragePoolsCmd extends BaseListCmd { response.setResponseName(getCommandName()); this.setResponseObject(response); } - - public String getScope() { - return scope; - } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java new file mode 100644 index 00000000000..497179d25ef --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = UpdateObjectStoragePoolCmd.APINAME, description = "Updates object storage pool", responseObject = ObjectStoreResponse.class, entityType = {ObjectStore.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0") +public class UpdateObjectStoragePoolCmd extends BaseCmd { + public static final String APINAME = "updateObjectStoragePool"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, required = true, description = "Object Store ID") + private Long id; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name for the object store") + private String name; + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the url for the object store") + private String url; + + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getUrl() { + return url; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ObjectStore result = _storageService.updateObjectStore(getId(), this); + + ObjectStoreResponse storeResponse = null; + if (result != null) { + storeResponse = _responseGenerator.createObjectStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("objectstore"); + setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update object storage"); + } + } + + @Override + public String getCommandName() { + return APINAME; + } + + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java index 9e34684a09d..09ec5394921 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java @@ -61,6 +61,9 @@ public class UpdateStoragePoolCmd extends BaseCmd { " enable it back.") private Boolean enabled; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -89,6 +92,10 @@ public class UpdateStoragePoolCmd extends BaseCmd { return enabled; } + public Boolean isTagARule() { + return isTagARule; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..a0e99b55971 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "createSecondaryStorageSelector", description = "Creates a secondary storage selector, described by the heuristic rule.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, entityType = {Heuristic.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class CreateSecondaryStorageSelectorCmd extends BaseCmd{ + + @Parameter(name = ApiConstants.NAME, required = true, type = CommandType.STRING, description = "The name identifying the heuristic rule.") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, required = true, type = BaseCmd.CommandType.STRING, description = "The description of the heuristic rule.") + private String description; + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = BaseCmd.CommandType.UUID, description = "The zone in which the heuristic " + + "rule will be applied.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, required = true, type = BaseCmd.CommandType.STRING, description = + "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.createSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create a secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java new file mode 100644 index 00000000000..c437cd660e7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "listSecondaryStorageSelectors", description = "Lists the secondary storage selectors and their rules.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class ListSecondaryStorageSelectorsCmd extends BaseListCmd { + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = CommandType.UUID, description = "The zone ID to be used in the search filter.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = + "Whether to filter the selectors by type and, if so, which one. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Show removed heuristics.") + private boolean showRemoved = false; + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public boolean isShowRemoved() { + return showRemoved; + } + + @Override + public void execute() { + ListResponse response = _queryService.listSecondaryStorageSelectors(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..79554f44782 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "removeSecondaryStorageSelector", description = "Removes an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class RemoveSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector to be removed.") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() { + _storageService.removeSecondaryStorageHeuristic(this); + final SuccessResponse response = new SuccessResponse(); + response.setResponseName(getCommandName()); + response.setSuccess(true); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..e63d1cbfa27 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "updateSecondaryStorageSelector", description = "Updates an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class UpdateSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector.") + private Long id; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public Long getId() { + return id; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.updateSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java index b31520e1b88..f9bfcb253b4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java @@ -46,10 +46,14 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, required = true, description = "the ID of the public IP address" - + " to disassociate") + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, description = "the ID of the public IP address" + + " to disassociate. Mutually exclusive with the ipaddress parameter") private Long id; + @Parameter(name=ApiConstants.IP_ADDRESS, type=CommandType.STRING, since="4.19.0", description="IP Address to be disassociated." + + " Mutually exclusive with the id parameter") + private String ipAddress; + // unexposed parameter needed for events logging @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, expose = false) private Long ownerId; @@ -59,7 +63,18 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// public Long getIpAddressId() { - return id; + if (id != null & ipAddress != null) { + throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter"); + } + + if (id != null) { + return id; + } else if (ipAddress != null) { + IpAddress ip = getIpAddressByIp(ipAddress); + return ip.getId(); + } + + throw new InvalidParameterValueException("Please specify either IP address or IP address ID"); } ///////////////////////////////////////////////////// @@ -68,12 +83,13 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public void execute() throws InsufficientAddressCapacityException { - CallContext.current().setEventDetails("IP ID: " + getIpAddressId()); + Long ipAddressId = getIpAddressId(); + CallContext.current().setEventDetails("IP ID: " + ipAddressId); boolean result = false; - if (!isPortable(id)) { - result = _networkService.releaseIpAddress(getIpAddressId()); + if (!isPortable()) { + result = _networkService.releaseIpAddress(ipAddressId); } else { - result = _networkService.releasePortableIpAddress(getIpAddressId()); + result = _networkService.releasePortableIpAddress(ipAddressId); } if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); @@ -85,7 +101,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public String getEventType() { - if (!isPortable(id)) { + if (!isPortable()) { return EventTypes.EVENT_NET_IP_RELEASE; } else { return EventTypes.EVENT_PORTABLE_IP_RELEASE; @@ -100,10 +116,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { if (ownerId == null) { - IpAddress ip = getIpAddress(id); - if (ip == null) { - throw new InvalidParameterValueException("Unable to find IP address by ID=" + id); - } + IpAddress ip = getIpAddress(); ownerId = ip.getAccountId(); } @@ -120,11 +133,11 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public Long getSyncObjId() { - IpAddress ip = getIpAddress(id); + IpAddress ip = getIpAddress(); return ip.getAssociatedWithNetworkId(); } - private IpAddress getIpAddress(long id) { + private IpAddress getIpAddressById(Long id) { IpAddress ip = _entityMgr.findById(IpAddress.class, id); if (ip == null) { @@ -134,6 +147,29 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { } } + private IpAddress getIpAddressByIp(String ipAddress) { + IpAddress ip = _networkService.getIp(ipAddress); + if (ip == null) { + throw new InvalidParameterValueException("Unable to find IP address by IP address=" + ipAddress); + } else { + return ip; + } + } + + private IpAddress getIpAddress() { + if (id != null & ipAddress != null) { + throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter"); + } + + if (id != null) { + return getIpAddressById(id); + } else if (ipAddress != null){ + return getIpAddressByIp(ipAddress); + } + + throw new InvalidParameterValueException("Please specify either IP address or IP address ID"); + } + @Override public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.IpAddress; @@ -144,8 +180,8 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { return getIpAddressId(); } - private boolean isPortable(long id) { - IpAddress ip = getIpAddress(id); + private boolean isPortable() { + IpAddress ip = getIpAddress(); return ip.isPortable(); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListQuarantinedIpsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListQuarantinedIpsCmd.java new file mode 100644 index 00000000000..cc014702c81 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListQuarantinedIpsCmd.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.address; + +import com.cloud.network.PublicIpQuarantine; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.IpQuarantineResponse; +import org.apache.cloudstack.api.response.ListResponse; + +@APICommand(name = "listQuarantinedIps", responseObject = IpQuarantineResponse.class, description = "List public IP addresses in quarantine.", since = "4.19", + entityType = {PublicIpQuarantine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin, RoleType.DomainAdmin}) +public class ListQuarantinedIpsCmd extends BaseListCmd { + + @Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Show IPs removed from quarantine.") + private boolean showRemoved = false; + + @Parameter(name = ApiConstants.SHOW_INACTIVE, type = CommandType.BOOLEAN, description = "Show IPs that are no longer in quarantine.") + private boolean showInactive = false; + + public boolean isShowRemoved() { + return showRemoved; + } + public boolean isShowInactive() { + return showInactive; + } + + @Override + public void execute() { + ListResponse response = _queryService.listQuarantinedIps(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/RemoveQuarantinedIpCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/RemoveQuarantinedIpCmd.java new file mode 100644 index 00000000000..82e8373d93a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/RemoveQuarantinedIpCmd.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.address; + +import com.cloud.network.PublicIpQuarantine; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.IpQuarantineResponse; +import org.apache.cloudstack.api.response.SuccessResponse; + +@APICommand(name = "removeQuarantinedIp", responseObject = IpQuarantineResponse.class, description = "Removes a public IP address from quarantine. Only IPs in active " + + "quarantine can be removed.", + since = "4.19", entityType = {PublicIpQuarantine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.DomainAdmin}) +public class RemoveQuarantinedIpCmd extends BaseCmd { + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IpQuarantineResponse.class, description = "The ID of the public IP address in active quarantine. " + + "Either the IP address is informed, or the ID of the IP address in quarantine.") + private Long id; + + @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "The public IP address in active quarantine. Either the IP address is informed, or the ID" + + " of the IP address in quarantine.") + private String ipAddress; + + @Parameter(name = ApiConstants.REMOVAL_REASON, type = CommandType.STRING, required = true, description = "The reason for removing the public IP address from quarantine " + + "prematurely.") + private String removalReason; + + public Long getId() { + return id; + } + + public String getIpAddress() { + return ipAddress; + } + + public String getRemovalReason() { + return removalReason; + } + + @Override + public void execute() { + _networkService.removePublicIpAddressFromQuarantine(this); + final SuccessResponse response = new SuccessResponse(); + response.setResponseName(getCommandName()); + response.setSuccess(true); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/UpdateQuarantinedIpCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/UpdateQuarantinedIpCmd.java new file mode 100644 index 00000000000..b3b71c33781 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/UpdateQuarantinedIpCmd.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.address; + +import com.cloud.network.PublicIpQuarantine; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.IpQuarantineResponse; + +import java.util.Date; + +@APICommand(name = "updateQuarantinedIp", responseObject = IpQuarantineResponse.class, description = "Updates the quarantine end date for the given public IP address.", + since = "4.19", entityType = {PublicIpQuarantine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.DomainAdmin}) +public class UpdateQuarantinedIpCmd extends BaseCmd { + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IpQuarantineResponse.class, description = "The ID of the public IP address in " + + "active quarantine.") + private Long id; + + @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "The public IP address in active quarantine. Either the IP address is informed, or the ID" + + " of the IP address in quarantine.") + private String ipAddress; + + @Parameter(name = ApiConstants.END_DATE, type = BaseCmd.CommandType.DATE, required = true, description = "The date when the quarantine will no longer be active.") + private Date endDate; + + public Long getId() { + return id; + } + + public String getIpAddress() { + return ipAddress; + } + + public Date getEndDate() { + return endDate; + } + + @Override + public void execute() { + PublicIpQuarantine publicIpQuarantine = _networkService.updatePublicIpAddressInQuarantine(this); + if (publicIpQuarantine == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update public IP quarantine."); + } + IpQuarantineResponse response = _responseGenerator.createQuarantinedIpsResponse(publicIpQuarantine); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java new file mode 100644 index 00000000000..e9a140cf46e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceAllocationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "createBucket", responseObject = BucketResponse.class, + description = "Creates a bucket in the specified object storage pool. ", responseView = ResponseView.Restricted, + entityType = {Bucket.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(CreateBucketCmd.class.getName()); + private static final String s_name = "createbucketresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ACCOUNT, + type = CommandType.STRING, + description = "the account associated with the bucket. Must be used with the domainId parameter.") + private String accountName; + + @Parameter(name = ApiConstants.PROJECT_ID, + type = CommandType.UUID, + entityType = ProjectResponse.class, + description = "the project associated with the bucket. Mutually exclusive with account parameter") + private Long projectId; + + @Parameter(name = ApiConstants.DOMAIN_ID, + type = CommandType.UUID, + entityType = DomainResponse.class, + description = "the domain ID associated with the bucket. If used with the account parameter" + + " returns the bucket associated with the account for the specified domain.") + private Long domainId; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true,description = "the name of the bucket") + private String bucketName; + + @Parameter(name = ApiConstants.OBJECT_STORAGE_ID, type = CommandType.UUID, + entityType = ObjectStoreResponse.class, required = true, + description = "Id of the Object Storage Pool where bucket is created") + private long objectStoragePoolId; + + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + private Integer quota; + + @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable bucket encryption") + private boolean encryption; + + @Parameter(name = ApiConstants.VERSIONING, type = CommandType.BOOLEAN, description = "Enable bucket versioning") + private boolean versioning; + + @Parameter(name = ApiConstants.OBJECT_LOCKING, type = CommandType.BOOLEAN, description = "Enable object locking in bucket") + private boolean objectLocking; + + @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING,description = "The Bucket access policy") + private String policy; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getAccountName() { + return accountName; + } + + public Long getDomainId() { + return domainId; + } + + public String getBucketName() { + return bucketName; + } + + private Long getProjectId() { + return projectId; + } + + public long getObjectStoragePoolId() { + return objectStoragePoolId; + } + + public Integer getQuota() { + return quota; + } + + public boolean isEncryption() { + return encryption; + } + + public boolean isVersioning() { + return versioning; + } + + public boolean isObjectLocking() { + return objectLocking; + } + + public String getPolicy() { + return policy; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public String getCommandName() { + return s_name; + } + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + + return accountId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_BUCKET_CREATE; + } + + @Override + public String getEventDescription() { + return "creating bucket: " + getBucketName(); + } + + @Override + public void create() throws ResourceAllocationException { + Bucket bucket = _bucketService.allocBucket(this); + if (bucket != null) { + setEntityId(bucket.getId()); + setEntityUuid(bucket.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create bucket"); + } + } + + @Override + public void execute() { + CallContext.current().setEventDetails("Bucket Id: " + getEntityUuid()); + + Bucket bucket; + try { + bucket = _bucketService.createBucket(this); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + if (bucket != null) { + BucketResponse response = _responseGenerator.createBucketResponse(bucket); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create bucket with name: "+getBucketName()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java new file mode 100644 index 00000000000..bf9552b779e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "deleteBucket", description = "Deletes an empty Bucket.", responseObject = SuccessResponse.class, entityType = {Bucket.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class DeleteBucketCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(DeleteBucketCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=BucketResponse.class, + required=true, description="The ID of the Bucket") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public long getEntityOwnerId() { + Bucket Bucket = _entityMgr.findById(Bucket.class, getId()); + if (Bucket != null) { + return Bucket.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public Long getApiResourceId() { + return id; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() throws ConcurrentOperationException { + CallContext.current().setEventDetails("Bucket Id: " + this._uuidMgr.getUuid(Bucket.class, getId())); + boolean result = _bucketService.deleteBucket(id, CallContext.current().getCallingAccount()); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java new file mode 100644 index 00000000000..897b9fc6696 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.log4j.Logger; + +import java.util.List; + +@APICommand(name = "listBuckets", description = "Lists all Buckets.", responseObject = BucketResponse.class, responseView = ResponseView.Restricted, entityType = { + Bucket.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListBucketsCmd extends BaseListTaggedResourcesCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(ListBucketsCmd.class.getName()); + + private static final String s_name = "listbucketsresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = BucketResponse.class, description = "the ID of the bucket") + private Long id; + + @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = BucketResponse.class, description = "the IDs of the Buckets, mutually exclusive with id") + private List ids; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the bucket") + private String bucketName; + + @Parameter(name = ApiConstants.OBJECT_STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "the ID of the object storage pool, available to ROOT admin only", authorized = { + RoleType.Admin}) + private Long objectStorageId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getBucketName() { + return bucketName; + } + + public Long getObjectStorageId() { + return objectStorageId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() { + ListResponse response = _queryService.searchForBuckets(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + public List getIds() { + return ids; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java new file mode 100644 index 00000000000..b3b7e00770d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "updateBucket", description = "Updates Bucket properties", responseObject = SuccessResponse.class, entityType = {Bucket.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class UpdateBucketCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(UpdateBucketCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=BucketResponse.class, + required=true, description="The ID of the Bucket") + private Long id; + + @Parameter(name = ApiConstants.VERSIONING, type = CommandType.BOOLEAN, description = "Enable/Disable Bucket Versioning") + private Boolean versioning; + + @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable/Disable Bucket encryption") + private Boolean encryption; + + @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING, description = "Bucket Access Policy") + private String policy; + + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + private Integer quota; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Boolean getVersioning() { + return versioning; + } + + public Boolean getEncryption() { + return encryption; + } + + public String getPolicy() { + return policy; + } + + public Integer getQuota() { + return quota; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public long getEntityOwnerId() { + Bucket Bucket = _entityMgr.findById(Bucket.class, getId()); + if (Bucket != null) { + return Bucket.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public Long getApiResourceId() { + return id; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() throws ConcurrentOperationException { + CallContext.current().setEventDetails("Bucket Id: " + this._uuidMgr.getUuid(Bucket.class, getId())); + boolean result = false; + try { + result = _bucketService.updateBucket(this, CallContext.current().getCallingAccount()); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Error while updating bucket. "+e.getMessage()); + } + if(result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update bucket"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java index a7a418f5a86..e1759566299 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -54,7 +55,6 @@ public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { @Parameter(name = ApiConstants.DISPLAY_TEXT, type = BaseCmd.CommandType.STRING, - required = true, description = "the display text of the ISO. This is usually used for display purposes.", length = 4096) private String displayText; @@ -85,7 +85,7 @@ public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { } public String getDisplayText() { - return displayText; + return StringUtils.isBlank(displayText) ? getName() : displayText; } public Boolean isFeatured() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java index ecab3930e8b..1d750038042 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java @@ -177,6 +177,9 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd { } public Long getZoneId() { + if (zoneId == null || zoneId == -1) { + return null; + } return zoneId; } @@ -220,6 +223,10 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd { return directDownload == null ? false : directDownload; } + public void setDirectDownload(Boolean directDownload) { + this.directDownload = directDownload; + } + public boolean isPasswordEnabled() { return passwordEnabled == null ? false : passwordEnabled; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java index 14dbfcafd7b..e5dbcc7b6d1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.network; +import com.cloud.exception.PermissionDeniedException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -26,6 +27,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.NetworkACLResponse; import org.apache.cloudstack.api.response.VpcResponse; +import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; @@ -35,7 +37,8 @@ import com.cloud.network.vpc.NetworkACL; import com.cloud.network.vpc.Vpc; import com.cloud.user.Account; -@APICommand(name = "createNetworkACLList", description = "Creates a network ACL for the given VPC", responseObject = NetworkACLResponse.class, +@APICommand(name = "createNetworkACLList", description = "Creates a network ACL. If no VPC is given, then it creates a global ACL that can be used by everyone.", + responseObject = NetworkACLResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { public static final Logger s_logger = Logger.getLogger(CreateNetworkACLListCmd.class.getName()); @@ -53,7 +56,6 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.VPC_ID, type = CommandType.UUID, - required = true, entityType = VpcResponse.class, description = "ID of the VPC associated with this network ACL list") private Long vpcId; @@ -77,6 +79,10 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { return vpcId; } + public void setVpcId(Long vpcId) { + this.vpcId = vpcId; + } + @Override public boolean isDisplay() { if (display != null) { @@ -92,6 +98,9 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { @Override public void create() { + if (getVpcId() == null) { + setVpcId(0L); + } NetworkACL result = _networkACLService.createNetworkACL(getName(), getDescription(), getVpcId(), isDisplay()); setEntityId(result.getId()); setEntityUuid(result.getUuid()); @@ -111,12 +120,21 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Vpc vpc = _entityMgr.findById(Vpc.class, getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Invalid vpcId is given"); - } + Account account; + if (isAclAttachedToVpc(this.vpcId)) { + Vpc vpc = _entityMgr.findById(Vpc.class, this.vpcId); + if (vpc == null) { + throw new InvalidParameterValueException(String.format("Invalid VPC ID [%s] provided.", this.vpcId)); + } + account = _accountService.getAccount(vpc.getAccountId()); + } else { + account = CallContext.current().getCallingAccount(); + if (!Account.Type.ADMIN.equals(account.getType())) { + s_logger.warn(String.format("Only Root Admin can create global ACLs. Account [%s] cannot create any global ACL.", account)); + throw new PermissionDeniedException("Only Root Admin can create global ACLs."); + } - Account account = _accountService.getAccount(vpc.getAccountId()); + } return account.getId(); } @@ -139,4 +157,8 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.NetworkAcl; } + + public boolean isAclAttachedToVpc(Long aclVpcId) { + return aclVpcId != null && aclVpcId != 0; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java index 5fa24ec1630..e7284d515a2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.offering; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -23,14 +24,14 @@ import org.apache.log4j.Logger; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListDomainResourcesCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.ListResponse; @APICommand(name = "listDiskOfferings", description = "Lists all available disk offerings.", responseObject = DiskOfferingResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListDiskOfferingsCmd extends BaseListDomainResourcesCmd { +public class ListDiskOfferingsCmd extends BaseListProjectAndAccountResourcesCmd { public static final Logger s_logger = Logger.getLogger(ListDiskOfferingsCmd.class.getName()); @@ -60,6 +61,12 @@ public class ListDiskOfferingsCmd extends BaseListDomainResourcesCmd { @Parameter(name = ApiConstants.ENCRYPT, type = CommandType.BOOLEAN, description = "listed offerings support disk encryption", since = "4.18") private Boolean encrypt; + @Parameter(name = ApiConstants.STORAGE_TYPE, + type = CommandType.STRING, + description = "the storage type of the service offering. Values are local and shared.", + since = "4.19") + private String storageType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -84,6 +91,10 @@ public class ListDiskOfferingsCmd extends BaseListDomainResourcesCmd { public Boolean getEncrypt() { return encrypt; } + public String getStorageType() { + return storageType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index 3208ef58a4f..a9a699ed3ef 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -16,20 +16,21 @@ // under the License. package org.apache.cloudstack.api.command.user.offering; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListDomainResourcesCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.BaseCmd.CommandType; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.UserVmResponse; @APICommand(name = "listServiceOfferings", description = "Lists all available service offerings.", responseObject = ServiceOfferingResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd { +public class ListServiceOfferingsCmd extends BaseListProjectAndAccountResourcesCmd { public static final Logger s_logger = Logger.getLogger(ListServiceOfferingsCmd.class.getName()); @@ -88,6 +89,12 @@ public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd { since = "4.18") private Boolean encryptRoot; + @Parameter(name = ApiConstants.STORAGE_TYPE, + type = CommandType.STRING, + description = "the storage type of the service offering. Values are local and shared.", + since = "4.19") + private String storageType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -130,6 +137,10 @@ public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd { public Boolean getEncryptRoot() { return encryptRoot; } + public String getStorageType() { + return storageType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java index 73a6155c8c5..6c39ab6d3c7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java @@ -21,18 +21,6 @@ import java.util.List; import java.util.Map; import org.apache.cloudstack.acl.SecurityChecker; -import org.apache.cloudstack.api.command.user.UserCmd; -import org.apache.cloudstack.api.response.GuestOSResponse; -import org.apache.cloudstack.api.response.SnapshotResponse; -import org.apache.cloudstack.api.response.TemplateResponse; -import org.apache.cloudstack.api.response.UserVmResponse; -import org.apache.cloudstack.api.response.VolumeResponse; -import org.apache.cloudstack.api.response.ProjectResponse; - -import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -41,13 +29,23 @@ import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.GuestOSResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; -import com.cloud.projects.Project; import com.cloud.storage.Snapshot; import com.cloud.storage.Volume; import com.cloud.template.VirtualMachineTemplate; @@ -139,6 +137,19 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the zone for the template. Can be specified with snapshot only", since = "4.19.0") private Long zoneId; + @Parameter(name = ApiConstants.DOMAIN_ID, + type = CommandType.UUID, + entityType = DomainResponse.class, + description = "an optional domainId. If the account parameter is used, domainId must also be used.", + since = "4.19.0") + private Long domainId; + + @Parameter(name = ApiConstants.ACCOUNT, + type = CommandType.STRING, + description = "an optional accountName. Must be used with domainId.", + since = "4.19.0") + private String accountName; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -217,6 +228,14 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { return zoneId; } + public Long getDomainId() { + return domainId; + } + + public String getAccountName() { + return accountName; + } + // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// // /////////////////////////////////////////////////// @@ -232,47 +251,12 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { @Override public long getEntityOwnerId() { - Long volumeId = getVolumeId(); - Long snapshotId = getSnapshotId(); Account callingAccount = CallContext.current().getCallingAccount(); - if (volumeId != null) { - Volume volume = _entityMgr.findById(Volume.class, volumeId); - if (volume != null) { - _accountService.checkAccess(callingAccount, SecurityChecker.AccessType.UseEntry, false, volume); - } else { - throw new InvalidParameterValueException("Unable to find volume by id=" + volumeId); - } - } else { - Snapshot snapshot = _entityMgr.findById(Snapshot.class, snapshotId); - if (snapshot != null) { - _accountService.checkAccess(callingAccount, SecurityChecker.AccessType.UseEntry, false, snapshot); - } else { - throw new InvalidParameterValueException("Unable to find snapshot by id=" + snapshotId); - } - } - - if(projectId != null){ - final Project project = _projectService.getProject(projectId); - if (project != null) { - if (project.getState() == Project.State.Active) { - Account projectAccount= _accountService.getAccount(project.getProjectAccountId()); - _accountService.checkAccess(callingAccount, SecurityChecker.AccessType.UseEntry, false, projectAccount); - return project.getProjectAccountId(); - } else { - final PermissionDeniedException ex = - new PermissionDeniedException("Can't add resources to the project with specified projectId in state=" + project.getState() + - " as it's no longer active"); - ex.addProxyObject(project.getUuid(), "projectId"); - throw ex; - } - } else { - throw new InvalidParameterValueException("Unable to find project by id"); - } - } - - return callingAccount.getId(); + ensureAccessCheck(callingAccount); + return findAccountIdToUse(callingAccount); } + @Override public String getEventType() { return EventTypes.EVENT_TEMPLATE_CREATE; @@ -330,4 +314,47 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { } } + + /*** + * Performs access check on volume and snapshot for given account + * @param account + */ + private void ensureAccessCheck(Account account) { + if (volumeId != null) { + Volume volume = _entityMgr.findById(Volume.class, volumeId); + if (volume != null) { + _accountService.checkAccess(account, SecurityChecker.AccessType.UseEntry, false, volume); + } else { + throw new InvalidParameterValueException("Unable to find volume by id=" + volumeId); + } + } else { + Snapshot snapshot = _entityMgr.findById(Snapshot.class, snapshotId); + if (snapshot != null) { + _accountService.checkAccess(account, SecurityChecker.AccessType.UseEntry, false, snapshot); + } else { + throw new InvalidParameterValueException("Unable to find snapshot by id=" + snapshotId); + } + } + } + + /*** + * Find accountId based on accountName and domainId or projectId + * if not found, return callingAccountId for further use + * @param callingAccount + * @return accountId + */ + private Long findAccountIdToUse(Account callingAccount) { + Long accountIdToUse = null; + try { + accountIdToUse = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + } catch (InvalidParameterValueException | PermissionDeniedException ex) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("An exception occurred while finalizing account id with accountName, domainId and projectId" + + "using callingAccountId=%s", callingAccount.getUuid()), ex); + } + s_logger.warn("Unable to find accountId associated with accountName=" + accountName + " and domainId=" + + domainId + " or projectId=" + projectId + ", using callingAccountId=" + callingAccount.getUuid()); + } + return accountIdToUse != null ? accountIdToUse : callingAccount.getAccountId(); + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java index 3a9e1c8b429..ab872b84edb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.exception.ResourceAllocationException; @@ -46,7 +47,7 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { private static final String s_name = "postuploadtemplateresponse"; - @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, required = true, description = "the display text of the template. This is usually used for display purposes.", length = 4096) + @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, description = "the display text of the template. This is usually used for display purposes.", length = 4096) private String displayText; @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, required = true, description = "the target hypervisor for the template") @@ -95,7 +96,7 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { private Boolean deployAsIs; public String getDisplayText() { - return displayText; + return StringUtils.isBlank(displayText) ? getName() : displayText; } public String getHypervisor() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java index 9c9f1c0f50f..f3b452008c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java @@ -125,6 +125,9 @@ public class AddVpnUserCmd extends BaseAsyncCreateCmd { vpnResponse.setId(vpnUser.getUuid()); vpnResponse.setUserName(vpnUser.getUsername()); vpnResponse.setAccountName(account.getAccountName()); + // re-retrieve the vpnuser, as the call to `applyVpnUsers` might have changed the state + vpnUser = _entityMgr.findById(VpnUser.class, getEntityId()); + vpnResponse.setState(vpnUser.getState().toString()); Domain domain = _entityMgr.findById(Domain.class, account.getDomainId()); if (domain != null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java index a74c95780d1..7270fa949c8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java @@ -124,7 +124,7 @@ public class AcquireIPAddressResponse extends BaseResponse implements Controlle private String networkId; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated and Releasing") private String state; @SerializedName(ApiConstants.PHYSICAL_NETWORK_ID) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java new file mode 100644 index 00000000000..b75f3604324 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponseWithTagInformation; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.storage.object.Bucket; + +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.Set; + +@EntityReference(value = Bucket.class) +@SuppressWarnings("unused") +public class BucketResponse extends BaseResponseWithTagInformation implements ControlledViewEntityResponse, ControlledEntityResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the Bucket") + private String id; + @SerializedName(ApiConstants.NAME) + @Param(description = "name of the Bucket") + private String name; + @SerializedName(ApiConstants.CREATED) + @Param(description = "the date the Bucket was created") + private Date created; + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "the account associated with the Bucket") + private String accountName; + @SerializedName(ApiConstants.PROJECT_ID) + @Param(description = "the project id of the bucket") + private String projectId; + @SerializedName(ApiConstants.PROJECT) + @Param(description = "the project name of the bucket") + private String projectName; + @SerializedName(ApiConstants.DOMAIN_ID) + @Param(description = "the ID of the domain associated with the bucket") + private String domainId; + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "the domain associated with the bucket") + private String domainName; + @SerializedName(ApiConstants.OBJECT_STORAGE_ID) + @Param(description = "id of the object storage hosting the Bucket; returned to admin user only") + private String objectStoragePoolId; + + @SerializedName(ApiConstants.OBJECT_STORAGE) + @Param(description = "Name of the object storage hosting the Bucket; returned to admin user only") + private String objectStoragePool; + + @SerializedName(ApiConstants.SIZE) + @Param(description = "Total size of objects in Bucket") + private Long size; + + @SerializedName(ApiConstants.STATE) + @Param(description = "State of the Bucket") + private String state; + + @SerializedName(ApiConstants.QUOTA) + @Param(description = "Bucket Quota in GB") + private Integer quota; + + @SerializedName(ApiConstants.ENCRYPTION) + @Param(description = "Bucket Encryption") + private Boolean encryption; + + @SerializedName(ApiConstants.VERSIONING) + @Param(description = "Bucket Versioning") + private Boolean versioning; + + @SerializedName(ApiConstants.OBJECT_LOCKING) + @Param(description = "Bucket Object Locking") + private Boolean objectLock; + + @SerializedName(ApiConstants.POLICY) + @Param(description = "Bucket Access Policy") + private String policy; + + @SerializedName(ApiConstants.URL) + @Param(description = "Bucket URL") + private String bucketURL; + + @SerializedName(ApiConstants.ACCESS_KEY) + @Param(description = "Bucket Access Key") + private String accessKey; + + @SerializedName(ApiConstants.SECRET_KEY) + @Param(description = "Bucket Secret Key") + private String secretKey; + + @SerializedName(ApiConstants.PROVIDER) + @Param(description = "Object storage provider") + private String provider; + + public BucketResponse() { + tags = new LinkedHashSet(); + } + + @Override + public String getObjectId() { + return this.getId(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setCreated(Date created) { + this.created = created; + } + + @Override + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + @Override + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + @Override + public void setDomainName(String domainName) { + this.domainName = domainName; + } + + @Override + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + @Override + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setTags(Set tags) { + this.tags = tags; + } + + public void setObjectStoragePoolId(String objectStoragePoolId) { + this.objectStoragePoolId = objectStoragePoolId; + } + + public String getName() { + return name; + } + + public Date getCreated() { + return created; + } + + public String getAccountName() { + return accountName; + } + + public String getProjectId() { + return projectId; + } + + public String getProjectName() { + return projectName; + } + + public String getDomainId() { + return domainId; + } + + public String getDomainName() { + return domainName; + } + + public String getObjectStoragePoolId() { + return objectStoragePoolId; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + public long getQuota() { + return quota; + } + + public void setQuota(Integer quota) { + this.quota = quota; + } + + public boolean isVersioning() { + return versioning; + } + + public void setVersioning(boolean versioning) { + this.versioning = versioning; + } + + public boolean isEncryption() { + return encryption; + } + + public void setEncryption(boolean encryption) { + this.encryption = encryption; + } + + public boolean isObjectLock() { + return objectLock; + } + + public void setObjectLock(boolean objectLock) { + this.objectLock = objectLock; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(String policy) { + this.policy = policy; + } + + public String getBucketURL() { + return bucketURL; + } + + public void setBucketURL(String bucketURL) { + this.bucketURL = bucketURL; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setState(Bucket.State state) { + this.state = state.toString(); + } + + public String getState() { + return state; + } + + public void setObjectStoragePool(String objectStoragePool) { + this.objectStoragePool = objectStoragePool; + } + + public String getObjectStoragePool() { + return objectStoragePool; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index e1f1e5ee241..d72d23b99c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -221,6 +221,10 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "comma-separated list of tags for the host") private String hostTags; + @SerializedName(ApiConstants.IS_TAG_A_RULE) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @SerializedName("hasenoughcapacity") @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise") private Boolean hasEnoughCapacity; @@ -732,4 +736,12 @@ public class HostResponse extends BaseResponseWithAnnotations { public void setEncryptionSupported(Boolean encryptionSupported) { this.encryptionSupported = encryptionSupported; } + + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean tagARule) { + isTagARule = tagARule; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java index 2ad997f39d1..8a9bf7789dd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java @@ -128,7 +128,7 @@ public class IPAddressResponse extends BaseResponseWithAnnotations implements Co private String networkId; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing, Reserved and Free") private String state; @SerializedName(ApiConstants.PHYSICAL_NETWORK_ID) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/IpQuarantineResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/IpQuarantineResponse.java new file mode 100644 index 00000000000..55720296315 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/IpQuarantineResponse.java @@ -0,0 +1,130 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.network.PublicIpQuarantine; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import java.util.Date; + +@EntityReference(value = {PublicIpQuarantine.class}) +public class IpQuarantineResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the quarantine process.") + private String id; + + @SerializedName(ApiConstants.IP_ADDRESS) + @Param(description = "The public IP address in quarantine.") + private String publicIpAddress; + + @SerializedName(ApiConstants.PREVIOUS_OWNER_ID) + @Param(description = "Account ID of the previous public IP address owner.") + private String previousOwnerId; + + @SerializedName(ApiConstants.PREVIOUS_OWNER_NAME) + @Param(description = "Account name of the previous public IP address owner.") + private String previousOwnerName; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "When the quarantine was created.") + private Date created; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "When the quarantine was removed.") + private Date removed; + + @SerializedName(ApiConstants.END_DATE) + @Param(description = "End date for the quarantine.") + private Date endDate; + + @SerializedName(ApiConstants.REMOVAL_REASON) + @Param(description = "The reason for removing the IP from quarantine prematurely.") + private String removalReason; + + public IpQuarantineResponse() { + super("quarantinedips"); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getPublicIpAddress() { + return publicIpAddress; + } + + public void setPublicIpAddress(String publicIpAddress) { + this.publicIpAddress = publicIpAddress; + } + + public String getPreviousOwnerId() { + return previousOwnerId; + } + + public void setPreviousOwnerId(String previousOwnerId) { + this.previousOwnerId = previousOwnerId; + } + + public String getPreviousOwnerName() { + return previousOwnerName; + } + + public void setPreviousOwnerName(String previousOwnerName) { + this.previousOwnerName = previousOwnerName; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public String getRemovalReason() { + return removalReason; + } + + public void setRemovalReason(String removalReason) { + this.removalReason = removalReason; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java new file mode 100644 index 00000000000..e4030799aa7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import org.apache.cloudstack.storage.object.ObjectStore; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.BaseResponseWithAnnotations; +import org.apache.cloudstack.api.EntityReference; + +@EntityReference(value = ObjectStore.class) +public class ObjectStoreResponse extends BaseResponseWithAnnotations { + @SerializedName("id") + @Param(description = "the ID of the object store") + private String id; + + @SerializedName("name") + @Param(description = "the name of the object store") + private String name; + + @SerializedName("url") + @Param(description = "the url of the object store") + private String url; + + @SerializedName("providername") + @Param(description = "the provider name of the object store") + private String providerName; + + @SerializedName("storagetotal") + @Param(description = "the total size of the object store") + private Long storageTotal; + + @SerializedName("storageused") + @Param(description = "the object store currently used size") + private Long storageUsed; + + public ObjectStoreResponse() { + } + + @Override + public String getObjectId() { + return this.getId(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getProviderName() { + return providerName; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public Long getStorageTotal() { + return storageTotal; + } + + public void setStorageTotal(Long storageTotal) { + this.storageTotal = storageTotal; + } + + public Long getStorageUsed() { + return storageUsed; + } + + public void setStorageUsed(Long storageUsed) { + this.storageUsed = storageUsed; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java index 73008b01ccd..e477b111561 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java @@ -68,7 +68,7 @@ public class PortableIpResponse extends BaseResponse { private Date allocated; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing and Free") private String state; public void setRegionId(Integer regionId) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java new file mode 100644 index 00000000000..25a6b2eca75 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java @@ -0,0 +1,141 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import java.util.Date; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@EntityReference(value = {Heuristic.class}) +public class SecondaryStorageHeuristicsResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the heuristic.") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the heuristic.") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "Description of the heuristic.") + private String description; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "The zone which the heuristic is valid upon.") + private String zoneId; + + @SerializedName(ApiConstants.TYPE) + @Param(description = "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @SerializedName(ApiConstants.HEURISTIC_RULE) + @Param(description = "The heuristic rule, in JavaScript language, used to select a secondary storage to be directed.") + private String heuristicRule; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "When the heuristic was created.") + private Date created; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "When the heuristic was removed.") + private Date removed; + + + public SecondaryStorageHeuristicsResponse(String id, String name, String description, String zoneId, String type, String heuristicRule, Date created, Date removed) { + super("heuristics"); + this.id = id; + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + this.created = created; + this.removed = removed; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java index 89256c26473..183290ec9eb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -101,6 +101,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "the tags for the storage pool") private String tags; + @SerializedName(ApiConstants.IS_TAG_A_RULE) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @SerializedName(ApiConstants.STATE) @Param(description = "the state of the storage pool") private StoragePoolStatus state; @@ -304,6 +308,14 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { this.tags = tags; } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean tagARule) { + isTagARule = tagARule; + } + public StoragePoolStatus getState() { return state; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java index 3a0e84285aa..d3e4d941678 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java @@ -57,7 +57,7 @@ public class VpnUsersResponse extends BaseResponse implements ControlledEntityRe private String projectName; @SerializedName(ApiConstants.STATE) - @Param(description = "the state of the Vpn User") + @Param(description = "the state of the Vpn User, can be 'Add', 'Revoke' or 'Active'.") private String state; public void setId(String id) { diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 097a3c3f262..3299e7537a2 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -28,13 +28,17 @@ import org.apache.cloudstack.api.command.admin.resource.icon.ListResourceIconCmd import org.apache.cloudstack.api.command.admin.router.GetRouterHealthCheckResultsCmd; import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; import org.apache.cloudstack.api.command.admin.user.ListUsersCmd; import org.apache.cloudstack.api.command.user.account.ListAccountsCmd; import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; +import org.apache.cloudstack.api.command.user.address.ListQuarantinedIpsCmd; import org.apache.cloudstack.api.command.user.affinitygroup.ListAffinityGroupsCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; import org.apache.cloudstack.api.command.user.iso.ListIsosCmd; import org.apache.cloudstack.api.command.user.job.ListAsyncJobsCmd; @@ -55,6 +59,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; @@ -64,8 +69,10 @@ import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; +import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -73,6 +80,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; @@ -183,7 +191,15 @@ public interface QueryService { List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); + ListResponse listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd); + + ListResponse listQuarantinedIps(ListQuarantinedIpsCmd cmd); + ListResponse listSnapshots(ListSnapshotsCmd cmd); SnapshotResponse listSnapshot(CopySnapshotCmd cmd); + + ListResponse searchForObjectStores(ListObjectStoragePoolsCmd listObjectStoragePoolsCmd); + + ListResponse searchForBuckets(ListBucketsCmd listBucketsCmd); } diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java new file mode 100644 index 00000000000..2a0b8d6ea24 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface Heuristic extends InternalIdentity, Identity { + + String getName(); + + String getDescription(); + + Long getZoneId(); + + String getType(); + + String getHeuristicRule(); + + Date getCreated(); + + Date getRemoved(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java new file mode 100644 index 00000000000..f23e4b0b633 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +/** + * The type of the heuristic used in the allocation process of secondary storage resources. + * Valid options are: {@link #ISO}, {@link #SNAPSHOT}, {@link #TEMPLATE} and {@link #VOLUME} + */ +public enum HeuristicType { + ISO, SNAPSHOT, TEMPLATE, VOLUME +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java b/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java new file mode 100644 index 00000000000..c821dbac589 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface Bucket extends ControlledEntity, Identity, InternalIdentity { + + long getObjectStoreId(); + + Date getCreated(); + + State getState(); + + void setName(String name); + + Long getSize(); + + Integer getQuota(); + + boolean isVersioning(); + + boolean isEncryption(); + + boolean isObjectLock(); + + String getPolicy(); + + String getBucketURL(); + + String getAccessKey(); + + String getSecretKey(); + + public enum State { + Allocated, Created, Destroyed; + @Override + public String toString() { + return this.name(); + } + + public boolean equals(String status) { + return this.toString().equalsIgnoreCase(status); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java new file mode 100644 index 00000000000..7e1361d1e71 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.cloud.exception.ResourceAllocationException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; + +public interface BucketApiService { + + + /** + * Creates the database object for a Bucket based on the given criteria + * + * @param cmd + * the API command wrapping the criteria (account/domainId [admin only], zone, diskOffering, snapshot, + * name) + * @return the Bucket object + */ + Bucket allocBucket(CreateBucketCmd cmd) throws ResourceAllocationException; + + /** + * Creates the Bucket based on the given criteria + * + * @param cmd + * the API command wrapping the criteria (account/domainId [admin only], zone, diskOffering, snapshot, + * name) + * @return the Bucket object + */ + Bucket createBucket(CreateBucketCmd cmd); + + boolean deleteBucket(long bucketId, Account caller); + + boolean updateBucket(UpdateBucketCmd cmd, Account caller); + + void getBucketUsage(); +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java b/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java new file mode 100644 index 00000000000..47741fc67f4 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface ObjectStore extends Identity, InternalIdentity { + + /** + * @return name of the object store. + */ + String getName(); + + /** + * @return object store provider name + */ + String getProviderName(); + + /** + * + * @return uri + */ + String getUrl(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java index 48cff3076fd..5e0f03ff583 100644 --- a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java +++ b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java @@ -45,6 +45,7 @@ public class UsageTypes { public static final int VOLUME_SECONDARY = 26; public static final int VM_SNAPSHOT_ON_PRIMARY = 27; public static final int BACKUP = 28; + public static final int BUCKET = 29; public static List listUsageTypes() { List responseList = new ArrayList(); @@ -70,6 +71,7 @@ public class UsageTypes { responseList.add(new UsageTypeResponse(VOLUME_SECONDARY, "Volume on secondary storage usage")); responseList.add(new UsageTypeResponse(VM_SNAPSHOT_ON_PRIMARY, "VM Snapshot on primary storage usage")); responseList.add(new UsageTypeResponse(BACKUP, "Backup storage usage")); + responseList.add(new UsageTypeResponse(BUCKET, "Bucket storage usage")); return responseList; } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..f64df167e25 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.exception.DiscoveryException; +import com.cloud.storage.StorageService; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.anyObject; + +@RunWith(MockitoJUnitRunner.class) +public class AddObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(AddObjectStoragePoolCmdTest.class.getName()); + + @Mock + StorageService storageService; + + @Mock + ObjectStore objectStore; + + @Mock + ResponseGenerator responseGenerator; + + @Spy + AddObjectStoragePoolCmd addObjectStoragePoolCmdSpy; + + String name = "testObjStore"; + + String url = "testURL"; + + String provider = "Simulator"; + + Map details; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + details = new HashMap<>(); + addObjectStoragePoolCmdSpy = Mockito.spy(new AddObjectStoragePoolCmd()); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "name", name); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "url", url); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "providerName", provider); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "details", details); + addObjectStoragePoolCmdSpy._storageService = storageService; + addObjectStoragePoolCmdSpy._responseGenerator = responseGenerator; + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testAddObjectStore() throws DiscoveryException { + Mockito.doReturn(objectStore).when(storageService).discoverObjectStore(Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), anyObject()); + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(anyObject()); + addObjectStoragePoolCmdSpy.execute(); + + Mockito.verify(storageService, Mockito.times(1)) + .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..35be56d0c75 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.storage.StorageService; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +public class DeleteObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(DeleteObjectStoragePoolCmdTest.class.getName()); + @Mock + private StorageService storageService; + + @Spy + DeleteObjectStoragePoolCmd deleteObjectStoragePoolCmd; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + deleteObjectStoragePoolCmd = Mockito.spy(new DeleteObjectStoragePoolCmd()); + deleteObjectStoragePoolCmd._storageService = storageService; + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testDeleteObjectStore() { + Mockito.doReturn(true).when(storageService).deleteObjectStore(deleteObjectStoragePoolCmd); + deleteObjectStoragePoolCmd.execute(); + Mockito.verify(storageService, Mockito.times(1)) + .deleteObjectStore(deleteObjectStoragePoolCmd); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..ef66c2a1a64 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.storage.StorageService; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.mockito.ArgumentMatchers.anyObject; + +public class UpdateObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(UpdateObjectStoragePoolCmdTest.class.getName()); + + @Mock + private StorageService storageService; + + @Spy + UpdateObjectStoragePoolCmd updateObjectStoragePoolCmd; + + @Mock + ObjectStore objectStore; + + @Mock + ResponseGenerator responseGenerator; + + private String name = "testObjStore"; + + private String url = "testURL"; + + private String provider = "Simulator"; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + updateObjectStoragePoolCmd = Mockito.spy(new UpdateObjectStoragePoolCmd()); + updateObjectStoragePoolCmd._storageService = storageService; + updateObjectStoragePoolCmd._responseGenerator = responseGenerator; + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "name", name); + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "url", url); + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "id", 1L); + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testUpdateObjectStore() { + Mockito.doReturn(objectStore).when(storageService).updateObjectStore(1L, updateObjectStoragePoolCmd); + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(anyObject()); + updateObjectStoragePoolCmd.execute(); + Mockito.verify(storageService, Mockito.times(1)) + .updateObjectStore(1L, updateObjectStoragePoolCmd); + } + +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java index d8af670a7b2..55e9c8dba1e 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmdTest.java @@ -29,4 +29,20 @@ public class CreateTemplateCmdTest { ReflectionTestUtils.setField(cmd, "zoneId", id); Assert.assertEquals(id, cmd.getZoneId()); } + + @Test + public void testDomainId() { + final CreateTemplateCmd cmd = new CreateTemplateCmd(); + Long id = 2L; + ReflectionTestUtils.setField(cmd, "domainId", id); + Assert.assertEquals(id, cmd.getDomainId()); + } + + @Test + public void testGetAccountName() { + final CreateTemplateCmd cmd = new CreateTemplateCmd(); + String accountName = "user1"; + ReflectionTestUtils.setField(cmd, "accountName", accountName); + Assert.assertEquals(accountName, cmd.getAccountName()); + } } diff --git a/client/pom.xml b/client/pom.xml index 05aa7f8b53e..048bddbe549 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -582,6 +582,21 @@ cloud-plugin-shutdown ${project.version} + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-plugin-storage-object-minio + ${project.version} + + + org.apache.cloudstack + cloud-plugin-storage-object-simulator + ${project.version} + diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java new file mode 100644 index 00000000000..366c7c8fb41 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +import org.apache.cloudstack.storage.object.Bucket; + +public interface BucketInfo extends DataObject, Bucket { + + void addPayload(Object data); + + Object getPayload(); + + Bucket getBucket(); +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 3ee5803a91a..de26a09c6e5 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -35,6 +35,8 @@ public interface DataStoreManager { List getImageStoresByScopeExcludingReadOnly(ZoneScope scope); + List getImageStoresByZoneIds(Long ... zoneIds); + DataStore getRandomImageStore(long zoneId); DataStore getRandomUsableImageStore(long zoneId); @@ -55,5 +57,7 @@ public interface DataStoreManager { boolean isRegionStore(DataStore store); + DataStore getImageStoreByUuid(String uuid); + Long getStoreZoneId(long storeId, DataStoreRole role); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java index 3e5761ef37f..41c1d940745 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java @@ -31,7 +31,7 @@ public interface DataStoreProvider { String DEFAULT_PRIMARY = "DefaultPrimary"; enum DataStoreProviderType { - PRIMARY, IMAGE, ImageCache + PRIMARY, IMAGE, ImageCache, OBJECT } DataStoreLifeCycle getDataStoreLifeCycle(); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java index e476d8f3b35..379911a54fb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java @@ -33,4 +33,6 @@ public interface DataStoreProviderManager extends Manager, DataStoreProviderApiS DataStoreProvider getDefaultCacheDataStoreProvider(); List getProviders(); + + DataStoreProvider getDefaultObjectStoreProvider(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java new file mode 100644 index 00000000000..691da7ecc8d --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.subsystem.api.storage; + +public interface ObjectStorageService { + +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java new file mode 100644 index 00000000000..71b3762b441 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +public interface ObjectStoreProvider extends DataStoreProvider { + +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java index 1dbff59a891..1b18264df15 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java @@ -43,6 +43,8 @@ public class PrimaryDataStoreParameters { private boolean managed; private Long capacityIops; + private Boolean isTagARule; + /** * @return the userInfo */ @@ -277,4 +279,12 @@ public class PrimaryDataStoreParameters { public void setUsedBytes(long usedBytes) { this.usedBytes = usedBytes; } + + public Boolean isTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java new file mode 100644 index 00000000000..9ee94b083cf --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; + +import java.util.List; +import java.util.Map; + +public interface ObjectStoreEntity extends DataStore, ObjectStore { + Bucket createBucket(Bucket bucket, boolean objectLock); + + List listBuckets(); + + boolean createUser(long accountId); + + boolean deleteBucket(String name); + + boolean setBucketEncryption(String name); + + boolean deleteBucketEncryption(String name); + + boolean setBucketVersioning(String name); + + boolean deleteBucketVersioning(String name); + + void setBucketPolicy(String name, String policy); + + void setQuota(String name, int quota); + + Map getAllBucketsUsage(); +} diff --git a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java index ab179d302d0..36937460b20 100644 --- a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.network; +import java.util.Date; import java.util.List; import org.apache.cloudstack.api.response.AcquirePodIpCmdResponse; @@ -238,5 +239,52 @@ public interface IpAddressManager { public static final String MESSAGE_ASSIGN_IPADDR_EVENT = "Message.AssignIpAddr.Event"; public static final String MESSAGE_RELEASE_IPADDR_EVENT = "Message.ReleaseIpAddr.Event"; + + /** + * Checks if the given public IP address is not in active quarantine. + * It returns `true` if: + *
    + *
  • The IP was never in quarantine;
    • + *
  • The IP was in quarantine, but the quarantine expired;
    • + *
  • The IP is still in quarantine; however, the new owner is the same as the previous owner, therefore, the IP can be allocated.
    • + *
+ * + * It returns `false` if: + *
    + *
  • The IP is in active quarantine and the new owner is different from the previous owner.
    • + *
+ * + * @param ip used to check if it is in active quarantine. + * @param account used to identify the new owner of the public IP. + * @return true if the IP can be allocated, and false otherwise. + */ + boolean canPublicIpAddressBeAllocated(IpAddress ip, Account account); + + /** + * Adds the given public IP address to quarantine for the duration of the global configuration `public.ip.address.quarantine.duration` value. + * + * @param publicIpAddress to be quarantined. + * @param domainId used to retrieve the quarantine duration. + * @return the {@link PublicIpQuarantine} persisted in the database. + */ + PublicIpQuarantine addPublicIpAddressToQuarantine(IpAddress publicIpAddress, Long domainId); + + /** + * Prematurely removes a public IP address from quarantine. It is required to provide a reason for removing it. + * + * @param quarantineProcessId the ID of the active quarantine process. + * @param removalReason for prematurely removing the public IP address from quarantine. + */ + void removePublicIpAddressFromQuarantine(Long quarantineProcessId, String removalReason); + + /** + * Updates the end date of a public IP address in active quarantine. It can increase and decrease the duration of the quarantine. + * + * @param quarantineProcessId the ID of the quarantine process. + * @param endDate the new end date for the quarantine. + * @return the updated quarantine object. + */ + PublicIpQuarantine updatePublicIpAddressInQuarantine(Long quarantineProcessId, Date endDate); + void updateSourceNatIpAddress(IPAddressVO requestedIp, List userIps) throws Exception; } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 28e7c89419a..1437457725a 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -192,6 +192,9 @@ public interface StorageManager extends StorageService { ConfigKey.Scope.Global, null); + ConfigKey HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000", + "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true); + /** * should we execute in sequence not involving any storages? * @return tru if commands should execute in sequence diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 2c129dfd6a5..3b3537d5488 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -34,6 +34,7 @@ import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachineProfile; @@ -116,7 +117,7 @@ public interface TemplateManager { Long getTemplateSize(long templateId, long zoneId); - DataStore getImageStore(String storeUuid, Long zoneId); + DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume); String getChecksum(DataStore store, String templatePath, String algorithm); diff --git a/engine/pom.xml b/engine/pom.xml index b2f41834403..6ecfb4bebf4 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -55,6 +55,7 @@ storage/configdrive storage/datamotion storage/image + storage/object storage/snapshot storage/volume userdata/cloud-init diff --git a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java index 6b8b754849c..cd4ac29738d 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java @@ -24,6 +24,7 @@ import javax.persistence.Id; import javax.persistence.Table; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.BooleanUtils; @Entity @Table(name = "host_tags") @@ -39,12 +40,22 @@ public class HostTagVO implements InternalIdentity { @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + protected HostTagVO() { } public HostTagVO(long hostId, String tag) { this.hostId = hostId; this.tag = tag; + this.isTagARule = false; + } + + public HostTagVO(long hostId, String tag, Boolean isTagARule) { + this.hostId = hostId; + this.tag = tag; + this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false); } public long getHostId() { @@ -59,6 +70,11 @@ public class HostTagVO implements InternalIdentity { this.tag = tag; } + public boolean getIsTagARule() { + return isTagARule; + } + + @Override public long getId() { return id; diff --git a/engine/schema/src/main/java/com/cloud/host/HostVO.java b/engine/schema/src/main/java/com/cloud/host/HostVO.java index d6e7ea1fd87..697401ad069 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostVO.java @@ -39,6 +39,7 @@ import javax.persistence.TemporalType; import javax.persistence.Transient; import com.cloud.agent.api.VgpuTypesInfo; +import com.cloud.host.dao.HostTagsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceState; @@ -46,7 +47,10 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.GenericDao; import java.util.Arrays; + +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang3.StringUtils; @Entity @@ -159,6 +163,14 @@ public class HostVO implements Host { @Transient List hostTags; + /** + * This is a delayed load value. + * If the value is null, then this field has not been loaded yet. + * Call host dao to load it. + */ + @Transient + Boolean isTagARule; + // This value is only for saving and current cannot be loaded. @Transient HashMap> groupDetails = new HashMap>(); @@ -322,8 +334,13 @@ public class HostVO implements Host { return hostTags; } - public void setHostTags(List hostTags) { + public void setHostTags(List hostTags, Boolean isTagARule) { this.hostTags = hostTags; + this.isTagARule = isTagARule; + } + + public Boolean getIsTagARule() { + return isTagARule; } public HashMap> getGpuGroupDetails() { @@ -748,6 +765,11 @@ public class HostVO implements Host { if (serviceOffering == null) { return false; } + + if (BooleanUtils.isTrue(this.getIsTagARule())) { + return TagAsRuleHelper.interpretTagAsRule(this.getHostTags().get(0), serviceOffering.getHostTag(), HostTagsDao.hostTagRuleExecutionTimeout.value()); + } + if (StringUtils.isEmpty(serviceOffering.getHostTag())) { return true; } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index ca51ad3f428..fe30722feb1 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -108,6 +108,8 @@ public interface HostDao extends GenericDao, StateDao listAllHostsByZoneAndHypervisorType(long zoneId, HypervisorType hypervisorType); + List listAllHostsThatHaveNoRuleTag(Host.Type type, Long clusterId, Long podId, Long dcId); + List listAllHostsByType(Host.Type type); HostVO findByPublicIp(String publicIp); @@ -161,4 +163,8 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); + + List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + + List findClustersThatMatchHostTagRule(String computeOfferingTags); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 392b623299f..136351527f6 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -22,9 +22,11 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; @@ -32,6 +34,8 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.persistence.TableGenerator; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.VgpuTypesInfo; @@ -80,8 +84,8 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao private static final Logger state_logger = Logger.getLogger(ResourceState.class); private static final String LIST_HOST_IDS_BY_COMPUTETAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count " - + "FROM (SELECT host_id, tag FROM host_tags GROUP BY host_id,tag) AS filtered " - + "WHERE tag IN(%s) " + + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag) AS filtered " + + "WHERE tag IN(%s) AND is_tag_a_rule = 0 " + "GROUP BY host_id " + "HAVING tag_count = %s "; private static final String SEPARATOR = ","; @@ -148,6 +152,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected GenericSearchBuilder AllClustersSearch; protected SearchBuilder HostsInClusterSearch; + protected SearchBuilder searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag; + + protected SearchBuilder searchBuilderFindByRuleTag; + protected Attribute _statusAttr; protected Attribute _resourceStateAttr; protected Attribute _msIdAttr; @@ -455,6 +463,22 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ); HostIdSearch.done(); + searchBuilderFindByRuleTag = _hostTagsDao.createSearchBuilder(); + searchBuilderFindByRuleTag.and("is_tag_a_rule", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.EQ); + searchBuilderFindByRuleTag.or("tagDoesNotExist", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.NULL); + + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag = createSearchBuilder(); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("type", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getType(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("cluster_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getClusterId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("pod_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getPodId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("data_center_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getDataCenterId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.join("id", searchBuilderFindByRuleTag, searchBuilderFindByRuleTag.entity().getHostId(), + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), JoinType.LEFTOUTER); + + searchBuilderFindByRuleTag.done(); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.done(); + _statusAttr = _allAttributes.get("status"); _msIdAttr = _allAttributes.get("managementServerId"); _pingTimeAttr = _allAttributes.get("lastPinged"); @@ -792,9 +816,12 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public List listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag) { - SearchBuilder hostTagSearch = null; + SearchBuilder hostTagSearch = _hostTagsDao.createSearchBuilder(); + hostTagSearch.and(); + hostTagSearch.op("isTagARule", hostTagSearch.entity().getIsTagARule(), Op.EQ); + hostTagSearch.or("tagDoesNotExist", hostTagSearch.entity().getIsTagARule(), Op.NULL); + hostTagSearch.cp(); if (haTag != null && !haTag.isEmpty()) { - hostTagSearch = _hostTagsDao.createSearchBuilder(); hostTagSearch.and().op("tag", hostTagSearch.entity().getTag(), SearchCriteria.Op.NEQ); hostTagSearch.or("tagNull", hostTagSearch.entity().getTag(), SearchCriteria.Op.NULL); hostTagSearch.cp(); @@ -809,12 +836,14 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao hostSearch.and("status", hostSearch.entity().getStatus(), SearchCriteria.Op.EQ); hostSearch.and("resourceState", hostSearch.entity().getResourceState(), SearchCriteria.Op.EQ); - if (haTag != null && !haTag.isEmpty()) { - hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER); - } + + hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER); + SearchCriteria sc = hostSearch.create(); + sc.setJoinParameters("hostTagSearch", "isTagARule", false); + if (haTag != null && !haTag.isEmpty()) { sc.setJoinParameters("hostTagSearch", "tag", haTag); } @@ -846,8 +875,13 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public void loadHostTags(HostVO host) { - List hostTags = _hostTagsDao.getHostTags(host.getId()); - host.setHostTags(hostTags); + List hostTagVOList = _hostTagsDao.getHostTags(host.getId()); + if (CollectionUtils.isNotEmpty(hostTagVOList)) { + List hostTagList = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); + host.setHostTags(hostTagList, hostTagVOList.get(0).getIsTagARule()); + } else { + host.setHostTags(null, null); + } } @DB @@ -881,10 +915,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected void saveHostTags(HostVO host) { List hostTags = host.getHostTags(); - if (hostTags == null || (hostTags != null && hostTags.isEmpty())) { + if (CollectionUtils.isEmpty(hostTags)) { return; } - _hostTagsDao.persist(host.getId(), hostTags); + _hostTagsDao.persist(host.getId(), hostTags, host.getIsTagARule()); } protected void saveGpuRecords(HostVO host) { @@ -1244,6 +1278,26 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public List listAllHostsThatHaveNoRuleTag(Type type, Long clusterId, Long podId, Long dcId) { + SearchCriteria sc = searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.create(); + if (type != null) { + sc.setParameters("type", type); + } + if (clusterId != null) { + sc.setParameters("cluster_id", clusterId); + } + if (podId != null) { + sc.setParameters("pod_id", podId); + } + if (dcId != null) { + sc.setParameters("data_center_id", dcId); + } + sc.setJoinParameters("id", "is_tag_a_rule", false); + + return search(sc, null); + } + @Override public List listClustersByHostTag(String computeOfferingTags) { TransactionLegacy txn = TransactionLegacy.currentTxn(); @@ -1266,9 +1320,6 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao result.add(rs.getLong(1)); } pstmt.close(); - if(result.isEmpty()){ - throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags); - } return result; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + sql, e); @@ -1293,15 +1344,33 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao result.add(rs.getLong(1)); } pstmt.close(); - if(result.isEmpty()){ - throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags); - } return result; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + select, e); } } + public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + List hostTagVOList = _hostTagsDao.findHostRuleTags(); + List result = new ArrayList<>(); + for (HostTagVO rule: hostTagVOList) { + if (TagAsRuleHelper.interpretTagAsRule(rule.getTag(), computeOfferingTags, HostTagsDao.hostTagRuleExecutionTimeout.value())) { + result.add(findById(rule.getHostId())); + } + } + + return result; + } + + public List findClustersThatMatchHostTagRule(String computeOfferingTags) { + Set result = new HashSet<>(); + List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + for (HostVO host: hosts) { + result.add(host.getClusterId()); + } + return new ArrayList<>(result); + } + private String getHostIdsByComputeTags(List offeringTags){ List questionMarks = new ArrayList(); offeringTags.forEach((tag) -> { questionMarks.add("?"); }); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 0fb5370d81a..d134db33403 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -20,15 +20,21 @@ import java.util.List; import com.cloud.host.HostTagVO; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.framework.config.ConfigKey; public interface HostTagsDao extends GenericDao { - void persist(long hostId, List hostTags); + ConfigKey hostTagRuleExecutionTimeout = new ConfigKey<>("Advanced", Long.class, "host.tag.rule.execution.timeout", "2000", "The maximum runtime, in milliseconds, " + + "to execute a host tag rule; if it is reached, a timeout will happen.", true); - List getHostTags(long hostId); + void persist(long hostId, List hostTags, Boolean isTagARule); + + List getHostTags(long hostId); List getDistinctImplicitHostTags(List hostIds, String[] implicitHostTags); void deleteTags(long hostId); + List findHostRuleTags(); + } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index a73899b7b33..65deb1d1c9b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -16,10 +16,10 @@ // under the License. package com.cloud.host.dao; -import java.util.ArrayList; import java.util.List; - +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.springframework.stereotype.Component; import com.cloud.host.HostTagVO; @@ -31,13 +31,14 @@ import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Func; @Component -public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao { +public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao, Configurable { protected final SearchBuilder HostSearch; protected final GenericSearchBuilder DistinctImplictTagsSearch; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ); + HostSearch.and("isTagARule", HostSearch.entity().getIsTagARule(), SearchCriteria.Op.EQ); HostSearch.done(); DistinctImplictTagsSearch = createSearchBuilder(String.class); @@ -48,17 +49,11 @@ public class HostTagsDaoImpl extends GenericDaoBase implements } @Override - public List getHostTags(long hostId) { + public List getHostTags(long hostId) { SearchCriteria sc = HostSearch.create(); sc.setParameters("hostId", hostId); - List results = search(sc, null); - List hostTags = new ArrayList(results.size()); - for (HostTagVO result : results) { - hostTags.add(result.getTag()); - } - - return hostTags; + return search(sc, null); } @Override @@ -80,7 +75,15 @@ public class HostTagsDaoImpl extends GenericDaoBase implements } @Override - public void persist(long hostId, List hostTags) { + public List findHostRuleTags() { + SearchCriteria sc = HostSearch.create(); + sc.setParameters("isTagARule", true); + + return search(sc, null); + } + + @Override + public void persist(long hostId, List hostTags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); @@ -91,10 +94,20 @@ public class HostTagsDaoImpl extends GenericDaoBase implements for (String tag : hostTags) { tag = tag.trim(); if (tag.length() > 0) { - HostTagVO vo = new HostTagVO(hostId, tag); + HostTagVO vo = new HostTagVO(hostId, tag, isTagARule); persist(vo); } } txn.commit(); } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {hostTagRuleExecutionTimeout}; + } + + @Override + public String getConfigComponentName() { + return HostTagsDaoImpl.class.getSimpleName(); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java index bb4822ebb38..f75dc8a6661 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDao.java @@ -21,6 +21,7 @@ import java.util.List; import com.cloud.dc.Vlan.VlanType; import com.cloud.network.IpAddress.State; import com.cloud.utils.db.GenericDao; +import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.net.Ip; public interface IPAddressDao extends GenericDao { @@ -100,4 +101,6 @@ public interface IPAddressDao extends GenericDao { List listByDcIdAndAssociatedNetwork(long dcId); List listByNetworkId(long networkId); + + void buildQuarantineSearchCriteria(SearchCriteria sc); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java index d82ffbe2af0..9ffc4c9159c 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressDaoImpl.java @@ -24,6 +24,7 @@ import java.util.List; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -32,6 +33,7 @@ import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.VlanVO; import com.cloud.dc.dao.VlanDao; import com.cloud.network.IpAddress.State; +import com.cloud.network.vo.PublicIpQuarantineVO; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.utils.db.DB; @@ -69,6 +71,9 @@ public class IPAddressDaoImpl extends GenericDaoBase implemen @Inject UserIpAddressDetailsDao _detailsDao; + @Inject + PublicIpQuarantineDao publicIpQuarantineDao; + // make it public for JUnit test public IPAddressDaoImpl() { } @@ -534,4 +539,19 @@ public class IPAddressDaoImpl extends GenericDaoBase implemen sc.setParameters("state", State.Allocated); return listBy(sc); } + + @Override + public void buildQuarantineSearchCriteria(SearchCriteria sc) { + long accountId = CallContext.current().getCallingAccount().getAccountId(); + SearchBuilder listAllIpsInQuarantine = publicIpQuarantineDao.createSearchBuilder(); + listAllIpsInQuarantine.and("quarantineEndDate", listAllIpsInQuarantine.entity().getEndDate(), SearchCriteria.Op.GT); + listAllIpsInQuarantine.and("previousOwnerId", listAllIpsInQuarantine.entity().getPreviousOwnerId(), Op.NEQ); + + SearchCriteria searchCriteria = listAllIpsInQuarantine.create(); + searchCriteria.setParameters("quarantineEndDate", new Date()); + searchCriteria.setParameters("previousOwnerId", accountId); + Object[] quarantinedIpsIdsAllowedToUser = publicIpQuarantineDao.search(searchCriteria, null).stream().map(PublicIpQuarantineVO::getPublicIpAddressId).toArray(); + + sc.setParametersIfNotNull("quarantinedPublicIpsIdsNIN", quarantinedIpsIdsAllowedToUser); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java index 7c4d56bd1ee..4c7569a55b9 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/IPAddressVO.java @@ -29,7 +29,6 @@ import javax.persistence.Id; import javax.persistence.Table; import javax.persistence.Temporal; import javax.persistence.TemporalType; -import javax.persistence.Transient; import com.cloud.network.IpAddress; import com.cloud.utils.db.GenericDao; @@ -97,14 +96,6 @@ public class IPAddressVO implements IpAddress { @Column(name = "is_system") private boolean system; - @Column(name = "account_id") - @Transient - private Long accountId = null; - - @Transient - @Column(name = "domain_id") - private Long domainId = null; - @Column(name = "vpc_id") private Long vpcId; diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java new file mode 100644 index 00000000000..ccba6bb1889 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import com.cloud.network.vo.PublicIpQuarantineVO; +import com.cloud.utils.db.GenericDao; + +public interface PublicIpQuarantineDao extends GenericDao { + + PublicIpQuarantineVO findByPublicIpAddressId(long publicIpAddressId); + + PublicIpQuarantineVO findByIpAddress(String publicIpAddress); +} diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java new file mode 100644 index 00000000000..a1b789b8a46 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.dao; + +import com.cloud.network.vo.PublicIpQuarantineVO; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.JoinBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; + +@Component +public class PublicIpQuarantineDaoImpl extends GenericDaoBase implements PublicIpQuarantineDao { + private SearchBuilder publicIpAddressByIdSearch; + + private SearchBuilder ipAddressSearchBuilder; + + @Inject + IPAddressDao ipAddressDao; + + @PostConstruct + public void init() { + publicIpAddressByIdSearch = createSearchBuilder(); + publicIpAddressByIdSearch.and("publicIpAddressId", publicIpAddressByIdSearch.entity().getPublicIpAddressId(), SearchCriteria.Op.EQ); + + ipAddressSearchBuilder = ipAddressDao.createSearchBuilder(); + ipAddressSearchBuilder.and("ipAddress", ipAddressSearchBuilder.entity().getAddress(), SearchCriteria.Op.EQ); + ipAddressSearchBuilder.and("removed", ipAddressSearchBuilder.entity().getRemoved(), SearchCriteria.Op.NULL); + publicIpAddressByIdSearch.join("quarantineJoin", ipAddressSearchBuilder, ipAddressSearchBuilder.entity().getId(), + publicIpAddressByIdSearch.entity().getPublicIpAddressId(), JoinBuilder.JoinType.INNER); + + ipAddressSearchBuilder.done(); + publicIpAddressByIdSearch.done(); + } + + @Override + public PublicIpQuarantineVO findByPublicIpAddressId(long publicIpAddressId) { + SearchCriteria sc = publicIpAddressByIdSearch.create(); + sc.setParameters("publicIpAddressId", publicIpAddressId); + final Filter filter = new Filter(PublicIpQuarantineVO.class, "created", false); + + return findOneBy(sc, filter); + } + + @Override + public PublicIpQuarantineVO findByIpAddress(String publicIpAddress) { + SearchCriteria sc = publicIpAddressByIdSearch.create(); + sc.setJoinParameters("quarantineJoin", "ipAddress", publicIpAddress); + final Filter filter = new Filter(PublicIpQuarantineVO.class, "created", false); + + return findOneBy(sc, filter); + } +} diff --git a/engine/schema/src/main/java/com/cloud/network/vo/PublicIpQuarantineVO.java b/engine/schema/src/main/java/com/cloud/network/vo/PublicIpQuarantineVO.java new file mode 100644 index 00000000000..56d167a0060 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/network/vo/PublicIpQuarantineVO.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.network.vo; + +import com.cloud.network.PublicIpQuarantine; +import com.cloud.utils.db.GenericDao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "quarantined_ips") +public class PublicIpQuarantineVO implements PublicIpQuarantine { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Long id; + + @Column(name = "uuid", nullable = false) + private String uuid = UUID.randomUUID().toString(); + + @Column(name = "public_ip_address_id", nullable = false) + private Long publicIpAddressId; + + @Column(name = "previous_owner_id", nullable = false) + private Long previousOwnerId; + + @Column(name = GenericDao.CREATED_COLUMN, nullable = false) + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed = null; + + @Column(name = "end_date", nullable = false) + @Temporal(value = TemporalType.TIMESTAMP) + private Date endDate; + + @Column(name = "removal_reason") + private String removalReason = null; + + public PublicIpQuarantineVO() { + } + + public PublicIpQuarantineVO(Long publicIpAddressId, Long previousOwnerId, Date created, Date endDate) { + this.publicIpAddressId = publicIpAddressId; + this.previousOwnerId = previousOwnerId; + this.created = created; + this.endDate = endDate; + } + + @Override + public long getId() { + return id; + } + + @Override + public Long getPublicIpAddressId() { + return publicIpAddressId; + } + + @Override + public Long getPreviousOwnerId() { + return previousOwnerId; + } + + @Override + public Date getEndDate() { + return endDate; + } + + @Override + public String getRemovalReason() { + return removalReason; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public void setRemovalReason(String removalReason) { + this.removalReason = removalReason; + } + + @Override + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + @Override + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } +} diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java index 4eaa2b575e0..280d5dfaf4b 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java @@ -17,6 +17,8 @@ package com.cloud.network.vpc; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + import java.util.UUID; import javax.persistence.Column; @@ -85,6 +87,11 @@ public class NetworkACLVO implements NetworkACL { return name; } + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "uuid", "name", "vpcId"); + } + public void setUuid(String uuid) { this.uuid = uuid; } diff --git a/engine/schema/src/main/java/com/cloud/storage/BucketVO.java b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java new file mode 100644 index 00000000000..181b02e5a1b --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage; + +import com.cloud.utils.db.GenericDao; +import com.google.gson.annotations.Expose; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "bucket") +public class BucketVO implements Bucket { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "account_id") + long accountId; + + @Column(name = "domain_id") + long domainId; + + @Column(name = "object_store_id") + long objectStoreId; + + @Expose + @Column(name = "name") + String name; + + @Expose + @Column(name = "state", updatable = true, nullable = false) + @Enumerated(value = EnumType.STRING) + private State state; + + @Column(name = "size") + Long size; + + @Column(name = "quota") + Integer quota; + + @Column(name = "versioning") + boolean versioning; + + @Column(name = "encryption") + boolean encryption; + + @Column(name = "object_lock") + boolean objectLock; + + @Column(name = "policy") + String policy; + + @Column(name = "bucket_url") + String bucketURL; + + @Column(name = "access_key") + String accessKey; + + @Column(name = "secret_key") + String secretKey; + + @Column(name = GenericDao.CREATED_COLUMN) + Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + Date removed; + + @Column(name = "uuid") + String uuid; + + public BucketVO() { + } + + public BucketVO(long accountId, long domainId, long objectStoreId, String name, Integer quota, boolean versioning, + boolean encryption, boolean objectLock, String policy) + { + this.accountId = accountId; + this.domainId = domainId; + this.objectStoreId = objectStoreId; + this.name = name; + state = State.Allocated; + uuid = UUID.randomUUID().toString(); + this.quota = quota; + this.versioning = versioning; + this.encryption = encryption; + this.objectLock = objectLock; + this.policy = policy; + this.size = 0L; + } + + @Override + public long getId() { + return id; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getObjectStoreId() { + return objectStoreId; + } + + @Override + public String getName() { + return name; + } + + public Long getSize() { + return size; + } + + @Override + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + @Override + public State getState() { + return state; + } + + @Override + public void setName(String name) { + this.name = name; + } + + public void setState(State state) { + this.state = state; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public Integer getQuota() { + return quota; + } + + public void setQuota(Integer quota) { + this.quota = quota; + } + + public boolean isVersioning() { + return versioning; + } + + public void setVersioning(boolean versioning) { + this.versioning = versioning; + } + + public boolean isEncryption() { + return encryption; + } + + public void setEncryption(boolean encryption) { + this.encryption = encryption; + } + + public boolean isObjectLock() { + return objectLock; + } + + public void setObjectLock(boolean objectLock) { + this.objectLock = objectLock; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(String policy) { + this.policy = policy; + } + + public String getBucketURL() { + return bucketURL; + } + public void setBucketURL(String bucketURL) { + this.bucketURL = bucketURL; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setSize(Long size) { + this.size = size; + } + + @Override + public Class getEntityType() { + return Bucket.class; + } + + @Override + public String toString() { + return String.format("Bucket %s", new ToStringBuilder(this, ToStringStyle.JSON_STYLE).append("uuid", getUuid()).append("name", getName()) + .append("ObjectStoreId", getObjectStoreId()).toString()); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java b/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java index 18c0dc3c326..2675c36f27f 100755 --- a/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java @@ -23,7 +23,9 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import com.cloud.utils.NumbersUtil; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.BooleanUtils; @Entity @Table(name = "storage_pool_tags") @@ -43,9 +45,19 @@ public class StoragePoolTagVO implements InternalIdentity { @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + public StoragePoolTagVO(long poolId, String tag) { this.poolId = poolId; this.tag = tag; + this.isTagARule = false; + } + + public StoragePoolTagVO(long poolId, String tag, Boolean isTagARule) { + this.poolId = poolId; + this.tag = tag; + this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false); } @Override @@ -61,4 +73,20 @@ public class StoragePoolTagVO implements InternalIdentity { return tag; } + public boolean isTagARule() { + return this.isTagARule; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof StoragePoolTagVO) { + return this.poolId == ((StoragePoolTagVO)obj).getPoolId(); + } + return false; + } + + @Override + public int hashCode() { + return NumbersUtil.hash(id); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java new file mode 100644 index 00000000000..f45f28b5c2c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.dao; + +import com.cloud.storage.BucketVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface BucketDao extends GenericDao { + List listByObjectStoreId(long objectStoreId); + + List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId); + + List searchByIds(Long[] ids); +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java new file mode 100644 index 00000000000..83b5f6bdb74 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.dao; + +import com.cloud.storage.BucketVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.naming.ConfigurationException; +import java.util.List; +import java.util.Map; + +@Component +public class BucketDaoImpl extends GenericDaoBase implements BucketDao { + public static final Logger s_logger = Logger.getLogger(BucketDaoImpl.class.getName()); + private SearchBuilder searchFilteringStoreId; + + private SearchBuilder bucketSearch; + + private static final String STORE_ID = "store_id"; + private static final String STATE = "state"; + private static final String ACCOUNT_ID = "account_id"; + + protected BucketDaoImpl() { + + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + searchFilteringStoreId = createSearchBuilder(); + searchFilteringStoreId.and(STORE_ID, searchFilteringStoreId.entity().getObjectStoreId(), SearchCriteria.Op.EQ); + searchFilteringStoreId.and(ACCOUNT_ID, searchFilteringStoreId.entity().getAccountId(), SearchCriteria.Op.EQ); + searchFilteringStoreId.and(STATE, searchFilteringStoreId.entity().getState(), SearchCriteria.Op.NEQ); + searchFilteringStoreId.done(); + + bucketSearch = createSearchBuilder(); + bucketSearch.and("idIN", bucketSearch.entity().getId(), SearchCriteria.Op.IN); + bucketSearch.done(); + + return true; + } + @Override + public List listByObjectStoreId(long objectStoreId) { + SearchCriteria sc = searchFilteringStoreId.create(); + sc.setParameters(STORE_ID, objectStoreId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return listBy(sc); + } + + @Override + public List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId) { + SearchCriteria sc = searchFilteringStoreId.create(); + sc.setParameters(STORE_ID, objectStoreId); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return listBy(sc); + } + + @Override + public List searchByIds(Long[] ids) { + SearchCriteria sc = bucketSearch.create(); + sc.setParameters("idIN", ids); + return search(sc, null, null, false); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java index 946b46b5cfe..9352ee21858 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java @@ -25,10 +25,14 @@ import com.cloud.utils.db.GenericDao; public interface StoragePoolTagsDao extends GenericDao { - void persist(long poolId, List storagePoolTags); + void persist(long poolId, List storagePoolTags, Boolean isTagARule); + + void persist(List storagePoolTags); List getStoragePoolTags(long poolId); void deleteTags(long poolId); List searchByIds(Long... stIds); StorageTagResponse newStorageTagResponse(StoragePoolTagVO tag); + List findStoragePoolTags(long poolId); + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java index f20e0c411de..c01c66763af 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java @@ -21,6 +21,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -50,7 +53,7 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase storagePoolTags) { + public void persist(long poolId, List storagePoolTags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); @@ -61,13 +64,23 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase 0) { - StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag); + StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag, isTagARule); persist(vo); } } txn.commit(); } + public void persist(List storagePoolTags) { + Transaction.execute(TransactionLegacy.CLOUD_DB, new TransactionCallbackNoReturn() { + @Override public void doInTransactionWithoutResult(TransactionStatus status) { + for (StoragePoolTagVO storagePoolTagVO : storagePoolTags) { + persist(storagePoolTagVO); + } + } + }); + } + @Override public List getStoragePoolTags(long poolId) { SearchCriteria sc = StoragePoolSearch.create(); @@ -157,4 +170,12 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase findStoragePoolTags(long poolId) { + SearchCriteria sc = StoragePoolSearch.create(); + sc.setParameters("poolId", poolId); + + return search(sc, null); + } + } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index 21331f654e3..697782dc1db 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -22,15 +22,18 @@ import static com.google.common.collect.ObjectArrays.concat; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; import java.util.Date; +import java.util.List; import javax.inject.Inject; +import com.cloud.utils.FileUtil; import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -123,6 +126,7 @@ import com.google.common.annotations.VisibleForTesting; public class DatabaseUpgradeChecker implements SystemIntegrityChecker { private static final Logger s_logger = Logger.getLogger(DatabaseUpgradeChecker.class); private final DatabaseVersionHierarchy hierarchy; + private static final String VIEWS_DIRECTORY = Paths.get("META-INF", "db", "views").toString(); @Inject VersionDao _dao; @@ -363,9 +367,33 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker { txn.close(); } } + + executeViewScripts(); updateSystemVmTemplates(upgrades); } + protected void executeViewScripts() { + s_logger.info(String.format("Executing VIEW scripts that are under resource directory [%s].", VIEWS_DIRECTORY)); + List filesPathUnderViewsDirectory = FileUtil.getFilesPathsUnderResourceDirectory(VIEWS_DIRECTORY); + + try (TransactionLegacy txn = TransactionLegacy.open("execute-view-scripts")) { + Connection conn = txn.getConnection(); + + for (String filePath : filesPathUnderViewsDirectory) { + s_logger.debug(String.format("Executing VIEW script [%s].", filePath)); + + InputStream viewScript = Thread.currentThread().getContextClassLoader().getResourceAsStream(filePath); + runScript(conn, viewScript); + } + + s_logger.info(String.format("Finished execution of VIEW scripts that are under resource directory [%s].", VIEWS_DIRECTORY)); + } catch (SQLException e) { + String message = String.format("Unable to execute VIEW scripts due to [%s].", e.getMessage()); + s_logger.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + @Override public void check() { GlobalLock lock = GlobalLock.getInternLock("DatabaseUpgrade"); diff --git a/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java b/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java new file mode 100644 index 00000000000..ab5fcfc493c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage; + +import org.apache.cloudstack.api.InternalIdentity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "bucket_statistics") +public class BucketStatisticsVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private Long id; + + @Column(name = "account_id", updatable = false) + private long accountId; + + @Column(name = "bucket_id", updatable = false) + private long bucketId; + + @Column(name = "size") + private long size; + + protected BucketStatisticsVO() { + } + + public BucketStatisticsVO(long accountId, long bucketId) { + this.accountId = accountId; + this.bucketId = bucketId; + } + + public long getAccountId() { + return accountId; + } + + public long getBucketId() { + return bucketId; + } + + @Override + public long getId() { + return id; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java new file mode 100644 index 00000000000..48388af9446 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface BucketStatisticsDao extends GenericDao { + BucketStatisticsVO findBy(long accountId, long bucketId); + + BucketStatisticsVO lock(long accountId, long bucketId); + + List listBy(long accountId); +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java new file mode 100644 index 00000000000..2261389eab6 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class BucketStatisticsDaoImpl extends GenericDaoBase implements BucketStatisticsDao { + private static final Logger s_logger = Logger.getLogger(BucketStatisticsDaoImpl.class); + private final SearchBuilder AllFieldsSearch; + private final SearchBuilder AccountSearch; + + public BucketStatisticsDaoImpl() { + AccountSearch = createSearchBuilder(); + AccountSearch.and("account", AccountSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AccountSearch.done(); + + AllFieldsSearch = createSearchBuilder(); + AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AllFieldsSearch.and("bucket", AllFieldsSearch.entity().getBucketId(), SearchCriteria.Op.EQ); + AllFieldsSearch.done(); + } + + @Override + public BucketStatisticsVO findBy(long accountId, long bucketId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("account", accountId); + sc.setParameters("bucket", bucketId); + return findOneBy(sc); + } + + @Override + public BucketStatisticsVO lock(long accountId, long bucketId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("account", accountId); + sc.setParameters("bucket", bucketId); + return lockOneRandomRow(sc, true); + } + + @Override + public List listBy(long accountId) { + SearchCriteria sc = AccountSearch.create(); + sc.setParameters("account", accountId); + return search(sc, null); + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java index 4099b3ada0d..ea490e60f9e 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.dao; +import com.cloud.usage.BucketStatisticsVO; import com.cloud.usage.UsageVO; import com.cloud.user.AccountVO; import com.cloud.user.UserStatisticsVO; @@ -45,6 +46,12 @@ public interface UsageDao extends GenericDao { Long getLastUserStatsId(); + Long getLastBucketStatsId(); + + void saveBucketStats(List userStats); + + void updateBucketStats(List userStats); + List listPublicTemplatesByAccount(long accountId); Long getLastVmDiskStatsId(); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java index 4553ed822b4..0d9e727abe2 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.dao; +import com.cloud.usage.BucketStatisticsVO; import com.cloud.usage.UsageVO; import com.cloud.user.AccountVO; import com.cloud.user.UserStatisticsVO; @@ -82,6 +83,13 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage + "WHERE cloud_usage.usage_type = ? AND cloud_usage.account_id = ? AND cloud_usage.start_date >= ? AND cloud_usage.end_date <= ? " + "GROUP BY cloud_usage.usage_id "; + private static final String GET_LAST_BUCKET_STATS_ID = "SELECT id FROM cloud_usage.bucket_statistics ORDER BY id DESC LIMIT 1"; + + private static final String INSERT_BUCKET_STATS = "INSERT INTO cloud_usage.bucket_statistics (id, account_id, bucket_id, size) VALUES (?,?,?,?)"; + + private static final String UPDATE_BUCKET_STATS = "UPDATE cloud_usage.bucket_statistics SET size=? WHERE id=?"; + + public UsageDaoImpl() { } @@ -285,6 +293,69 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage return null; } + @Override + public Long getLastBucketStatsId() { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + PreparedStatement pstmt = null; + String sql = GET_LAST_BUCKET_STATS_ID; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return Long.valueOf(rs.getLong(1)); + } + } catch (Exception ex) { + s_logger.error("error getting last bucket stats id", ex); + } + return null; + } + + @Override + public void saveBucketStats(List bucketStats) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + txn.start(); + String sql = INSERT_BUCKET_STATS; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); // in reality I just want CLOUD_USAGE dataSource connection + for (BucketStatisticsVO bucketStat : bucketStats) { + pstmt.setLong(1, bucketStat.getId()); + pstmt.setLong(2, bucketStat.getAccountId()); + pstmt.setLong(3, bucketStat.getBucketId()); + pstmt.setLong(4, bucketStat.getSize()); + pstmt.addBatch(); + } + pstmt.executeBatch(); + txn.commit(); + } catch (Exception ex) { + txn.rollback(); + s_logger.error("error saving bucket stats to cloud_usage db", ex); + throw new CloudRuntimeException(ex.getMessage()); + } + } + + @Override + public void updateBucketStats(List bucketStats) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + txn.start(); + String sql = UPDATE_BUCKET_STATS; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); // in reality I just want CLOUD_USAGE dataSource connection + for (BucketStatisticsVO bucketStat : bucketStats) { + pstmt.setLong(1, bucketStat.getSize()); + pstmt.setLong(2, bucketStat.getId()); + pstmt.addBatch(); + } + pstmt.executeBatch(); + txn.commit(); + } catch (Exception ex) { + txn.rollback(); + s_logger.error("error updating bucket stats to cloud_usage db", ex); + throw new CloudRuntimeException(ex.getMessage()); + } + } + @Override public List listPublicTemplatesByAccount(long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 22165641f7e..68f57329d77 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -79,7 +79,9 @@ public interface NicDao extends GenericDao { List listByNetworkIdTypeAndGatewayAndBroadcastUri(long networkId, VirtualMachine.Type vmType, String gateway, URI broadcastUri); - int countNicsForStartingVms(long networkId); + int countNicsForNonStoppedVms(long networkId); + + int countNicsForNonStoppedRunningVrs(long networkId); NicVO getControlNicForVM(long vmId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index 211e5480423..59d2417b073 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -44,7 +44,7 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { private GenericSearchBuilder IpSearch; private SearchBuilder NonReleasedSearch; private GenericSearchBuilder deviceIdSearch; - private GenericSearchBuilder CountByForStartingVms; + private GenericSearchBuilder CountByForNonStoppedVms; private SearchBuilder PeerRouterSearch; @Inject @@ -91,14 +91,16 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { deviceIdSearch.and("instance", deviceIdSearch.entity().getInstanceId(), Op.EQ); deviceIdSearch.done(); - CountByForStartingVms = createSearchBuilder(Integer.class); - CountByForStartingVms.select(null, Func.COUNT, CountByForStartingVms.entity().getId()); - CountByForStartingVms.and("networkId", CountByForStartingVms.entity().getNetworkId(), Op.EQ); - CountByForStartingVms.and("removed", CountByForStartingVms.entity().getRemoved(), Op.NULL); + CountByForNonStoppedVms = createSearchBuilder(Integer.class); + CountByForNonStoppedVms.select(null, Func.COUNT, CountByForNonStoppedVms.entity().getId()); + CountByForNonStoppedVms.and("vmType", CountByForNonStoppedVms.entity().getVmType(), Op.EQ); + CountByForNonStoppedVms.and("vmTypeNEQ", CountByForNonStoppedVms.entity().getVmType(), Op.NEQ); + CountByForNonStoppedVms.and("networkId", CountByForNonStoppedVms.entity().getNetworkId(), Op.EQ); + CountByForNonStoppedVms.and("removed", CountByForNonStoppedVms.entity().getRemoved(), Op.NULL); SearchBuilder join1 = _vmDao.createSearchBuilder(); - join1.and("state", join1.entity().getState(), Op.EQ); - CountByForStartingVms.join("vm", join1, CountByForStartingVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); - CountByForStartingVms.done(); + join1.and("state", join1.entity().getState(), Op.IN); + CountByForNonStoppedVms.join("vm", join1, CountByForNonStoppedVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); + CountByForNonStoppedVms.done(); PeerRouterSearch = createSearchBuilder(); PeerRouterSearch.and("instanceId", PeerRouterSearch.entity().getInstanceId(), Op.NEQ); @@ -346,10 +348,21 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { } @Override - public int countNicsForStartingVms(long networkId) { - SearchCriteria sc = CountByForStartingVms.create(); + public int countNicsForNonStoppedVms(long networkId) { + SearchCriteria sc = CountByForNonStoppedVms.create(); sc.setParameters("networkId", networkId); - sc.setJoinParameters("vm", "state", VirtualMachine.State.Starting); + sc.setParameters("vmType", VirtualMachine.Type.User); + sc.setJoinParameters("vm", "state", new Object[] {VirtualMachine.State.Starting, VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Migrating}); + List results = customSearch(sc, null); + return results.get(0); + } + + @Override + public int countNicsForNonStoppedRunningVrs(long networkId) { + SearchCriteria sc = CountByForNonStoppedVms.create(); + sc.setParameters("networkId", networkId); + sc.setParameters("vmTypeNEQ", VirtualMachine.Type.User); + sc.setJoinParameters("vm", "state", new Object[] {VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Migrating}); List results = customSearch(sc, null); return results.get(0); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java new file mode 100644 index 00000000000..b0da0c5e747 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "heuristics") +public class HeuristicVO implements Heuristic { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Long id; + + @Column(name = "uuid", nullable = false) + private String uuid = UUID.randomUUID().toString(); + + @Column(name = "name") + private String name; + + @Column(name = "description") + private String description; + + @Column(name = "zone_id", nullable = false) + private Long zoneId; + + @Column(name = "type", nullable = false) + private String type; + + @Column(name = "heuristic_rule", nullable = false, length = 65535) + private String heuristicRule; + + @Column(name = GenericDao.CREATED_COLUMN, nullable = false) + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed = null; + + public HeuristicVO() { + } + + public HeuristicVO(String name, String description, Long zoneId, String type, String heuristicRule) { + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + } + + @Override + public long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "heuristicRule", "type"); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java new file mode 100644 index 00000000000..1c4057a7beb --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; + +public interface SecondaryStorageHeuristicDao extends GenericDao { + + HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java new file mode 100644 index 00000000000..0b51b2aec6b --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class SecondaryStorageHeuristicDaoImpl extends GenericDaoBase implements SecondaryStorageHeuristicDao { + private SearchBuilder zoneAndTypeSearch; + + @PostConstruct + public void init() { + zoneAndTypeSearch = createSearchBuilder(); + zoneAndTypeSearch.and("zoneId", zoneAndTypeSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + zoneAndTypeSearch.and("type", zoneAndTypeSearch.entity().getType(), SearchCriteria.Op.IN); + zoneAndTypeSearch.done(); + } + + @Override + public HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type) { + SearchCriteria searchCriteria = zoneAndTypeSearch.create(); + searchCriteria.setParameters("zoneId", zoneId); + searchCriteria.setParameters("type", type.toString()); + final Filter filter = new Filter(HeuristicVO.class, "created", false); + + return findOneBy(searchCriteria, filter); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index ba9825c3c86..1c31b3e0cc4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -49,4 +49,6 @@ public interface ImageStoreDao extends GenericDao { List findByProtocol(String protocol); ImageStoreVO findOneByZoneAndProtocol(long zoneId, String protocol); + + List listImageStoresByZoneIds(Long... zoneIds); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java index 3468b6008d9..a4827a1beae 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java @@ -43,6 +43,8 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem private SearchBuilder protocolSearch; private SearchBuilder zoneProtocolSearch; + private SearchBuilder zonesInSearch; + public ImageStoreDaoImpl() { super(); protocolSearch = createSearchBuilder(); @@ -55,6 +57,11 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem zoneProtocolSearch.and("protocol", zoneProtocolSearch.entity().getProtocol(), SearchCriteria.Op.EQ); zoneProtocolSearch.and("role", zoneProtocolSearch.entity().getRole(), SearchCriteria.Op.EQ); zoneProtocolSearch.done(); + + zonesInSearch = createSearchBuilder(); + zonesInSearch.and("zonesIn", zonesInSearch.entity().getDcId(), SearchCriteria.Op.IN); + zonesInSearch.done(); + } @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -191,4 +198,12 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem List results = listBy(sc, filter); return results.size() == 0 ? null : results.get(0); } + + + @Override + public List listImageStoresByZoneIds(Long... zoneIds) { + SearchCriteria sc = zonesInSearch.create(); + sc.setParametersIfNotNull("zonesIn", zoneIds); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java new file mode 100644 index 00000000000..94f6b5ec372 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.api.response.ObjectStoreResponse; + +import java.util.List; + +public interface ObjectStoreDao extends GenericDao { + ObjectStoreVO findByName(String name); + + List findByProvider(String provider); + + ObjectStoreVO findByUrl(String url); + + List listObjectStores(); + + List searchByIds(Long[] osIds); + + ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store); + + ObjectStoreResponse setObjectStoreResponse(ObjectStoreResponse storeData, ObjectStoreVO store); + + Integer countAllObjectStores(); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java new file mode 100644 index 00000000000..51abde013b6 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreDaoImpl extends GenericDaoBase implements ObjectStoreDao { + private SearchBuilder nameSearch; + private SearchBuilder providerSearch; + @Inject + private ConfigurationDao _configDao; + private final SearchBuilder osSearch; + + private SearchBuilder urlSearch; + + protected ObjectStoreDaoImpl() { + osSearch = createSearchBuilder(); + osSearch.and("idIN", osSearch.entity().getId(), SearchCriteria.Op.IN); + osSearch.done(); + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + nameSearch = createSearchBuilder(); + nameSearch.and("name", nameSearch.entity().getName(), SearchCriteria.Op.EQ); + nameSearch.done(); + + providerSearch = createSearchBuilder(); + providerSearch.and("providerName", providerSearch.entity().getProviderName(), SearchCriteria.Op.EQ); + providerSearch.done(); + + urlSearch = createSearchBuilder(); + urlSearch.and("url", urlSearch.entity().getUrl(), SearchCriteria.Op.EQ); + urlSearch.done(); + + return true; + } + + @Override + public ObjectStoreVO findByName(String name) { + SearchCriteria sc = nameSearch.create(); + sc.setParameters("name", name); + return findOneBy(sc); + } + + @Override + public List findByProvider(String provider) { + SearchCriteria sc = providerSearch.create(); + sc.setParameters("providerName", provider); + return listBy(sc); + } + + @Override + public ObjectStoreVO findByUrl(String url) { + SearchCriteria sc = urlSearch.create(); + sc.setParameters("url", url); + return findOneBy(sc); + } + + @Override + public List listObjectStores() { + SearchCriteria sc = createSearchCriteria(); + return listBy(sc); + } + + @Override + public List searchByIds(Long[] osIds) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = _configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + // query details by batches + List osList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (osIds.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= osIds.length) { + Long[] ids = new Long[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + ids[k] = osIds[j]; + } + SearchCriteria sc = osSearch.create(); + sc.setParameters("idIN", ids); + List stores = searchIncludingRemoved(sc, null, null, false); + if (stores != null) { + osList.addAll(stores); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < osIds.length) { + int batch_size = (osIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = osIds[j]; + } + SearchCriteria sc = osSearch.create(); + sc.setParameters("idIN", ids); + List stores = searchIncludingRemoved(sc, null, null, false); + if (stores != null) { + osList.addAll(stores); + } + } + return osList; + } + + @Override + public ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store) { + ObjectStoreResponse osResponse = new ObjectStoreResponse(); + osResponse.setId(store.getUuid()); + osResponse.setName(store.getName()); + osResponse.setProviderName(store.getProviderName()); + String url = store.getUrl(); + osResponse.setUrl(url); + osResponse.setObjectName("objectstore"); + return osResponse; + } + + @Override + public ObjectStoreResponse setObjectStoreResponse(ObjectStoreResponse storeData, ObjectStoreVO store) { + return storeData; + } + + @Override + public Integer countAllObjectStores() { + SearchCriteria sc = createSearchCriteria(); + return getCount(sc); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java new file mode 100644 index 00000000000..1f4047f8f90 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import org.apache.cloudstack.api.ResourceDetail; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "object_store_details") +public class ObjectStoreDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "store_id") + long resourceId; + + @Column(name = "name") + String name; + + @Column(name = "value") + String value; + + public ObjectStoreDetailVO() { + } + public ObjectStoreDetailVO(long storeId, String name, String value) { + this.resourceId = storeId; + this.name = name; + this.value = value; + } + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java new file mode 100644 index 00000000000..170a28af502 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import java.util.Map; + +public interface ObjectStoreDetailsDao extends GenericDao, ResourceDetailsDao { + + void update(long storeId, Map details); + + Map getDetails(long storeId); + + void deleteDetails(long storeId); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java new file mode 100644 index 00000000000..e1000e5d045 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreDetailsDaoImpl extends ResourceDetailsDaoBase implements ObjectStoreDetailsDao { + + protected final SearchBuilder storeSearch; + + public ObjectStoreDetailsDaoImpl() { + super(); + storeSearch = createSearchBuilder(); + storeSearch.and("store", storeSearch.entity().getResourceId(), Op.EQ); + storeSearch.done(); + } + + @Override + public void update(long storeId, Map details) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + txn.start(); + expunge(sc); + for (Map.Entry entry : details.entrySet()) { + ObjectStoreDetailVO detail = new ObjectStoreDetailVO(storeId, entry.getKey(), entry.getValue()); + persist(detail); + } + txn.commit(); + } + + @Override + public Map getDetails(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + List details = listBy(sc); + Map detailsMap = new HashMap(); + for (ObjectStoreDetailVO detail : details) { + String name = detail.getName(); + String value = detail.getValue(); + if (name.equals(ApiConstants.KEY)) { + value = DBEncryptionUtil.decrypt(value); + } + detailsMap.put(name, value); + } + + return detailsMap; + } + + @Override + public void deleteDetails(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + List results = search(sc, null); + for (ObjectStoreDetailVO result : results) { + remove(result.getId()); + } + } + + @Override + public ObjectStoreDetailVO findDetail(long storeId, String name) { + QueryBuilder sc = QueryBuilder.create(ObjectStoreDetailVO.class); + sc.and(sc.entity().getResourceId(), Op.EQ, storeId); + sc.and(sc.entity().getName(), Op.EQ, name); + return sc.find(); + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + // ToDo: Add Display + super.addDetail(new ObjectStoreDetailVO(resourceId, key, value)); + } + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java new file mode 100644 index 00000000000..885cbfd98ab --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import org.apache.cloudstack.storage.object.ObjectStore; +import com.cloud.utils.db.GenericDao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.TableGenerator; +import javax.persistence.Transient; +import java.util.Date; +import java.util.Map; + +@Entity +@Table(name = "object_store") +public class ObjectStoreVO implements ObjectStore { + @Id + @TableGenerator(name = "object_store_sq", table = "sequence", pkColumnName = "name", valueColumnName = "value", pkColumnValue = "object_store_seq", allocationSize = 1) + @Column(name = "id", nullable = false) + private long id; + + @Column(name = "name", nullable = false) + private String name; + + @Column(name = "uuid", nullable = false) + private String uuid; + + @Column(name = "url", nullable = false, length = 2048) + private String url; + + @Column(name = "object_provider_name", nullable = false) + private String providerName; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + @Column(name = "total_size") + private Long totalSize; + + @Column(name = "used_bytes") + private Long usedBytes; + + @Transient + Map details; + + @Override + public long getId() { + return this.id; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public String getProviderName() { + return this.providerName; + } + + public void setName(String name) { + this.name = name; + } + + public void setProviderName(String provider) { + this.providerName = provider; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String getUuid() { + return this.uuid; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Long getTotalSize() { + return totalSize; + } + + public void setTotalSize(Long totalSize) { + this.totalSize = totalSize; + } + + public Long getUsedBytes() { + return usedBytes; + } + + public void setUsedBytes(Long usedBytes) { + this.usedBytes = usedBytes; + } + + public void setDetails(Map details) { + this.details = details; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 80fddc9bd94..8f77b4ba63e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -53,7 +53,7 @@ public interface PrimaryDataStoreDao extends GenericDao { */ void updateCapacityIops(long id, long capacityIops); - StoragePoolVO persist(StoragePoolVO pool, Map details, List tags); + StoragePoolVO persist(StoragePoolVO pool, Map details, List tags, Boolean isTagARule); /** * Find pool by name. @@ -77,7 +77,7 @@ public interface PrimaryDataStoreDao extends GenericDao { */ List findPoolsByDetails(long dcId, long podId, Long clusterId, Map details, ScopeType scope); - List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags); + List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout); List findDisabledPoolsByScope(long dcId, Long podId, Long clusterId, ScopeType scope); @@ -112,9 +112,9 @@ public interface PrimaryDataStoreDao extends GenericDao { List listPoolsByCluster(long clusterId); - List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags); + List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule); - List findZoneWideStoragePoolsByTags(long dcId, String[] tags); + List findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule); List findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 2fc52b3fb28..af7dbdc0225 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -67,11 +67,11 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase protected final String DetailsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_details ON storage_pool.id = storage_pool_details.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; protected final String DetailsSqlSuffix = ") GROUP BY storage_pool_details.pool_id HAVING COUNT(storage_pool_details.name) >= ?"; - private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and storage_pool.scope = ? and ("; + private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and storage_pool.scope = ? and ("; private final String ZoneWideTagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?"; // Storage tags are now separate from storage_pool_details, leaving only details on that table - protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; + protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; protected final String TagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?"; private static final String GET_STORAGE_POOLS_OF_VOLUMES_WITHOUT_OR_NOT_HAVING_TAGS = "SELECT s.* " + @@ -278,7 +278,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase @Override @DB - public StoragePoolVO persist(StoragePoolVO pool, Map details, List tags) { + public StoragePoolVO persist(StoragePoolVO pool, Map details, List tags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); pool = super.persist(pool); @@ -289,7 +289,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } } if (CollectionUtils.isNotEmpty(tags)) { - _tagsDao.persist(pool.getId(), tags); + _tagsDao.persist(pool.getId(), tags, isTagARule); } txn.commit(); return pool; @@ -404,10 +404,15 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags) { + public List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout) { List storagePools = null; if (tags == null || tags.length == 0) { storagePools = listBy(dcId, podId, clusterId, ScopeType.CLUSTER); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } + } else { String sqlValues = getSqlValuesFromStorageTags(tags); storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.CLUSTER, sqlValues, ValueType.TAGS, tags.length); @@ -437,10 +442,14 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags) { + public List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule) { List storagePools = null; if (tags == null || tags.length == 0) { storagePools = listBy(dcId, podId, clusterId, ScopeType.HOST); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } } else { String sqlValues = getSqlValuesFromStorageTags(tags); storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.HOST, sqlValues, ValueType.TAGS, tags.length); @@ -483,13 +492,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findZoneWideStoragePoolsByTags(long dcId, String[] tags) { + public List findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule) { if (tags == null || tags.length == 0) { QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); sc.and(sc.entity().getDataCenterId(), Op.EQ, dcId); sc.and(sc.entity().getStatus(), Op.EQ, Status.Up); sc.and(sc.entity().getScope(), Op.EQ, ScopeType.ZONE); - return sc.list(); + + List storagePools = sc.list(); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } + + return storagePools; } else { String sqlValues = getSqlValuesFromStorageTags(tags); String sql = getSqlPreparedStatement(ZoneWideTagsSqlPrefix, ZoneWideTagsSqlSuffix, sqlValues, null); @@ -497,6 +513,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } } + protected List getPoolsWithoutTagRule(List storagePools) { + List storagePoolsToReturn = new ArrayList<>(); + for (StoragePoolVO storagePool : storagePools) { + + List poolTags = _tagsDao.findStoragePoolTags(storagePool.getId()); + + if (CollectionUtils.isEmpty(poolTags) || !poolTags.get(0).isTagARule()) { + storagePoolsToReturn.add(storagePool); + } + } + + return storagePoolsToReturn; + } + @Override public List searchForStoragePoolTags(long poolId) { return _tagsDao.getStoragePoolTags(poolId); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 1cdbd986c59..6e3b8046ed1 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -277,10 +277,15 @@ + + + + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 25f23b1653c..57ebcff0f66 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -21,124 +21,44 @@ ALTER TABLE `cloud`.`mshost` MODIFY COLUMN `state` varchar(25); -DROP VIEW IF EXISTS `cloud`.`async_job_view`; -CREATE VIEW `cloud`.`async_job_view` AS - select - account.id account_id, - account.uuid account_uuid, - account.account_name account_name, - account.type account_type, - domain.id domain_id, - domain.uuid domain_uuid, - domain.name domain_name, - domain.path domain_path, - user.id user_id, - user.uuid user_uuid, - async_job.id, - async_job.uuid, - async_job.job_cmd, - async_job.job_status, - async_job.job_process_status, - async_job.job_result_code, - async_job.job_result, - async_job.created, - async_job.removed, - async_job.instance_type, - async_job.instance_id, - async_job.job_executing_msid, - CASE - WHEN async_job.instance_type = 'Volume' THEN volumes.uuid - WHEN - async_job.instance_type = 'Template' - or async_job.instance_type = 'Iso' - THEN - vm_template.uuid - WHEN - async_job.instance_type = 'VirtualMachine' - or async_job.instance_type = 'ConsoleProxy' - or async_job.instance_type = 'SystemVm' - or async_job.instance_type = 'DomainRouter' - THEN - vm_instance.uuid - WHEN async_job.instance_type = 'Snapshot' THEN snapshots.uuid - WHEN async_job.instance_type = 'Host' THEN host.uuid - WHEN async_job.instance_type = 'StoragePool' THEN storage_pool.uuid - WHEN async_job.instance_type = 'IpAddress' THEN user_ip_address.uuid - WHEN async_job.instance_type = 'SecurityGroup' THEN security_group.uuid - WHEN async_job.instance_type = 'PhysicalNetwork' THEN physical_network.uuid - WHEN async_job.instance_type = 'TrafficType' THEN physical_network_traffic_types.uuid - WHEN async_job.instance_type = 'PhysicalNetworkServiceProvider' THEN physical_network_service_providers.uuid - WHEN async_job.instance_type = 'FirewallRule' THEN firewall_rules.uuid - WHEN async_job.instance_type = 'Account' THEN acct.uuid - WHEN async_job.instance_type = 'User' THEN us.uuid - WHEN async_job.instance_type = 'StaticRoute' THEN static_routes.uuid - WHEN async_job.instance_type = 'PrivateGateway' THEN vpc_gateways.uuid - WHEN async_job.instance_type = 'Counter' THEN counter.uuid - WHEN async_job.instance_type = 'Condition' THEN conditions.uuid - WHEN async_job.instance_type = 'AutoScalePolicy' THEN autoscale_policies.uuid - WHEN async_job.instance_type = 'AutoScaleVmProfile' THEN autoscale_vmprofiles.uuid - WHEN async_job.instance_type = 'AutoScaleVmGroup' THEN autoscale_vmgroups.uuid - ELSE null - END instance_uuid - from - `cloud`.`async_job` - left join - `cloud`.`account` ON async_job.account_id = account.id - left join - `cloud`.`domain` ON domain.id = account.domain_id - left join - `cloud`.`user` ON async_job.user_id = user.id - left join - `cloud`.`volumes` ON async_job.instance_id = volumes.id - left join - `cloud`.`vm_template` ON async_job.instance_id = vm_template.id - left join - `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id - left join - `cloud`.`snapshots` ON async_job.instance_id = snapshots.id - left join - `cloud`.`host` ON async_job.instance_id = host.id - left join - `cloud`.`storage_pool` ON async_job.instance_id = storage_pool.id - left join - `cloud`.`user_ip_address` ON async_job.instance_id = user_ip_address.id - left join - `cloud`.`security_group` ON async_job.instance_id = security_group.id - left join - `cloud`.`physical_network` ON async_job.instance_id = physical_network.id - left join - `cloud`.`physical_network_traffic_types` ON async_job.instance_id = physical_network_traffic_types.id - left join - `cloud`.`physical_network_service_providers` ON async_job.instance_id = physical_network_service_providers.id - left join - `cloud`.`firewall_rules` ON async_job.instance_id = firewall_rules.id - left join - `cloud`.`account` acct ON async_job.instance_id = acct.id - left join - `cloud`.`user` us ON async_job.instance_id = us.id - left join - `cloud`.`static_routes` ON async_job.instance_id = static_routes.id - left join - `cloud`.`vpc_gateways` ON async_job.instance_id = vpc_gateways.id - left join - `cloud`.`counter` ON async_job.instance_id = counter.id - left join - `cloud`.`conditions` ON async_job.instance_id = conditions.id - left join - `cloud`.`autoscale_policies` ON async_job.instance_id = autoscale_policies.id - left join - `cloud`.`autoscale_vmprofiles` ON async_job.instance_id = autoscale_vmprofiles.id - left join - `cloud`.`autoscale_vmgroups` ON async_job.instance_id = autoscale_vmgroups.id; - -- Invalidate existing console_session records UPDATE `cloud`.`console_session` SET removed=now(); -- Modify acquired column in console_session to datetime type ALTER TABLE `cloud`.`console_session` DROP `acquired`, ADD `acquired` datetime COMMENT 'When the session was acquired' AFTER `host_id`; +-- IP quarantine PR#7378 +CREATE TABLE IF NOT EXISTS `cloud`.`quarantined_ips` ( + `id` bigint(20) unsigned NOT NULL auto_increment, + `uuid` varchar(255) UNIQUE, + `public_ip_address_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the quarantined public IP address, foreign key to `user_ip_address` table', + `previous_owner_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the previous owner of the public IP address, foreign key to `account` table', + `created` datetime NOT NULL, + `removed` datetime DEFAULT NULL, + `end_date` datetime NOT NULL, + `removal_reason` VARCHAR(255) DEFAULT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `fk_quarantined_ips__public_ip_address_id` FOREIGN KEY(`public_ip_address_id`) REFERENCES `cloud`.`user_ip_address`(`id`), + CONSTRAINT `fk_quarantined_ips__previous_owner_id` FOREIGN KEY(`previous_owner_id`) REFERENCES `cloud`.`account`(`id`) +); + -- create_public_parameter_on_roles. #6960 ALTER TABLE `cloud`.`roles` ADD COLUMN `public_role` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Indicates whether the role will be visible to all users (public) or only to root admins (private). If this parameter is not specified during the creation of the role its value will be defaulted to true (public).'; +-- Create heuristic table for dynamic allocating resources to the secondary storage +CREATE TABLE IF NOT EXISTS `cloud`.`heuristics` ( + `id` bigint(20) unsigned NOT NULL auto_increment, + `uuid` varchar(255) UNIQUE NOT NULL, + `name` text NOT NULL, + `description` text DEFAULT NULL, + `zone_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the zone to apply the heuristic, foreign key to `data_center` table', + `type` varchar(255) NOT NULL, + `heuristic_rule` text NOT NULL COMMENT 'JS script that defines to which secondary storage the resource will be allocated.', + `created` datetime NOT NULL, + `removed` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `fk_heuristics__zone_id` FOREIGN KEY(`zone_id`) REFERENCES `cloud`.`data_center`(`id`) +); + -- Add tables for VM Scheduler DROP TABLE IF EXISTS `cloud`.`vm_schedule`; CREATE TABLE `cloud`.`vm_schedule` ( @@ -337,203 +257,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`vnf_template_details` ( CONSTRAINT `fk_vnf_template_details__template_id` FOREIGN KEY (`template_id`) REFERENCES `vm_template` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -DROP VIEW IF EXISTS `cloud`.`user_vm_view`; -CREATE - VIEW `user_vm_view` AS -SELECT - `vm_instance`.`id` AS `id`, - `vm_instance`.`name` AS `name`, - `user_vm`.`display_name` AS `display_name`, - `user_vm`.`user_data` AS `user_data`, - `account`.`id` AS `account_id`, - `account`.`uuid` AS `account_uuid`, - `account`.`account_name` AS `account_name`, - `account`.`type` AS `account_type`, - `domain`.`id` AS `domain_id`, - `domain`.`uuid` AS `domain_uuid`, - `domain`.`name` AS `domain_name`, - `domain`.`path` AS `domain_path`, - `projects`.`id` AS `project_id`, - `projects`.`uuid` AS `project_uuid`, - `projects`.`name` AS `project_name`, - `instance_group`.`id` AS `instance_group_id`, - `instance_group`.`uuid` AS `instance_group_uuid`, - `instance_group`.`name` AS `instance_group_name`, - `vm_instance`.`uuid` AS `uuid`, - `vm_instance`.`user_id` AS `user_id`, - `vm_instance`.`last_host_id` AS `last_host_id`, - `vm_instance`.`vm_type` AS `type`, - `vm_instance`.`limit_cpu_use` AS `limit_cpu_use`, - `vm_instance`.`created` AS `created`, - `vm_instance`.`state` AS `state`, - `vm_instance`.`update_time` AS `update_time`, - `vm_instance`.`removed` AS `removed`, - `vm_instance`.`ha_enabled` AS `ha_enabled`, - `vm_instance`.`hypervisor_type` AS `hypervisor_type`, - `vm_instance`.`instance_name` AS `instance_name`, - `vm_instance`.`guest_os_id` AS `guest_os_id`, - `vm_instance`.`display_vm` AS `display_vm`, - `guest_os`.`uuid` AS `guest_os_uuid`, - `vm_instance`.`pod_id` AS `pod_id`, - `host_pod_ref`.`uuid` AS `pod_uuid`, - `vm_instance`.`private_ip_address` AS `private_ip_address`, - `vm_instance`.`private_mac_address` AS `private_mac_address`, - `vm_instance`.`vm_type` AS `vm_type`, - `data_center`.`id` AS `data_center_id`, - `data_center`.`uuid` AS `data_center_uuid`, - `data_center`.`name` AS `data_center_name`, - `data_center`.`is_security_group_enabled` AS `security_group_enabled`, - `data_center`.`networktype` AS `data_center_network_type`, - `host`.`id` AS `host_id`, - `host`.`uuid` AS `host_uuid`, - `host`.`name` AS `host_name`, - `host`.`cluster_id` AS `cluster_id`, - `host`.`status` AS `host_status`, - `host`.`resource_state` AS `host_resource_state`, - `vm_template`.`id` AS `template_id`, - `vm_template`.`uuid` AS `template_uuid`, - `vm_template`.`name` AS `template_name`, - `vm_template`.`type` AS `template_type`, - `vm_template`.`display_text` AS `template_display_text`, - `vm_template`.`enable_password` AS `password_enabled`, - `iso`.`id` AS `iso_id`, - `iso`.`uuid` AS `iso_uuid`, - `iso`.`name` AS `iso_name`, - `iso`.`display_text` AS `iso_display_text`, - `service_offering`.`id` AS `service_offering_id`, - `service_offering`.`uuid` AS `service_offering_uuid`, - `disk_offering`.`uuid` AS `disk_offering_uuid`, - `disk_offering`.`id` AS `disk_offering_id`, - (CASE - WHEN ISNULL(`service_offering`.`cpu`) THEN `custom_cpu`.`value` - ELSE `service_offering`.`cpu` - END) AS `cpu`, - (CASE - WHEN ISNULL(`service_offering`.`speed`) THEN `custom_speed`.`value` - ELSE `service_offering`.`speed` - END) AS `speed`, - (CASE - WHEN ISNULL(`service_offering`.`ram_size`) THEN `custom_ram_size`.`value` - ELSE `service_offering`.`ram_size` - END) AS `ram_size`, - `backup_offering`.`uuid` AS `backup_offering_uuid`, - `backup_offering`.`id` AS `backup_offering_id`, - `service_offering`.`name` AS `service_offering_name`, - `disk_offering`.`name` AS `disk_offering_name`, - `backup_offering`.`name` AS `backup_offering_name`, - `storage_pool`.`id` AS `pool_id`, - `storage_pool`.`uuid` AS `pool_uuid`, - `storage_pool`.`pool_type` AS `pool_type`, - `volumes`.`id` AS `volume_id`, - `volumes`.`uuid` AS `volume_uuid`, - `volumes`.`device_id` AS `volume_device_id`, - `volumes`.`volume_type` AS `volume_type`, - `security_group`.`id` AS `security_group_id`, - `security_group`.`uuid` AS `security_group_uuid`, - `security_group`.`name` AS `security_group_name`, - `security_group`.`description` AS `security_group_description`, - `nics`.`id` AS `nic_id`, - `nics`.`uuid` AS `nic_uuid`, - `nics`.`device_id` AS `nic_device_id`, - `nics`.`network_id` AS `network_id`, - `nics`.`ip4_address` AS `ip_address`, - `nics`.`ip6_address` AS `ip6_address`, - `nics`.`ip6_gateway` AS `ip6_gateway`, - `nics`.`ip6_cidr` AS `ip6_cidr`, - `nics`.`default_nic` AS `is_default_nic`, - `nics`.`gateway` AS `gateway`, - `nics`.`netmask` AS `netmask`, - `nics`.`mac_address` AS `mac_address`, - `nics`.`broadcast_uri` AS `broadcast_uri`, - `nics`.`isolation_uri` AS `isolation_uri`, - `vpc`.`id` AS `vpc_id`, - `vpc`.`uuid` AS `vpc_uuid`, - `networks`.`uuid` AS `network_uuid`, - `networks`.`name` AS `network_name`, - `networks`.`traffic_type` AS `traffic_type`, - `networks`.`guest_type` AS `guest_type`, - `user_ip_address`.`id` AS `public_ip_id`, - `user_ip_address`.`uuid` AS `public_ip_uuid`, - `user_ip_address`.`public_ip_address` AS `public_ip_address`, - `ssh_details`.`value` AS `keypair_names`, - `resource_tags`.`id` AS `tag_id`, - `resource_tags`.`uuid` AS `tag_uuid`, - `resource_tags`.`key` AS `tag_key`, - `resource_tags`.`value` AS `tag_value`, - `resource_tags`.`domain_id` AS `tag_domain_id`, - `domain`.`uuid` AS `tag_domain_uuid`, - `domain`.`name` AS `tag_domain_name`, - `resource_tags`.`account_id` AS `tag_account_id`, - `account`.`account_name` AS `tag_account_name`, - `resource_tags`.`resource_id` AS `tag_resource_id`, - `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, - `resource_tags`.`resource_type` AS `tag_resource_type`, - `resource_tags`.`customer` AS `tag_customer`, - `async_job`.`id` AS `job_id`, - `async_job`.`uuid` AS `job_uuid`, - `async_job`.`job_status` AS `job_status`, - `async_job`.`account_id` AS `job_account_id`, - `affinity_group`.`id` AS `affinity_group_id`, - `affinity_group`.`uuid` AS `affinity_group_uuid`, - `affinity_group`.`name` AS `affinity_group_name`, - `affinity_group`.`description` AS `affinity_group_description`, - `autoscale_vmgroups`.`id` AS `autoscale_vmgroup_id`, - `autoscale_vmgroups`.`uuid` AS `autoscale_vmgroup_uuid`, - `autoscale_vmgroups`.`name` AS `autoscale_vmgroup_name`, - `vm_instance`.`dynamically_scalable` AS `dynamically_scalable`, - `user_data`.`id` AS `user_data_id`, - `user_data`.`uuid` AS `user_data_uuid`, - `user_data`.`name` AS `user_data_name`, - `user_vm`.`user_data_details` AS `user_data_details`, - `vm_template`.`user_data_link_policy` AS `user_data_policy` -FROM - (((((((((((((((((((((((((((((((((((`user_vm` - JOIN `vm_instance` ON (((`vm_instance`.`id` = `user_vm`.`id`) - AND ISNULL(`vm_instance`.`removed`)))) - JOIN `account` ON ((`vm_instance`.`account_id` = `account`.`id`))) - JOIN `domain` ON ((`vm_instance`.`domain_id` = `domain`.`id`))) - LEFT JOIN `guest_os` ON ((`vm_instance`.`guest_os_id` = `guest_os`.`id`))) - LEFT JOIN `host_pod_ref` ON ((`vm_instance`.`pod_id` = `host_pod_ref`.`id`))) - LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) - LEFT JOIN `instance_group_vm_map` ON ((`vm_instance`.`id` = `instance_group_vm_map`.`instance_id`))) - LEFT JOIN `instance_group` ON ((`instance_group_vm_map`.`group_id` = `instance_group`.`id`))) - LEFT JOIN `data_center` ON ((`vm_instance`.`data_center_id` = `data_center`.`id`))) - LEFT JOIN `host` ON ((`vm_instance`.`host_id` = `host`.`id`))) - LEFT JOIN `vm_template` ON ((`vm_instance`.`vm_template_id` = `vm_template`.`id`))) - LEFT JOIN `vm_template` `iso` ON ((`iso`.`id` = `user_vm`.`iso_id`))) - LEFT JOIN `volumes` ON ((`vm_instance`.`id` = `volumes`.`instance_id`))) - LEFT JOIN `service_offering` ON ((`vm_instance`.`service_offering_id` = `service_offering`.`id`))) - LEFT JOIN `disk_offering` `svc_disk_offering` ON ((`volumes`.`disk_offering_id` = `svc_disk_offering`.`id`))) - LEFT JOIN `disk_offering` ON ((`volumes`.`disk_offering_id` = `disk_offering`.`id`))) - LEFT JOIN `backup_offering` ON ((`vm_instance`.`backup_offering_id` = `backup_offering`.`id`))) - LEFT JOIN `storage_pool` ON ((`volumes`.`pool_id` = `storage_pool`.`id`))) - LEFT JOIN `security_group_vm_map` ON ((`vm_instance`.`id` = `security_group_vm_map`.`instance_id`))) - LEFT JOIN `security_group` ON ((`security_group_vm_map`.`security_group_id` = `security_group`.`id`))) - LEFT JOIN `user_data` ON ((`user_data`.`id` = `user_vm`.`user_data_id`))) - LEFT JOIN `nics` ON (((`vm_instance`.`id` = `nics`.`instance_id`) - AND ISNULL(`nics`.`removed`)))) - LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`))) - LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`) - AND ISNULL(`vpc`.`removed`)))) - LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`))) - LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`) - AND (`ssh_details`.`name` = 'SSH.KeyPairNames')))) - LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`) - AND (`resource_tags`.`resource_type` = 'UserVm')))) - LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `vm_instance`.`id`) - AND (`async_job`.`instance_type` = 'VirtualMachine') - AND (`async_job`.`job_status` = 0)))) - LEFT JOIN `affinity_group_vm_map` ON ((`vm_instance`.`id` = `affinity_group_vm_map`.`instance_id`))) - LEFT JOIN `affinity_group` ON ((`affinity_group_vm_map`.`affinity_group_id` = `affinity_group`.`id`))) - LEFT JOIN `autoscale_vmgroup_vm_map` ON ((`autoscale_vmgroup_vm_map`.`instance_id` = `vm_instance`.`id`))) - LEFT JOIN `autoscale_vmgroups` ON ((`autoscale_vmgroup_vm_map`.`vmgroup_id` = `autoscale_vmgroups`.`id`))) - LEFT JOIN `user_vm_details` `custom_cpu` ON (((`custom_cpu`.`vm_id` = `vm_instance`.`id`) - AND (`custom_cpu`.`name` = 'CpuNumber')))) - LEFT JOIN `user_vm_details` `custom_speed` ON (((`custom_speed`.`vm_id` = `vm_instance`.`id`) - AND (`custom_speed`.`name` = 'CpuSpeed')))) - LEFT JOIN `user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `vm_instance`.`id`) - AND (`custom_ram_size`.`name` = 'memory')))); - -- Add tables for Cluster DRS DROP TABLE IF EXISTS `cloud`.`cluster_drs_plan`; CREATE TABLE `cloud`.`cluster_drs_plan` ( @@ -754,3 +477,78 @@ CREATE TABLE `cloud`.`oauth_provider` ( `removed` datetime COMMENT 'date removed if not null', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Flexible tags +ALTER TABLE `cloud`.`storage_pool_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0; + +ALTER TABLE `cloud`.`storage_pool_tags` MODIFY tag text NOT NULL; + +ALTER TABLE `cloud`.`host_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0; + +ALTER TABLE `cloud`.`host_tags` MODIFY tag text NOT NULL; + +DROP TABLE IF EXISTS `cloud`.`object_store`; +CREATE TABLE `cloud`.`object_store` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `name` varchar(255) NOT NULL COMMENT 'name of object store', + `object_provider_name` varchar(255) NOT NULL COMMENT 'id of object_store_provider', + `url` varchar(255) NOT NULL COMMENT 'url of the object store', + `uuid` varchar(255) COMMENT 'uuid of object store', + `created` datetime COMMENT 'date the object store first signed on', + `removed` datetime COMMENT 'date removed if not null', + `total_size` bigint unsigned COMMENT 'storage total size statistics', + `used_bytes` bigint unsigned COMMENT 'storage available bytes statistics', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`object_store_details`; +CREATE TABLE `cloud`.`object_store_details` ( + `id` bigint unsigned UNIQUE NOT NULL AUTO_INCREMENT COMMENT 'id', + `store_id` bigint unsigned NOT NULL COMMENT 'store the detail is related to', + `name` varchar(255) NOT NULL COMMENT 'name of the detail', + `value` varchar(255) NOT NULL COMMENT 'value of the detail', + PRIMARY KEY (`id`), + CONSTRAINT `fk_object_store_details__store_id` FOREIGN KEY `fk_object_store__store_id`(`store_id`) REFERENCES `object_store`(`id`) ON DELETE CASCADE, + INDEX `i_object_store__name__value`(`name`(128), `value`(128)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`bucket`; +CREATE TABLE `cloud`.`bucket` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `name` varchar(255) NOT NULL COMMENT 'name of bucket', + `object_store_id` varchar(255) NOT NULL COMMENT 'id of object_store', + `state` varchar(255) NOT NULL COMMENT 'state of the bucket', + `uuid` varchar(255) COMMENT 'uuid of bucket', + `domain_id` bigint unsigned NOT NULL COMMENT 'domain the bucket belongs to', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + `quota` bigint unsigned COMMENT 'Allocated bucket quota in GB', + `versioning` boolean COMMENT 'versioning enable/disable', + `encryption` boolean COMMENT 'encryption enable/disbale', + `object_lock` boolean COMMENT 'Lock objects in bucket', + `policy` varchar(255) COMMENT 'Bucket Access Policy', + `access_key` varchar(255) COMMENT 'Bucket Access Key', + `secret_key` varchar(255) COMMENT 'Bucket Secret Key', + `bucket_url` varchar(255) COMMENT 'URL to access bucket', + `created` datetime COMMENT 'date the bucket was created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`bucket_statistics`; +CREATE TABLE `cloud`.`bucket_statistics` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `bucket_id` bigint unsigned NOT NULL COMMENT 'id of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud_usage`.`bucket_statistics`; +CREATE TABLE `cloud_usage`.`bucket_statistics` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `bucket_id` bigint unsigned NOT NULL COMMENT 'id of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql new file mode 100644 index 00000000000..8e941a04e8e --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.async_job_view.sql @@ -0,0 +1,129 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`async_job_view`; + +DROP VIEW IF EXISTS `cloud`.`async_job_view`; + +CREATE VIEW `cloud`.`async_job_view` AS +select + account.id account_id, + account.uuid account_uuid, + account.account_name account_name, + account.type account_type, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + user.id user_id, + user.uuid user_uuid, + async_job.id, + async_job.uuid, + async_job.job_cmd, + async_job.job_status, + async_job.job_process_status, + async_job.job_result_code, + async_job.job_result, + async_job.created, + async_job.removed, + async_job.instance_type, + async_job.instance_id, + async_job.job_executing_msid, + CASE + WHEN async_job.instance_type = 'Volume' THEN volumes.uuid + WHEN + async_job.instance_type = 'Template' + or async_job.instance_type = 'Iso' + THEN + vm_template.uuid + WHEN + async_job.instance_type = 'VirtualMachine' + or async_job.instance_type = 'ConsoleProxy' + or async_job.instance_type = 'SystemVm' + or async_job.instance_type = 'DomainRouter' + THEN + vm_instance.uuid + WHEN async_job.instance_type = 'Snapshot' THEN snapshots.uuid + WHEN async_job.instance_type = 'Host' THEN host.uuid + WHEN async_job.instance_type = 'StoragePool' THEN storage_pool.uuid + WHEN async_job.instance_type = 'IpAddress' THEN user_ip_address.uuid + WHEN async_job.instance_type = 'SecurityGroup' THEN security_group.uuid + WHEN async_job.instance_type = 'PhysicalNetwork' THEN physical_network.uuid + WHEN async_job.instance_type = 'TrafficType' THEN physical_network_traffic_types.uuid + WHEN async_job.instance_type = 'PhysicalNetworkServiceProvider' THEN physical_network_service_providers.uuid + WHEN async_job.instance_type = 'FirewallRule' THEN firewall_rules.uuid + WHEN async_job.instance_type = 'Account' THEN acct.uuid + WHEN async_job.instance_type = 'User' THEN us.uuid + WHEN async_job.instance_type = 'StaticRoute' THEN static_routes.uuid + WHEN async_job.instance_type = 'PrivateGateway' THEN vpc_gateways.uuid + WHEN async_job.instance_type = 'Counter' THEN counter.uuid + WHEN async_job.instance_type = 'Condition' THEN conditions.uuid + WHEN async_job.instance_type = 'AutoScalePolicy' THEN autoscale_policies.uuid + WHEN async_job.instance_type = 'AutoScaleVmProfile' THEN autoscale_vmprofiles.uuid + WHEN async_job.instance_type = 'AutoScaleVmGroup' THEN autoscale_vmgroups.uuid + ELSE null + END instance_uuid +from + `cloud`.`async_job` + left join + `cloud`.`account` ON async_job.account_id = account.id + left join + `cloud`.`domain` ON domain.id = account.domain_id + left join + `cloud`.`user` ON async_job.user_id = user.id + left join + `cloud`.`volumes` ON async_job.instance_id = volumes.id + left join + `cloud`.`vm_template` ON async_job.instance_id = vm_template.id + left join + `cloud`.`vm_instance` ON async_job.instance_id = vm_instance.id + left join + `cloud`.`snapshots` ON async_job.instance_id = snapshots.id + left join + `cloud`.`host` ON async_job.instance_id = host.id + left join + `cloud`.`storage_pool` ON async_job.instance_id = storage_pool.id + left join + `cloud`.`user_ip_address` ON async_job.instance_id = user_ip_address.id + left join + `cloud`.`security_group` ON async_job.instance_id = security_group.id + left join + `cloud`.`physical_network` ON async_job.instance_id = physical_network.id + left join + `cloud`.`physical_network_traffic_types` ON async_job.instance_id = physical_network_traffic_types.id + left join + `cloud`.`physical_network_service_providers` ON async_job.instance_id = physical_network_service_providers.id + left join + `cloud`.`firewall_rules` ON async_job.instance_id = firewall_rules.id + left join + `cloud`.`account` acct ON async_job.instance_id = acct.id + left join + `cloud`.`user` us ON async_job.instance_id = us.id + left join + `cloud`.`static_routes` ON async_job.instance_id = static_routes.id + left join + `cloud`.`vpc_gateways` ON async_job.instance_id = vpc_gateways.id + left join + `cloud`.`counter` ON async_job.instance_id = counter.id + left join + `cloud`.`conditions` ON async_job.instance_id = conditions.id + left join + `cloud`.`autoscale_policies` ON async_job.instance_id = autoscale_policies.id + left join + `cloud`.`autoscale_vmprofiles` ON async_job.instance_id = autoscale_vmprofiles.id + left join + `cloud`.`autoscale_vmgroups` ON async_job.instance_id = autoscale_vmgroups.id; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.data_center_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.data_center_view.sql new file mode 100644 index 00000000000..c34df4f1cbf --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.data_center_view.sql @@ -0,0 +1,59 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`data_center_view`; + +DROP VIEW IF EXISTS `cloud`.`data_center_view`; + +CREATE VIEW `cloud`.`data_center_view` AS +select + data_center.id, + data_center.uuid, + data_center.name, + data_center.is_security_group_enabled, + data_center.is_local_storage_enabled, + data_center.description, + data_center.dns1, + data_center.dns2, + data_center.ip6_dns1, + data_center.ip6_dns2, + data_center.internal_dns1, + data_center.internal_dns2, + data_center.guest_network_cidr, + data_center.domain, + data_center.networktype, + data_center.allocation_state, + data_center.zone_token, + data_center.dhcp_provider, + data_center.type, + data_center.removed, + data_center.sort_key, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + dedicated_resources.affinity_group_id, + dedicated_resources.account_id, + affinity_group.uuid affinity_group_uuid +from + `cloud`.`data_center` + left join + `cloud`.`domain` ON data_center.domain_id = domain.id + left join + `cloud`.`dedicated_resources` ON data_center.id = dedicated_resources.data_center_id + left join + `cloud`.`affinity_group` ON dedicated_resources.affinity_group_id = affinity_group.id; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.disk_offering_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.disk_offering_view.sql new file mode 100644 index 00000000000..10dd3c2f9de --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.disk_offering_view.sql @@ -0,0 +1,82 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`disk_offering_view`; + +DROP VIEW IF EXISTS `cloud`.`disk_offering_view`; + +CREATE VIEW `cloud`.`disk_offering_view` AS +SELECT + `disk_offering`.`id` AS `id`, + `disk_offering`.`uuid` AS `uuid`, + `disk_offering`.`name` AS `name`, + `disk_offering`.`display_text` AS `display_text`, + `disk_offering`.`provisioning_type` AS `provisioning_type`, + `disk_offering`.`disk_size` AS `disk_size`, + `disk_offering`.`min_iops` AS `min_iops`, + `disk_offering`.`max_iops` AS `max_iops`, + `disk_offering`.`created` AS `created`, + `disk_offering`.`tags` AS `tags`, + `disk_offering`.`customized` AS `customized`, + `disk_offering`.`customized_iops` AS `customized_iops`, + `disk_offering`.`removed` AS `removed`, + `disk_offering`.`use_local_storage` AS `use_local_storage`, + `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`, + `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`, + `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`, + `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`, + `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`, + `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`, + `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`, + `disk_offering`.`iops_read_rate` AS `iops_read_rate`, + `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`, + `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`, + `disk_offering`.`iops_write_rate` AS `iops_write_rate`, + `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`, + `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`, + `disk_offering`.`cache_mode` AS `cache_mode`, + `disk_offering`.`sort_key` AS `sort_key`, + `disk_offering`.`compute_only` AS `compute_only`, + `disk_offering`.`display_offering` AS `display_offering`, + `disk_offering`.`state` AS `state`, + `disk_offering`.`disk_size_strictness` AS `disk_size_strictness`, + `vsphere_storage_policy`.`value` AS `vsphere_storage_policy`, + `disk_offering`.`encrypt` AS `encrypt`, + GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, + GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, + GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, + GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path, + GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id, + GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid, + GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name +FROM + `cloud`.`disk_offering` + LEFT JOIN + `cloud`.`disk_offering_details` AS `domain_details` ON `domain_details`.`offering_id` = `disk_offering`.`id` AND `domain_details`.`name`='domainid' + LEFT JOIN + `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`) + LEFT JOIN + `cloud`.`disk_offering_details` AS `zone_details` ON `zone_details`.`offering_id` = `disk_offering`.`id` AND `zone_details`.`name`='zoneid' + LEFT JOIN + `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`) + LEFT JOIN + `cloud`.`disk_offering_details` AS `vsphere_storage_policy` ON `vsphere_storage_policy`.`offering_id` = `disk_offering`.`id` + AND `vsphere_storage_policy`.`name` = 'storagepolicy' +WHERE + `disk_offering`.`state`='Active' +GROUP BY + `disk_offering`.`id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql new file mode 100644 index 00000000000..70394e8fd6d --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql @@ -0,0 +1,126 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`domain_router_view`; + +DROP VIEW IF EXISTS `cloud`.`domain_router_view`; + +CREATE VIEW `cloud`.`domain_router_view` AS +select + vm_instance.id id, + vm_instance.name name, + account.id account_id, + account.uuid account_uuid, + account.account_name account_name, + account.type account_type, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + projects.id project_id, + projects.uuid project_uuid, + projects.name project_name, + vm_instance.uuid uuid, + vm_instance.created created, + vm_instance.state state, + vm_instance.removed removed, + vm_instance.pod_id pod_id, + vm_instance.instance_name instance_name, + host_pod_ref.uuid pod_uuid, + data_center.id data_center_id, + data_center.uuid data_center_uuid, + data_center.name data_center_name, + data_center.networktype data_center_network_type, + data_center.dns1 dns1, + data_center.dns2 dns2, + data_center.ip6_dns1 ip6_dns1, + data_center.ip6_dns2 ip6_dns2, + host.id host_id, + host.uuid host_uuid, + host.name host_name, + host.hypervisor_type, + host.cluster_id cluster_id, + host.status host_status, + host.resource_state host_resource_state, + vm_template.id template_id, + vm_template.uuid template_uuid, + service_offering.id service_offering_id, + service_offering.uuid service_offering_uuid, + service_offering.name service_offering_name, + nics.id nic_id, + nics.uuid nic_uuid, + nics.network_id network_id, + nics.ip4_address ip_address, + nics.ip6_address ip6_address, + nics.ip6_gateway ip6_gateway, + nics.ip6_cidr ip6_cidr, + nics.default_nic is_default_nic, + nics.gateway gateway, + nics.netmask netmask, + nics.mac_address mac_address, + nics.broadcast_uri broadcast_uri, + nics.isolation_uri isolation_uri, + nics.mtu mtu, + vpc.id vpc_id, + vpc.uuid vpc_uuid, + vpc.name vpc_name, + networks.uuid network_uuid, + networks.name network_name, + networks.network_domain network_domain, + networks.traffic_type traffic_type, + networks.guest_type guest_type, + async_job.id job_id, + async_job.uuid job_uuid, + async_job.job_status job_status, + async_job.account_id job_account_id, + domain_router.template_version template_version, + domain_router.scripts_version scripts_version, + domain_router.is_redundant_router is_redundant_router, + domain_router.redundant_state redundant_state, + domain_router.stop_pending stop_pending, + domain_router.role role, + domain_router.software_version software_version +from + `cloud`.`domain_router` + inner join + `cloud`.`vm_instance` ON vm_instance.id = domain_router.id + inner join + `cloud`.`account` ON vm_instance.account_id = account.id + inner join + `cloud`.`domain` ON vm_instance.domain_id = domain.id + left join + `cloud`.`host_pod_ref` ON vm_instance.pod_id = host_pod_ref.id + left join + `cloud`.`projects` ON projects.project_account_id = account.id + left join + `cloud`.`data_center` ON vm_instance.data_center_id = data_center.id + left join + `cloud`.`host` ON vm_instance.host_id = host.id + left join + `cloud`.`vm_template` ON vm_instance.vm_template_id = vm_template.id + left join + `cloud`.`service_offering` ON vm_instance.service_offering_id = service_offering.id + left join + `cloud`.`nics` ON vm_instance.id = nics.instance_id and nics.removed is null + left join + `cloud`.`networks` ON nics.network_id = networks.id + left join + `cloud`.`vpc` ON domain_router.vpc_id = vpc.id and vpc.removed is null + left join + `cloud`.`async_job` ON async_job.instance_id = vm_instance.id + and async_job.instance_type = 'DomainRouter' + and async_job.job_status = 0; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql new file mode 100644 index 00000000000..5c6d4fd772b --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql @@ -0,0 +1,111 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`host_view`; + +DROP VIEW IF EXISTS `cloud`.`host_view`; + +CREATE VIEW `cloud`.`host_view` AS +SELECT + host.id, + host.uuid, + host.name, + host.status, + host.disconnected, + host.type, + host.private_ip_address, + host.version, + host.hypervisor_type, + host.hypervisor_version, + host.capabilities, + host.last_ping, + host.created, + host.removed, + host.resource_state, + host.mgmt_server_id, + host.cpu_sockets, + host.cpus, + host.speed, + host.ram, + cluster.id cluster_id, + cluster.uuid cluster_uuid, + cluster.name cluster_name, + cluster.cluster_type, + data_center.id data_center_id, + data_center.uuid data_center_uuid, + data_center.name data_center_name, + data_center.networktype data_center_type, + host_pod_ref.id pod_id, + host_pod_ref.uuid pod_uuid, + host_pod_ref.name pod_name, + GROUP_CONCAT(DISTINCT(host_tags.tag)) AS tag, + `host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, + guest_os_category.id guest_os_category_id, + guest_os_category.uuid guest_os_category_uuid, + guest_os_category.name guest_os_category_name, + mem_caps.used_capacity memory_used_capacity, + mem_caps.reserved_capacity memory_reserved_capacity, + cpu_caps.used_capacity cpu_used_capacity, + cpu_caps.reserved_capacity cpu_reserved_capacity, + async_job.id job_id, + async_job.uuid job_uuid, + async_job.job_status job_status, + async_job.account_id job_account_id, + oobm.enabled AS `oobm_enabled`, + oobm.power_state AS `oobm_power_state`, + ha_config.enabled AS `ha_enabled`, + ha_config.ha_state AS `ha_state`, + ha_config.provider AS `ha_provider`, + `last_annotation_view`.`annotation` AS `annotation`, + `last_annotation_view`.`created` AS `last_annotated`, + `user`.`username` AS `username` +FROM + `cloud`.`host` + LEFT JOIN + `cloud`.`cluster` ON host.cluster_id = cluster.id + LEFT JOIN + `cloud`.`data_center` ON host.data_center_id = data_center.id + LEFT JOIN + `cloud`.`host_pod_ref` ON host.pod_id = host_pod_ref.id + LEFT JOIN + `cloud`.`host_details` ON host.id = host_details.host_id + AND host_details.name = 'guest.os.category.id' + LEFT JOIN + `cloud`.`guest_os_category` ON guest_os_category.id = CONVERT ( host_details.value, UNSIGNED ) + LEFT JOIN + `cloud`.`host_tags` ON host_tags.host_id = host.id + LEFT JOIN + `cloud`.`op_host_capacity` mem_caps ON host.id = mem_caps.host_id + AND mem_caps.capacity_type = 0 + LEFT JOIN + `cloud`.`op_host_capacity` cpu_caps ON host.id = cpu_caps.host_id + AND cpu_caps.capacity_type = 1 + LEFT JOIN + `cloud`.`async_job` ON async_job.instance_id = host.id + AND async_job.instance_type = 'Host' + AND async_job.job_status = 0 + LEFT JOIN + `cloud`.`oobm` ON oobm.host_id = host.id + left join + `cloud`.`ha_config` ON ha_config.resource_id=host.id + and ha_config.resource_type='Host' + LEFT JOIN + `cloud`.`last_annotation_view` ON `last_annotation_view`.`entity_uuid` = `host`.`uuid` + LEFT JOIN + `cloud`.`user` ON `user`.`uuid` = `last_annotation_view`.`user_uuid` +GROUP BY + `host`.`id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.network_offering_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.network_offering_view.sql new file mode 100644 index 00000000000..8ba291e154c --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.network_offering_view.sql @@ -0,0 +1,85 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`network_offering_view`; + +DROP VIEW IF EXISTS `cloud`.`network_offering_view`; + +CREATE VIEW `cloud`.`network_offering_view` AS +SELECT + `network_offerings`.`id` AS `id`, + `network_offerings`.`uuid` AS `uuid`, + `network_offerings`.`name` AS `name`, + `network_offerings`.`unique_name` AS `unique_name`, + `network_offerings`.`display_text` AS `display_text`, + `network_offerings`.`nw_rate` AS `nw_rate`, + `network_offerings`.`mc_rate` AS `mc_rate`, + `network_offerings`.`traffic_type` AS `traffic_type`, + `network_offerings`.`tags` AS `tags`, + `network_offerings`.`system_only` AS `system_only`, + `network_offerings`.`specify_vlan` AS `specify_vlan`, + `network_offerings`.`service_offering_id` AS `service_offering_id`, + `network_offerings`.`conserve_mode` AS `conserve_mode`, + `network_offerings`.`created` AS `created`, + `network_offerings`.`removed` AS `removed`, + `network_offerings`.`default` AS `default`, + `network_offerings`.`availability` AS `availability`, + `network_offerings`.`dedicated_lb_service` AS `dedicated_lb_service`, + `network_offerings`.`shared_source_nat_service` AS `shared_source_nat_service`, + `network_offerings`.`sort_key` AS `sort_key`, + `network_offerings`.`redundant_router_service` AS `redundant_router_service`, + `network_offerings`.`state` AS `state`, + `network_offerings`.`guest_type` AS `guest_type`, + `network_offerings`.`elastic_ip_service` AS `elastic_ip_service`, + `network_offerings`.`eip_associate_public_ip` AS `eip_associate_public_ip`, + `network_offerings`.`elastic_lb_service` AS `elastic_lb_service`, + `network_offerings`.`specify_ip_ranges` AS `specify_ip_ranges`, + `network_offerings`.`inline` AS `inline`, + `network_offerings`.`is_persistent` AS `is_persistent`, + `network_offerings`.`internal_lb` AS `internal_lb`, + `network_offerings`.`public_lb` AS `public_lb`, + `network_offerings`.`egress_default_policy` AS `egress_default_policy`, + `network_offerings`.`concurrent_connections` AS `concurrent_connections`, + `network_offerings`.`keep_alive_enabled` AS `keep_alive_enabled`, + `network_offerings`.`supports_streched_l2` AS `supports_streched_l2`, + `network_offerings`.`supports_public_access` AS `supports_public_access`, + `network_offerings`.`supports_vm_autoscaling` AS `supports_vm_autoscaling`, + `network_offerings`.`for_vpc` AS `for_vpc`, + `network_offerings`.`for_tungsten` AS `for_tungsten`, + `network_offerings`.`service_package_id` AS `service_package_id`, + GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, + GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, + GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, + GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path, + GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id, + GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid, + GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name, + `offering_details`.value AS internet_protocol +FROM + `cloud`.`network_offerings` + LEFT JOIN + `cloud`.`network_offering_details` AS `domain_details` ON `domain_details`.`network_offering_id` = `network_offerings`.`id` AND `domain_details`.`name`='domainid' + LEFT JOIN + `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`) + LEFT JOIN + `cloud`.`network_offering_details` AS `zone_details` ON `zone_details`.`network_offering_id` = `network_offerings`.`id` AND `zone_details`.`name`='zoneid' + LEFT JOIN + `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`) + LEFT JOIN + `cloud`.`network_offering_details` AS `offering_details` ON `offering_details`.`network_offering_id` = `network_offerings`.`id` AND `offering_details`.`name`='internetProtocol' +GROUP BY + `network_offerings`.`id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.service_offering_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.service_offering_view.sql new file mode 100644 index 00000000000..e859af482b4 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.service_offering_view.sql @@ -0,0 +1,114 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`service_offering_view`; + +DROP VIEW IF EXISTS `cloud`.`service_offering_view`; + +CREATE VIEW `cloud`.`service_offering_view` AS +SELECT + `service_offering`.`id` AS `id`, + `service_offering`.`uuid` AS `uuid`, + `service_offering`.`name` AS `name`, + `service_offering`.`display_text` AS `display_text`, + `disk_offering`.`provisioning_type` AS `provisioning_type`, + `service_offering`.`created` AS `created`, + `disk_offering`.`tags` AS `tags`, + `service_offering`.`removed` AS `removed`, + `disk_offering`.`use_local_storage` AS `use_local_storage`, + `service_offering`.`system_use` AS `system_use`, + `disk_offering`.`id` AS `disk_offering_id`, + `disk_offering`.`name` AS `disk_offering_name`, + `disk_offering`.`uuid` AS `disk_offering_uuid`, + `disk_offering`.`display_text` AS `disk_offering_display_text`, + `disk_offering`.`customized_iops` AS `customized_iops`, + `disk_offering`.`min_iops` AS `min_iops`, + `disk_offering`.`max_iops` AS `max_iops`, + `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`, + `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`, + `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`, + `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`, + `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`, + `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`, + `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`, + `disk_offering`.`iops_read_rate` AS `iops_read_rate`, + `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`, + `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`, + `disk_offering`.`iops_write_rate` AS `iops_write_rate`, + `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`, + `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`, + `disk_offering`.`cache_mode` AS `cache_mode`, + `disk_offering`.`disk_size` AS `root_disk_size`, + `disk_offering`.`encrypt` AS `encrypt_root`, + `service_offering`.`cpu` AS `cpu`, + `service_offering`.`speed` AS `speed`, + `service_offering`.`ram_size` AS `ram_size`, + `service_offering`.`nw_rate` AS `nw_rate`, + `service_offering`.`mc_rate` AS `mc_rate`, + `service_offering`.`ha_enabled` AS `ha_enabled`, + `service_offering`.`limit_cpu_use` AS `limit_cpu_use`, + `service_offering`.`host_tag` AS `host_tag`, + `service_offering`.`default_use` AS `default_use`, + `service_offering`.`vm_type` AS `vm_type`, + `service_offering`.`sort_key` AS `sort_key`, + `service_offering`.`is_volatile` AS `is_volatile`, + `service_offering`.`deployment_planner` AS `deployment_planner`, + `service_offering`.`dynamic_scaling_enabled` AS `dynamic_scaling_enabled`, + `service_offering`.`disk_offering_strictness` AS `disk_offering_strictness`, + `vsphere_storage_policy`.`value` AS `vsphere_storage_policy`, + GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, + GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, + GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, + GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path, + GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id, + GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid, + GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name, + IFNULL(`min_compute_details`.`value`, `cpu`) AS min_cpu, + IFNULL(`max_compute_details`.`value`, `cpu`) AS max_cpu, + IFNULL(`min_memory_details`.`value`, `ram_size`) AS min_memory, + IFNULL(`max_memory_details`.`value`, `ram_size`) AS max_memory +FROM + `cloud`.`service_offering` + INNER JOIN + `cloud`.`disk_offering_view` AS `disk_offering` ON service_offering.disk_offering_id = disk_offering.id + LEFT JOIN + `cloud`.`service_offering_details` AS `domain_details` ON `domain_details`.`service_offering_id` = `service_offering`.`id` AND `domain_details`.`name`='domainid' + LEFT JOIN + `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `zone_details` ON `zone_details`.`service_offering_id` = `service_offering`.`id` AND `zone_details`.`name`='zoneid' + LEFT JOIN + `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `min_compute_details` ON `min_compute_details`.`service_offering_id` = `service_offering`.`id` + AND `min_compute_details`.`name` = 'mincpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_compute_details` ON `max_compute_details`.`service_offering_id` = `service_offering`.`id` + AND `max_compute_details`.`name` = 'maxcpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `min_memory_details` ON `min_memory_details`.`service_offering_id` = `service_offering`.`id` + AND `min_memory_details`.`name` = 'minmemory' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_memory_details` ON `max_memory_details`.`service_offering_id` = `service_offering`.`id` + AND `max_memory_details`.`name` = 'maxmemory' + LEFT JOIN + `cloud`.`service_offering_details` AS `vsphere_storage_policy` ON `vsphere_storage_policy`.`service_offering_id` = `service_offering`.`id` + AND `vsphere_storage_policy`.`name` = 'storagepolicy' +WHERE + `service_offering`.`state`='Active' +GROUP BY + `service_offering`.`id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.snapshot_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.snapshot_view.sql new file mode 100644 index 00000000000..c6b8d6b4d05 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.snapshot_view.sql @@ -0,0 +1,107 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`snapshot_view`; + +DROP VIEW IF EXISTS `cloud`.`snapshot_view`; + +CREATE VIEW `cloud`.`snapshot_view` AS +SELECT + `snapshots`.`id` AS `id`, + `snapshots`.`uuid` AS `uuid`, + `snapshots`.`name` AS `name`, + `snapshots`.`status` AS `status`, + `snapshots`.`disk_offering_id` AS `disk_offering_id`, + `snapshots`.`snapshot_type` AS `snapshot_type`, + `snapshots`.`type_description` AS `type_description`, + `snapshots`.`size` AS `size`, + `snapshots`.`created` AS `created`, + `snapshots`.`removed` AS `removed`, + `snapshots`.`location_type` AS `location_type`, + `snapshots`.`hypervisor_type` AS `hypervisor_type`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `volumes`.`id` AS `volume_id`, + `volumes`.`uuid` AS `volume_uuid`, + `volumes`.`name` AS `volume_name`, + `volumes`.`volume_type` AS `volume_type`, + `volumes`.`size` AS `volume_size`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `snapshot_store_ref`.`store_id` AS `store_id`, + IFNULL(`image_store`.`uuid`, `storage_pool`.`uuid`) AS `store_uuid`, + IFNULL(`image_store`.`name`, `storage_pool`.`name`) AS `store_name`, + `snapshot_store_ref`.`store_role` AS `store_role`, + `snapshot_store_ref`.`state` AS `store_state`, + `snapshot_store_ref`.`download_state` AS `download_state`, + `snapshot_store_ref`.`download_pct` AS `download_pct`, + `snapshot_store_ref`.`error_str` AS `error_str`, + `snapshot_store_ref`.`size` AS `store_size`, + `snapshot_store_ref`.`created` AS `created_on_store`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + CONCAT(`snapshots`.`id`, + '_', + IFNULL(`snapshot_store_ref`.`store_role`, 'UNKNOWN'), + '_', + IFNULL(`snapshot_store_ref`.`store_id`, 0)) AS `snapshot_store_pair` +FROM + ((((((((((`snapshots` + JOIN `account` ON ((`account`.`id` = `snapshots`.`account_id`))) + JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `volumes` ON ((`volumes`.`id` = `snapshots`.`volume_id`))) + LEFT JOIN `snapshot_store_ref` ON (((`snapshot_store_ref`.`snapshot_id` = `snapshots`.`id`) + AND (`snapshot_store_ref`.`state` != 'Destroyed') + AND (`snapshot_store_ref`.`display` = 1)))) + LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) + AND (`snapshot_store_ref`.`store_role` = 'Image') + AND (`snapshot_store_ref`.`store_id` IS NOT NULL) + AND (`image_store`.`id` = `snapshot_store_ref`.`store_id`)))) + LEFT JOIN `storage_pool` ON ((ISNULL(`storage_pool`.`removed`) + AND (`snapshot_store_ref`.`store_role` = 'Primary') + AND (`snapshot_store_ref`.`store_id` IS NOT NULL) + AND (`storage_pool`.`id` = `snapshot_store_ref`.`store_id`)))) + LEFT JOIN `snapshot_zone_ref` ON (((`snapshot_zone_ref`.`snapshot_id` = `snapshots`.`id`) + AND ISNULL(`snapshot_store_ref`.`store_id`) + AND ISNULL(`snapshot_zone_ref`.`removed`)))) + LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) + OR (`storage_pool`.`data_center_id` = `data_center`.`id`) + OR (`snapshot_zone_ref`.`zone_id` = `data_center`.`id`)))) + LEFT JOIN `resource_tags` ON ((`resource_tags`.`resource_id` = `snapshots`.`id`) + AND (`resource_tags`.`resource_type` = 'Snapshot'))); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql new file mode 100644 index 00000000000..e6cc9458208 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql @@ -0,0 +1,68 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`storage_pool_view`; + +DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; + +CREATE VIEW `cloud`.`storage_pool_view` AS +SELECT + `storage_pool`.`id` AS `id`, + `storage_pool`.`uuid` AS `uuid`, + `storage_pool`.`name` AS `name`, + `storage_pool`.`status` AS `status`, + `storage_pool`.`path` AS `path`, + `storage_pool`.`pool_type` AS `pool_type`, + `storage_pool`.`host_address` AS `host_address`, + `storage_pool`.`created` AS `created`, + `storage_pool`.`removed` AS `removed`, + `storage_pool`.`capacity_bytes` AS `capacity_bytes`, + `storage_pool`.`capacity_iops` AS `capacity_iops`, + `storage_pool`.`scope` AS `scope`, + `storage_pool`.`hypervisor` AS `hypervisor`, + `storage_pool`.`storage_provider_name` AS `storage_provider_name`, + `storage_pool`.`parent` AS `parent`, + `cluster`.`id` AS `cluster_id`, + `cluster`.`uuid` AS `cluster_uuid`, + `cluster`.`name` AS `cluster_name`, + `cluster`.`cluster_type` AS `cluster_type`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `data_center`.`networktype` AS `data_center_type`, + `host_pod_ref`.`id` AS `pod_id`, + `host_pod_ref`.`uuid` AS `pod_uuid`, + `host_pod_ref`.`name` AS `pod_name`, + `storage_pool_tags`.`tag` AS `tag`, + `storage_pool_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, + `op_host_capacity`.`used_capacity` AS `disk_used_capacity`, + `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id` +FROM + ((((((`cloud`.`storage_pool` + LEFT JOIN `cloud`.`cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`))) + LEFT JOIN `cloud`.`data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`))) + LEFT JOIN `cloud`.`host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`))) + LEFT JOIN `cloud`.`storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`)))) + LEFT JOIN `cloud`.`op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`) + AND (`op_host_capacity`.`capacity_type` IN (3 , 9))))) + LEFT JOIN `cloud`.`async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) + AND (`async_job`.`instance_type` = 'StoragePool') + AND (`async_job`.`job_status` = 0)))); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql new file mode 100644 index 00000000000..40b416b16de --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql @@ -0,0 +1,131 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`template_view`; + +DROP VIEW IF EXISTS `cloud`.`template_view`; + +CREATE VIEW `cloud`.`template_view` AS +SELECT + `vm_template`.`id` AS `id`, + `vm_template`.`uuid` AS `uuid`, + `vm_template`.`unique_name` AS `unique_name`, + `vm_template`.`name` AS `name`, + `vm_template`.`public` AS `public`, + `vm_template`.`featured` AS `featured`, + `vm_template`.`type` AS `type`, + `vm_template`.`hvm` AS `hvm`, + `vm_template`.`bits` AS `bits`, + `vm_template`.`url` AS `url`, + `vm_template`.`format` AS `format`, + `vm_template`.`created` AS `created`, + `vm_template`.`checksum` AS `checksum`, + `vm_template`.`display_text` AS `display_text`, + `vm_template`.`enable_password` AS `enable_password`, + `vm_template`.`dynamically_scalable` AS `dynamically_scalable`, + `vm_template`.`state` AS `template_state`, + `vm_template`.`guest_os_id` AS `guest_os_id`, + `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_name`, + `vm_template`.`bootable` AS `bootable`, + `vm_template`.`prepopulate` AS `prepopulate`, + `vm_template`.`cross_zones` AS `cross_zones`, + `vm_template`.`hypervisor_type` AS `hypervisor_type`, + `vm_template`.`extractable` AS `extractable`, + `vm_template`.`template_tag` AS `template_tag`, + `vm_template`.`sort_key` AS `sort_key`, + `vm_template`.`removed` AS `removed`, + `vm_template`.`enable_sshkey` AS `enable_sshkey`, + `parent_template`.`id` AS `parent_template_id`, + `parent_template`.`uuid` AS `parent_template_uuid`, + `source_template`.`id` AS `source_template_id`, + `source_template`.`uuid` AS `source_template_uuid`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `launch_permission`.`account_id` AS `lp_account_id`, + `template_store_ref`.`store_id` AS `store_id`, + `image_store`.`scope` AS `store_scope`, + `template_store_ref`.`state` AS `state`, + `template_store_ref`.`download_state` AS `download_state`, + `template_store_ref`.`download_pct` AS `download_pct`, + `template_store_ref`.`error_str` AS `error_str`, + `template_store_ref`.`size` AS `size`, + `template_store_ref`.physical_size AS `physical_size`, + `template_store_ref`.`destroyed` AS `destroyed`, + `template_store_ref`.`created` AS `created_on_store`, + `vm_template_details`.`name` AS `detail_name`, + `vm_template_details`.`value` AS `detail_value`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + CONCAT(`vm_template`.`id`, + '_', + IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`, + `vm_template`.`direct_download` AS `direct_download`, + `vm_template`.`deploy_as_is` AS `deploy_as_is`, + `user_data`.`id` AS `user_data_id`, + `user_data`.`uuid` AS `user_data_uuid`, + `user_data`.`name` AS `user_data_name`, + `user_data`.`params` AS `user_data_params`, + `vm_template`.`user_data_link_policy` AS `user_data_policy` +FROM + (((((((((((((`vm_template` + JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`))) + JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`))) + JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`))) + LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`) + AND (`template_store_ref`.`store_role` = 'Image') + AND (`template_store_ref`.`destroyed` = 0)))) + LEFT JOIN `vm_template` `parent_template` ON ((`parent_template`.`id` = `vm_template`.`parent_template_id`))) + LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) + AND (`template_store_ref`.`store_id` IS NOT NULL) + AND (`image_store`.`id` = `template_store_ref`.`store_id`)))) + LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`) + AND ISNULL(`template_store_ref`.`store_id`) + AND ISNULL(`template_zone_ref`.`removed`)))) + LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) + OR (`template_zone_ref`.`zone_id` = `data_center`.`id`)))) + LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `user_data` ON ((`user_data`.`id` = `vm_template`.`user_data_id`)) + LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`) + AND ((`resource_tags`.`resource_type` = 'Template') + OR (`resource_tags`.`resource_type` = 'ISO'))))); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql new file mode 100644 index 00000000000..7eedc03712b --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql @@ -0,0 +1,65 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`user_view`; + +DROP VIEW IF EXISTS `cloud`.`user_view`; + +CREATE VIEW `cloud`.`user_view` AS +select + user.id, + user.uuid, + user.username, + user.password, + user.firstname, + user.lastname, + user.email, + user.state, + user.api_key, + user.secret_key, + user.created, + user.removed, + user.timezone, + user.registration_token, + user.is_registered, + user.incorrect_login_attempts, + user.source, + user.default, + account.id account_id, + account.uuid account_uuid, + account.account_name account_name, + account.type account_type, + account.role_id account_role_id, + domain.id domain_id, + domain.uuid domain_uuid, + domain.name domain_name, + domain.path domain_path, + async_job.id job_id, + async_job.uuid job_uuid, + async_job.job_status job_status, + async_job.account_id job_account_id, + user.is_user_2fa_enabled is_user_2fa_enabled +from + `cloud`.`user` + inner join + `cloud`.`account` ON user.account_id = account.id + inner join + `cloud`.`domain` ON account.domain_id = domain.id + left join + `cloud`.`async_job` ON async_job.instance_id = user.id + and async_job.instance_type = 'User' + and async_job.job_status = 0; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql new file mode 100644 index 00000000000..7a057dc0330 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -0,0 +1,215 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`user_vm_view`; + +DROP VIEW IF EXISTS `cloud`.`user_vm_view`; + +CREATE VIEW `user_vm_view` AS +SELECT + `vm_instance`.`id` AS `id`, + `vm_instance`.`name` AS `name`, + `user_vm`.`display_name` AS `display_name`, + `user_vm`.`user_data` AS `user_data`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `instance_group`.`id` AS `instance_group_id`, + `instance_group`.`uuid` AS `instance_group_uuid`, + `instance_group`.`name` AS `instance_group_name`, + `vm_instance`.`uuid` AS `uuid`, + `vm_instance`.`user_id` AS `user_id`, + `vm_instance`.`last_host_id` AS `last_host_id`, + `vm_instance`.`vm_type` AS `type`, + `vm_instance`.`limit_cpu_use` AS `limit_cpu_use`, + `vm_instance`.`created` AS `created`, + `vm_instance`.`state` AS `state`, + `vm_instance`.`update_time` AS `update_time`, + `vm_instance`.`removed` AS `removed`, + `vm_instance`.`ha_enabled` AS `ha_enabled`, + `vm_instance`.`hypervisor_type` AS `hypervisor_type`, + `vm_instance`.`instance_name` AS `instance_name`, + `vm_instance`.`guest_os_id` AS `guest_os_id`, + `vm_instance`.`display_vm` AS `display_vm`, + `guest_os`.`uuid` AS `guest_os_uuid`, + `vm_instance`.`pod_id` AS `pod_id`, + `host_pod_ref`.`uuid` AS `pod_uuid`, + `vm_instance`.`private_ip_address` AS `private_ip_address`, + `vm_instance`.`private_mac_address` AS `private_mac_address`, + `vm_instance`.`vm_type` AS `vm_type`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `data_center`.`is_security_group_enabled` AS `security_group_enabled`, + `data_center`.`networktype` AS `data_center_network_type`, + `host`.`id` AS `host_id`, + `host`.`uuid` AS `host_uuid`, + `host`.`name` AS `host_name`, + `host`.`cluster_id` AS `cluster_id`, + `host`.`status` AS `host_status`, + `host`.`resource_state` AS `host_resource_state`, + `vm_template`.`id` AS `template_id`, + `vm_template`.`uuid` AS `template_uuid`, + `vm_template`.`name` AS `template_name`, + `vm_template`.`type` AS `template_type`, + `vm_template`.`display_text` AS `template_display_text`, + `vm_template`.`enable_password` AS `password_enabled`, + `iso`.`id` AS `iso_id`, + `iso`.`uuid` AS `iso_uuid`, + `iso`.`name` AS `iso_name`, + `iso`.`display_text` AS `iso_display_text`, + `service_offering`.`id` AS `service_offering_id`, + `service_offering`.`uuid` AS `service_offering_uuid`, + `disk_offering`.`uuid` AS `disk_offering_uuid`, + `disk_offering`.`id` AS `disk_offering_id`, + (CASE + WHEN ISNULL(`service_offering`.`cpu`) THEN `custom_cpu`.`value` + ELSE `service_offering`.`cpu` + END) AS `cpu`, + (CASE + WHEN ISNULL(`service_offering`.`speed`) THEN `custom_speed`.`value` + ELSE `service_offering`.`speed` + END) AS `speed`, + (CASE + WHEN ISNULL(`service_offering`.`ram_size`) THEN `custom_ram_size`.`value` + ELSE `service_offering`.`ram_size` + END) AS `ram_size`, + `backup_offering`.`uuid` AS `backup_offering_uuid`, + `backup_offering`.`id` AS `backup_offering_id`, + `service_offering`.`name` AS `service_offering_name`, + `disk_offering`.`name` AS `disk_offering_name`, + `backup_offering`.`name` AS `backup_offering_name`, + `storage_pool`.`id` AS `pool_id`, + `storage_pool`.`uuid` AS `pool_uuid`, + `storage_pool`.`pool_type` AS `pool_type`, + `volumes`.`id` AS `volume_id`, + `volumes`.`uuid` AS `volume_uuid`, + `volumes`.`device_id` AS `volume_device_id`, + `volumes`.`volume_type` AS `volume_type`, + `security_group`.`id` AS `security_group_id`, + `security_group`.`uuid` AS `security_group_uuid`, + `security_group`.`name` AS `security_group_name`, + `security_group`.`description` AS `security_group_description`, + `nics`.`id` AS `nic_id`, + `nics`.`uuid` AS `nic_uuid`, + `nics`.`device_id` AS `nic_device_id`, + `nics`.`network_id` AS `network_id`, + `nics`.`ip4_address` AS `ip_address`, + `nics`.`ip6_address` AS `ip6_address`, + `nics`.`ip6_gateway` AS `ip6_gateway`, + `nics`.`ip6_cidr` AS `ip6_cidr`, + `nics`.`default_nic` AS `is_default_nic`, + `nics`.`gateway` AS `gateway`, + `nics`.`netmask` AS `netmask`, + `nics`.`mac_address` AS `mac_address`, + `nics`.`broadcast_uri` AS `broadcast_uri`, + `nics`.`isolation_uri` AS `isolation_uri`, + `vpc`.`id` AS `vpc_id`, + `vpc`.`uuid` AS `vpc_uuid`, + `networks`.`uuid` AS `network_uuid`, + `networks`.`name` AS `network_name`, + `networks`.`traffic_type` AS `traffic_type`, + `networks`.`guest_type` AS `guest_type`, + `user_ip_address`.`id` AS `public_ip_id`, + `user_ip_address`.`uuid` AS `public_ip_uuid`, + `user_ip_address`.`public_ip_address` AS `public_ip_address`, + `ssh_details`.`value` AS `keypair_names`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id`, + `affinity_group`.`id` AS `affinity_group_id`, + `affinity_group`.`uuid` AS `affinity_group_uuid`, + `affinity_group`.`name` AS `affinity_group_name`, + `affinity_group`.`description` AS `affinity_group_description`, + `autoscale_vmgroups`.`id` AS `autoscale_vmgroup_id`, + `autoscale_vmgroups`.`uuid` AS `autoscale_vmgroup_uuid`, + `autoscale_vmgroups`.`name` AS `autoscale_vmgroup_name`, + `vm_instance`.`dynamically_scalable` AS `dynamically_scalable`, + `user_data`.`id` AS `user_data_id`, + `user_data`.`uuid` AS `user_data_uuid`, + `user_data`.`name` AS `user_data_name`, + `user_vm`.`user_data_details` AS `user_data_details`, + `vm_template`.`user_data_link_policy` AS `user_data_policy` +FROM + (((((((((((((((((((((((((((((((((((`user_vm` + JOIN `vm_instance` ON (((`vm_instance`.`id` = `user_vm`.`id`) + AND ISNULL(`vm_instance`.`removed`)))) + JOIN `account` ON ((`vm_instance`.`account_id` = `account`.`id`))) + JOIN `domain` ON ((`vm_instance`.`domain_id` = `domain`.`id`))) + LEFT JOIN `guest_os` ON ((`vm_instance`.`guest_os_id` = `guest_os`.`id`))) + LEFT JOIN `host_pod_ref` ON ((`vm_instance`.`pod_id` = `host_pod_ref`.`id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `instance_group_vm_map` ON ((`vm_instance`.`id` = `instance_group_vm_map`.`instance_id`))) + LEFT JOIN `instance_group` ON ((`instance_group_vm_map`.`group_id` = `instance_group`.`id`))) + LEFT JOIN `data_center` ON ((`vm_instance`.`data_center_id` = `data_center`.`id`))) + LEFT JOIN `host` ON ((`vm_instance`.`host_id` = `host`.`id`))) + LEFT JOIN `vm_template` ON ((`vm_instance`.`vm_template_id` = `vm_template`.`id`))) + LEFT JOIN `vm_template` `iso` ON ((`iso`.`id` = `user_vm`.`iso_id`))) + LEFT JOIN `volumes` ON ((`vm_instance`.`id` = `volumes`.`instance_id`))) + LEFT JOIN `service_offering` ON ((`vm_instance`.`service_offering_id` = `service_offering`.`id`))) + LEFT JOIN `disk_offering` `svc_disk_offering` ON ((`volumes`.`disk_offering_id` = `svc_disk_offering`.`id`))) + LEFT JOIN `disk_offering` ON ((`volumes`.`disk_offering_id` = `disk_offering`.`id`))) + LEFT JOIN `backup_offering` ON ((`vm_instance`.`backup_offering_id` = `backup_offering`.`id`))) + LEFT JOIN `storage_pool` ON ((`volumes`.`pool_id` = `storage_pool`.`id`))) + LEFT JOIN `security_group_vm_map` ON ((`vm_instance`.`id` = `security_group_vm_map`.`instance_id`))) + LEFT JOIN `security_group` ON ((`security_group_vm_map`.`security_group_id` = `security_group`.`id`))) + LEFT JOIN `user_data` ON ((`user_data`.`id` = `user_vm`.`user_data_id`))) + LEFT JOIN `nics` ON (((`vm_instance`.`id` = `nics`.`instance_id`) + AND ISNULL(`nics`.`removed`)))) + LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`))) + LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`) + AND ISNULL(`vpc`.`removed`)))) + LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`))) + LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`) + AND (`ssh_details`.`name` = 'SSH.KeyPairNames')))) + LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`) + AND (`resource_tags`.`resource_type` = 'UserVm')))) + LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `vm_instance`.`id`) + AND (`async_job`.`instance_type` = 'VirtualMachine') + AND (`async_job`.`job_status` = 0)))) + LEFT JOIN `affinity_group_vm_map` ON ((`vm_instance`.`id` = `affinity_group_vm_map`.`instance_id`))) + LEFT JOIN `affinity_group` ON ((`affinity_group_vm_map`.`affinity_group_id` = `affinity_group`.`id`))) + LEFT JOIN `autoscale_vmgroup_vm_map` ON ((`autoscale_vmgroup_vm_map`.`instance_id` = `vm_instance`.`id`))) + LEFT JOIN `autoscale_vmgroups` ON ((`autoscale_vmgroup_vm_map`.`vmgroup_id` = `autoscale_vmgroups`.`id`))) + LEFT JOIN `user_vm_details` `custom_cpu` ON (((`custom_cpu`.`vm_id` = `vm_instance`.`id`) + AND (`custom_cpu`.`name` = 'CpuNumber')))) + LEFT JOIN `user_vm_details` `custom_speed` ON (((`custom_speed`.`vm_id` = `vm_instance`.`id`) + AND (`custom_speed`.`name` = 'CpuSpeed')))) + LEFT JOIN `user_vm_details` `custom_ram_size` ON (((`custom_ram_size`.`vm_id` = `vm_instance`.`id`) + AND (`custom_ram_size`.`name` = 'memory')))); diff --git a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java index 9ab010dd6a7..76bc5270b4f 100755 --- a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java +++ b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java @@ -1,61 +1,84 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.host; - -import com.cloud.service.ServiceOfferingVO; -import com.cloud.vm.VirtualMachine; -import java.util.Arrays; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import org.junit.Test; -import org.junit.Before; - -public class HostVOTest { - HostVO host; - ServiceOfferingVO offering; - - @Before - public void setUp() throws Exception { - host = new HostVO(); - offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, - false, "TestSO", false,VirtualMachine.Type.User,false); - } - - @Test - public void testNoSO() { - assertFalse(host.checkHostServiceOfferingTags(null)); - } - - @Test - public void testNoTag() { - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testRightTag() { - host.setHostTags(Arrays.asList("tag1","tag2")); - offering.setHostTag("tag2,tag1"); - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testWrongTag() { - host.setHostTags(Arrays.asList("tag1","tag2")); - offering.setHostTag("tag2,tag4"); - assertFalse(host.checkHostServiceOfferingTags(offering)); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import com.cloud.service.ServiceOfferingVO; +import com.cloud.vm.VirtualMachine; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import org.junit.Test; +import org.junit.Before; + +public class HostVOTest { + HostVO host; + ServiceOfferingVO offering; + + @Before + public void setUp() throws Exception { + host = new HostVO(); + offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, + false, "TestSO", false,VirtualMachine.Type.User,false); + } + + @Test + public void testNoSO() { + assertFalse(host.checkHostServiceOfferingTags(null)); + } + + @Test + public void testNoTag() { + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testRightTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag1"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testWrongTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag4"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatMatches() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("A"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatDoesNotMatch() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("B"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithNullServiceTag() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag(null); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } +} diff --git a/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java b/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java index 7e9658e1dd3..6813a209157 100755 --- a/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java +++ b/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java @@ -17,6 +17,7 @@ package com.cloud.network.as; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -42,14 +43,14 @@ public class AutoScaleVmProfileVOTest { public void testCounterParamsForUpdate() { AutoScaleVmProfileVO profile = new AutoScaleVmProfileVO(); - Map> counterParamList = new HashMap<>(); - counterParamList.put("0", new HashMap<>() {{ put("name", "snmpcommunity"); put("value", "public"); }}); - counterParamList.put("1", new HashMap<>() {{ put("name", "snmpport"); put("value", "161"); }}); + Map> counterParamList = new LinkedHashMap<>(); + counterParamList.put("0", new LinkedHashMap<>() {{ put("name", "snmpcommunity"); put("value", "public"); }}); + counterParamList.put("1", new LinkedHashMap<>() {{ put("name", "snmpport"); put("value", "161"); }}); profile.setCounterParamsForUpdate(counterParamList); Assert.assertEquals("snmpcommunity=public&snmpport=161", profile.getCounterParamsString()); + List> counterParams = profile.getCounterParams(); - List> counterParams = profile.getCounterParams(); Assert.assertEquals(2, counterParams.size()); Assert.assertEquals("snmpcommunity", counterParams.get(0).first()); Assert.assertEquals("public", counterParams.get(0).second()); @@ -69,10 +70,17 @@ public class AutoScaleVmProfileVOTest { List> otherDeployParamsList = profile.getOtherDeployParamsList(); Assert.assertEquals(2, otherDeployParamsList.size()); - Assert.assertEquals("serviceofferingid", otherDeployParamsList.get(0).first()); - Assert.assertEquals("a7fb50f6-01d9-11ed-8bc1-77f8f0228926", otherDeployParamsList.get(0).second()); - Assert.assertEquals("rootdisksize", otherDeployParamsList.get(1).first()); - Assert.assertEquals("10", otherDeployParamsList.get(1).second()); + Assert.assertTrue(containsPair(otherDeployParamsList, "serviceofferingid", "a7fb50f6-01d9-11ed-8bc1-77f8f0228926")); + Assert.assertTrue(containsPair(otherDeployParamsList, "rootdisksize", "10")); + } + + private boolean containsPair(List> list, String key, String value) { + for (Pair pair : list) { + if (key.equals(pair.first()) && value.equals(pair.second())) { + return true; + } + } + return false; } @Test diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index 492ec74382b..8951b9d7c24 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -20,11 +20,10 @@ package org.apache.cloudstack.storage.image; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.direct.download.DirectDownloadManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -36,18 +35,21 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.store.TemplateObject; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.utils.exception.CloudRuntimeException; @Component public class TemplateDataFactoryImpl implements TemplateDataFactory { @@ -203,12 +205,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { * Given existing spool refs, return one pool id existing on pools and refs */ private Long getOneMatchingPoolIdFromRefs(List existingRefs, List pools) { - if (pools.isEmpty()) { - throw new CloudRuntimeException("No storage pools found"); - } - if (existingRefs.isEmpty()) { - return pools.get(0).getId(); - } else { + if (!existingRefs.isEmpty()) { for (VMTemplateStoragePoolVO ref : existingRefs) { for (StoragePoolVO p : pools) { if (ref.getPoolId() == p.getId()) { @@ -217,45 +214,51 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { } } } - return null; + return pools.get(0).getId(); } /** - * Retrieve storage pools with scope = cluster or zone matching clusterId or dataCenterId depending on their scope + * Retrieve storage pools with scope = cluster or zone or local matching clusterId or dataCenterId or hostId depending on their scope */ - private List getStoragePoolsFromClusterOrZone(Long clusterId, long dataCenterId, Hypervisor.HypervisorType hypervisorType) { + private List getStoragePoolsForScope(long dataCenterId, Long clusterId, long hostId, Hypervisor.HypervisorType hypervisorType) { List pools = new ArrayList<>(); if (clusterId != null) { List clusterPools = primaryDataStoreDao.listPoolsByCluster(clusterId); + clusterPools = clusterPools.stream().filter(p -> !p.isLocal()).collect(Collectors.toList()); pools.addAll(clusterPools); } List zonePools = primaryDataStoreDao.findZoneWideStoragePoolsByHypervisor(dataCenterId, hypervisorType); pools.addAll(zonePools); + List localPools = primaryDataStoreDao.findLocalStoragePoolsByHostAndTags(hostId, null); + pools.addAll(localPools); return pools; } + protected Long getBypassedTemplateExistingOrNewPoolId(VMTemplateVO templateVO, Long hostId) { + HostVO host = hostDao.findById(hostId); + List pools = getStoragePoolsForScope(host.getDataCenterId(), host.getClusterId(), hostId, host.getHypervisorType()); + if (CollectionUtils.isEmpty(pools)) { + throw new CloudRuntimeException(String.format("No storage pool found to download template: %s", templateVO.getName())); + } + List existingRefs = templatePoolDao.listByTemplateId(templateVO.getId()); + return getOneMatchingPoolIdFromRefs(existingRefs, pools); + } + @Override public TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId) { VMTemplateVO templateVO = imageDataDao.findById(templateId); if (templateVO == null || !templateVO.isDirectDownload()) { return null; } - Long pool = poolId; + Long templatePoolId = poolId; if (poolId == null) { - //Get ISO from existing pool ref - HostVO host = hostDao.findById(hostId); - List pools = getStoragePoolsFromClusterOrZone(host.getClusterId(), host.getDataCenterId(), host.getHypervisorType()); - List existingRefs = templatePoolDao.listByTemplateId(templateId); - pool = getOneMatchingPoolIdFromRefs(existingRefs, pools); + templatePoolId = getBypassedTemplateExistingOrNewPoolId(templateVO, hostId); } - if (pool == null) { - throw new CloudRuntimeException("No storage pool found where to download template: " + templateId); - } - VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId, null); + VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(templatePoolId, templateId, null); if (spoolRef == null) { - directDownloadManager.downloadTemplate(templateId, pool, hostId); + directDownloadManager.downloadTemplate(templateId, templatePoolId, hostId); } - DataStore store = storeMgr.getDataStore(pool, DataStoreRole.Primary); + DataStore store = storeMgr.getDataStore(templatePoolId, DataStoreRole.Primary); return this.getTemplate(templateId, store); } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index 6d25c481537..5bb0d19be74 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -94,7 +94,7 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, @Override public ImageStoreEntity getImageStore(String uuid) { ImageStoreVO dataStore = dataStoreDao.findByUuid(uuid); - return getImageStore(dataStore.getId()); + return dataStore == null ? null : getImageStore(dataStore.getId()); } @Override @@ -248,6 +248,16 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, return stores; } + @Override + public List listImageStoresFilteringByZoneIds(Long... zoneIds) { + List stores = dataStoreDao.listImageStoresByZoneIds(zoneIds); + List imageStores = new ArrayList<>(); + for (ImageStoreVO store : stores) { + imageStores.add(getImageStore(store.getId())); + } + return imageStores; + } + @Override public String getConfigComponentName() { return ImageStoreProviderManager.class.getSimpleName(); diff --git a/engine/storage/integration-test/src/test/resources/storageContext.xml b/engine/storage/integration-test/src/test/resources/storageContext.xml index fc24753127e..7c95345f673 100644 --- a/engine/storage/integration-test/src/test/resources/storageContext.xml +++ b/engine/storage/integration-test/src/test/resources/storageContext.xml @@ -87,4 +87,8 @@ + + + + diff --git a/engine/storage/object/pom.xml b/engine/storage/object/pom.xml new file mode 100644 index 00000000000..5bad4e8a3d6 --- /dev/null +++ b/engine/storage/object/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + cloud-engine-storage-object + Apache CloudStack Engine Storage Object Component + + org.apache.cloudstack + cloud-engine + 4.19.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java new file mode 100644 index 00000000000..418121503c1 --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataTO; +import org.apache.cloudstack.engine.subsystem.api.storage.BucketInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; + +import java.util.Date; + +public class BucketObject implements BucketInfo { + + private String name; + + @Override + public long getDomainId() { + return 0; + } + + @Override + public long getAccountId() { + return 0; + } + + @Override + public Class getEntityType() { + return null; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public void addPayload(Object data) { + + } + + @Override + public Object getPayload() { + return null; + } + + @Override + public Bucket getBucket() { + return null; + } + + @Override + public long getId() { + return 0; + } + + @Override + public String getUri() { + return null; + } + + @Override + public DataTO getTO() { + return null; + } + + @Override + public DataStore getDataStore() { + return null; + } + + @Override + public Long getSize() { + return null; + } + + @Override + public Integer getQuota() { + return null; + } + + @Override + public boolean isVersioning() { + return false; + } + + @Override + public boolean isEncryption() { + return false; + } + + @Override + public boolean isObjectLock() { + return false; + } + + @Override + public String getPolicy() { + return null; + } + + @Override + public String getBucketURL() { + return null; + } + + @Override + public String getAccessKey() { + return null; + } + + @Override + public String getSecretKey() { + return null; + } + + @Override + public long getPhysicalSize() { + return 0; + } + + @Override + public DataObjectType getType() { + return null; + } + + @Override + public String getUuid() { + return null; + } + + @Override + public boolean delete() { + return false; + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event) { + + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answer) { + + } + + @Override + public void incRefCount() { + + } + + @Override + public void decRefCount() { + + } + + @Override + public Long getRefCount() { + return null; + } + + @Override + public long getObjectStoreId() { + return 0; + } + + @Override + public Date getCreated() { + return null; + } + + @Override + public State getState() { + return null; + } +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java new file mode 100644 index 00000000000..a0db89bad4e --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStorageService; +import org.apache.log4j.Logger; + +public class ObjectStorageServiceImpl implements ObjectStorageService { + + private static final Logger s_logger = Logger.getLogger(ObjectStorageServiceImpl.class); + + +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java new file mode 100644 index 00000000000..40f503692e1 --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.manager; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.ObjectStoreImpl; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreProviderManagerImpl implements ObjectStoreProviderManager, Configurable { + private static final Logger s_logger = Logger.getLogger(ObjectStoreProviderManagerImpl.class); + @Inject + ObjectStoreDao objectStoreDao; + + @Inject + DataStoreProviderManager providerManager; + + Map driverMaps; + + @PostConstruct + public void config() { + driverMaps = new HashMap(); + } + + @Override + public ObjectStoreEntity getObjectStore(long objectStoreId) { + ObjectStoreVO objectStore = objectStoreDao.findById(objectStoreId); + String providerName = objectStore.getProviderName(); + ObjectStoreProvider provider = (ObjectStoreProvider)providerManager.getDataStoreProvider(providerName); + ObjectStoreEntity objStore = ObjectStoreImpl.getDataStore(objectStore, driverMaps.get(provider.getName()), provider); + return objStore; + } + + @Override + public boolean registerDriver(String providerName, ObjectStoreDriver driver) { + if (driverMaps.containsKey(providerName)) { + return false; + } + driverMaps.put(providerName, driver); + return true; + } + + @Override + public ObjectStoreEntity getObjectStore(String uuid) { + ObjectStoreVO objectStore = objectStoreDao.findByUuid(uuid); + return getObjectStore(objectStore.getId()); + } + + @Override + public List listObjectStores() { + List stores = objectStoreDao.listObjectStores(); + List ObjectStores = new ArrayList(); + for (ObjectStoreVO store : stores) { + ObjectStores.add(getObjectStore(store.getId())); + } + return ObjectStores; + } + + @Override + public List listObjectStoreByProvider(String provider) { + List stores = objectStoreDao.findByProvider(provider); + List ObjectStores = new ArrayList(); + for (ObjectStoreVO store : stores) { + ObjectStores.add(getObjectStore(store.getId())); + } + return ObjectStores; + } + + @Override + public String getConfigComponentName() { + return ObjectStoreProviderManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { }; + } +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java new file mode 100644 index 00000000000..825b349bdfc --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.store; + +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.DataStoreRole; +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.log4j.Logger; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class ObjectStoreImpl implements ObjectStoreEntity { + private static final Logger s_logger = Logger.getLogger(ObjectStoreImpl.class); + + protected ObjectStoreDriver driver; + protected ObjectStoreVO objectStoreVO; + protected ObjectStoreProvider provider; + + public ObjectStoreImpl() { + super(); + } + + protected void configure(ObjectStoreVO objectStoreVO, ObjectStoreDriver objectStoreDriver, ObjectStoreProvider provider) { + this.driver = objectStoreDriver; + this.objectStoreVO = objectStoreVO; + this.provider = provider; + } + + public static ObjectStoreEntity getDataStore(ObjectStoreVO objectStoreVO, ObjectStoreDriver objectStoreDriver, ObjectStoreProvider provider) { + ObjectStoreImpl instance = ComponentContext.inject(ObjectStoreImpl.class); + instance.configure(objectStoreVO, objectStoreDriver, provider); + return instance; + } + + @Override + public DataStoreDriver getDriver() { + return this.driver; + } + + @Override + public DataStoreRole getRole() { + return null; + } + + @Override + public long getId() { + return this.objectStoreVO.getId(); + } + + @Override + public String getUri() { + return this.objectStoreVO.getUrl(); + } + + @Override + public Scope getScope() { + return null; + } + + + @Override + public String getUuid() { + return this.objectStoreVO.getUuid(); + } + + public Date getCreated() { + return this.objectStoreVO.getCreated(); + } + + @Override + public String getName() { + return objectStoreVO.getName(); + } + + @Override + public DataObject create(DataObject obj) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + return driver.createBucket(bucket, objectLock); + } + + @Override + public boolean deleteBucket(String bucketName) { + return driver.deleteBucket(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean setBucketEncryption(String bucketName) { + return driver.setBucketEncryption(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean deleteBucketEncryption(String bucketName) { + return driver.deleteBucketEncryption(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean setBucketVersioning(String bucketName) { + return driver.setBucketVersioning(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean deleteBucketVersioning(String bucketName) { + return driver.deleteBucketVersioning(bucketName, objectStoreVO.getId()); + } + + @Override + public void setBucketPolicy(String bucketName, String policy) { + driver.setBucketPolicy(bucketName, policy, objectStoreVO.getId()); + } + + @Override + public void setQuota(String bucketName, int quota) { + driver.setBucketQuota(bucketName, objectStoreVO.getId(), quota); + } + + @Override + public Map getAllBucketsUsage() { + return driver.getAllBucketsUsage(objectStoreVO.getId()); + } + + @Override + public List listBuckets() { + return driver.listBuckets(objectStoreVO.getId()); + } + + /* + Create user if not exists + */ + @Override + public boolean createUser(long accountId) { + return driver.createUser(accountId, objectStoreVO.getId()); + } + + @Override + public boolean delete(DataObject obj) { + return false; + } + + @Override + public DataStoreTO getTO() { + return null; + } + + @Override + public String getProviderName() { + return objectStoreVO.getProviderName(); + } + + @Override + public String getUrl() { + return objectStoreVO.getUrl(); + } + +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java new file mode 100644 index 00000000000..ff88262c4ba --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.store.lifecycle; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; + +public interface ObjectStoreLifeCycle extends DataStoreLifeCycle { +} diff --git a/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml b/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml new file mode 100644 index 00000000000..57bd9f87749 --- /dev/null +++ b/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml @@ -0,0 +1,36 @@ + + + + + + + + diff --git a/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java b/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java new file mode 100644 index 00000000000..392388ca8b8 --- /dev/null +++ b/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.object.manager; + +import junit.framework.TestCase; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.cloudstack.storage.object.store.ObjectStoreImpl; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ObjectStoreProviderManagerImplTest extends TestCase{ + + ObjectStoreProviderManagerImpl objectStoreProviderManagerImplSpy; + + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + ObjectStoreVO objectStoreVO; + + @Mock + DataStoreProviderManager providerManager; + + @Mock + ObjectStoreProvider provider; + + @Mock + Map driverMaps; + + @Mock + ObjectStoreDriver objectStoreDriver; + + @Mock + ObjectStoreEntity objectStoreEntity; + + MockedStatic mockObjectStoreImpl; + + @Before + public void setup(){ + objectStoreProviderManagerImplSpy = Mockito.spy(new ObjectStoreProviderManagerImpl()); + objectStoreProviderManagerImplSpy.objectStoreDao = objectStoreDao; + objectStoreProviderManagerImplSpy.providerManager = providerManager; + objectStoreProviderManagerImplSpy.driverMaps = driverMaps; + mockObjectStoreImpl = mockStatic(ObjectStoreImpl.class); + } + + @Test + public void getObjectStoreTest() { + Mockito.doReturn(objectStoreVO).when(objectStoreDao).findById(Mockito.anyLong()); + Mockito.doReturn(provider).when(providerManager).getDataStoreProvider(Mockito.anyString()); + Mockito.doReturn(objectStoreDriver).when(driverMaps).get(Mockito.anyString()); + Mockito.doReturn("Simulator").when(provider).getName(); + Mockito.doReturn("Simulator").when(objectStoreVO).getProviderName(); + + when(ObjectStoreImpl.getDataStore(Mockito.any(ObjectStoreVO.class), Mockito.any(ObjectStoreDriver.class), + Mockito.any(ObjectStoreProvider.class))).thenReturn(objectStoreEntity); + assertNotNull(objectStoreProviderManagerImplSpy.getObjectStore(1L)); + } + + @Test + public void listObjectStoresTest() { + List stores = new ArrayList<>(); + stores.add(objectStoreVO); + Mockito.doReturn(objectStoreVO).when(objectStoreDao).findById(Mockito.anyLong()); + Mockito.doReturn(provider).when(providerManager).getDataStoreProvider(Mockito.anyString()); + Mockito.doReturn(objectStoreDriver).when(driverMaps).get(Mockito.anyString()); + Mockito.doReturn("Simulator").when(provider).getName(); + Mockito.doReturn("Simulator").when(objectStoreVO).getProviderName(); + when(ObjectStoreImpl.getDataStore(Mockito.any(ObjectStoreVO.class), Mockito.any(ObjectStoreDriver.class), + Mockito.any(ObjectStoreProvider.class))).thenReturn(objectStoreEntity); + Mockito.doReturn(stores).when(objectStoreDao).listObjectStores(); + assertEquals(1, objectStoreProviderManagerImplSpy.listObjectStores().size()); + } + + @Override + @After + public void tearDown() throws Exception { + mockObjectStoreImpl.close(); + super.tearDown(); + } +} diff --git a/engine/storage/object/src/test/resource/testContext.xml b/engine/storage/object/src/test/resource/testContext.xml new file mode 100644 index 00000000000..7352b1148f7 --- /dev/null +++ b/engine/storage/object/src/test/resource/testContext.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.cloudstack.framework + + + + + + + + + + + + + + + + + + + + + + diff --git a/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 4268cf6446f..9c7ee983474 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -47,12 +47,14 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.log4j.Logger; @@ -99,6 +101,9 @@ public class SnapshotServiceImpl implements SnapshotService { @Inject ConfigurationDao _configDao; + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + static private class CreateSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; final AsyncCallFuture future; @@ -297,7 +302,7 @@ public class SnapshotServiceImpl implements SnapshotService { fullSnapshot = snapshotFullBackup; } if (fullSnapshot) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } else { SnapshotInfo parentSnapshot = snapshot.getParent(); // Note that DataStore information in parentSnapshot is for primary @@ -314,12 +319,25 @@ public class SnapshotServiceImpl implements SnapshotService { } } if (parentSnapshotOnBackupStore == null) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } return dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole()); } } + /** + * Verify if the data center has heuristic rules for allocating snapshots; if there is then returns the {@link DataStore} returned by the JS script. + * Otherwise, returns {@link DataStore}s with free capacity. + */ + protected DataStore getImageStoreForSnapshot(Long dataCenterId, SnapshotInfo snapshot) { + DataStore imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(dataCenterId, HeuristicType.SNAPSHOT, snapshot); + + if (imageStore == null) { + imageStore = dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + } + return imageStore; + } + @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotObject snapObj = (SnapshotObject)snapshot; diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java index 917fb2d9c75..6d59b6f36e0 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java @@ -18,6 +18,9 @@ */ package org.apache.cloudstack.storage.snapshot; +import com.cloud.storage.DataStoreRole; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -26,8 +29,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallFuture; -import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; -import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,8 +43,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import com.cloud.storage.DataStoreRole; - @RunWith(MockitoJUnitRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) public class SnapshotServiceImplTest { @@ -57,20 +58,28 @@ public class SnapshotServiceImplTest { SnapshotDataFactory _snapshotFactory; @Mock - AsyncCallbackDispatcher caller; + HeuristicRuleHelper heuristicRuleHelperMock; + + @Mock + SnapshotInfo snapshotMock; + + @Mock + VolumeInfo volumeInfoMock; + + @Mock + DataStoreManager dataStoreManagerMock; + + private static final long DUMMY_ID = 1L; @Test public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { - SnapshotInfo snapshot = Mockito.mock(SnapshotInfo.class); - VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class); - - Mockito.when(snapshot.getId()).thenReturn(1L); - Mockito.when(snapshot.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotMock.getId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotMock.getVolumeId()).thenReturn(DUMMY_ID); Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null); - Mockito.when(volFactory.getVolume(1L, DataStoreRole.Primary)).thenReturn(volumeInfo); + Mockito.when(volFactory.getVolume(DUMMY_ID, DataStoreRole.Primary)).thenReturn(volumeInfoMock); PrimaryDataStore store = Mockito.mock(PrimaryDataStore.class); - Mockito.when(volumeInfo.getDataStore()).thenReturn(store); + Mockito.when(volumeInfoMock.getDataStore()).thenReturn(store); PrimaryDataStoreDriver driver = Mockito.mock(PrimaryDataStoreDriver.class); Mockito.when(store.getDriver()).thenReturn(driver); @@ -80,7 +89,29 @@ public class SnapshotServiceImplTest { Mockito.when(mock.get()).thenReturn(result); Mockito.when(result.isFailed()).thenReturn(false); })) { - Assert.assertTrue(snapshotService.revertSnapshot(snapshot)); + Assert.assertTrue(snapshotService.revertSnapshot(snapshotMock)); } } + + @Test + public void getImageStoreForSnapshotTestShouldListFreeImageStoresWithNoHeuristicRule() { + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(null); + Mockito.when(snapshotMock.getDataCenterId()).thenReturn(DUMMY_ID); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + + @Test + public void getImageStoreForSnapshotTestShouldReturnImageStoreReturnedByTheHeuristicRule() { + DataStore dataStore = Mockito.mock(DataStore.class); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(dataStore); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index f2b0f17232b..89a7b577ae7 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -28,6 +28,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.ScopeType; import com.cloud.storage.StoragePoolStatus; @@ -85,6 +86,9 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement */ private SecureRandom secureRandom = new SecureRandom(); + @Inject + protected StoragePoolJoinDao storagePoolJoinDao; + @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java index fe49504fda1..9c0f84ab14a 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.VolumeApiServiceImpl; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -78,11 +79,12 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat logDisabledStoragePools(dcId, podId, clusterId, ScopeType.CLUSTER); } - List pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags()); + List pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags(), true, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value()); + pools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(dcId, podId, clusterId, ScopeType.CLUSTER, List.of(dskCh.getTags()))); s_logger.debug(String.format("Found pools [%s] that match with tags [%s].", pools, Arrays.toString(dskCh.getTags()))); // add remaining pools in cluster, that did not match tags, to avoid set - List allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null); + List allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null, false, 0); allPools.removeAll(pools); for (StoragePoolVO pool : allPools) { s_logger.trace(String.format("Adding pool [%s] to the 'avoid' set since it did not match any tags.", pool)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java index 774c2229a09..4ec15b9e43f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java @@ -101,7 +101,8 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { return null; } List availablePools = - storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags()); + storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags(), true); + availablePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), ScopeType.HOST, List.of(dskCh.getTags()))); for (StoragePoolVO pool : availablePools) { if (suitablePools.size() == returnUpTo) { break; @@ -117,7 +118,7 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { } // add remaining pools in cluster to the 'avoid' set which did not match tags - List allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null); + List allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null, false); allPools.removeAll(availablePools); for (StoragePoolVO pool : allPools) { avoid.addPool(pool.getId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java index 1b3835560df..ba130b4e2e5 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java @@ -63,9 +63,9 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { } List suitablePools = new ArrayList<>(); - - List storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags()); - if (storagePools == null) { + List storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags(), true); + storagePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), null, null, ScopeType.ZONE, List.of(dskCh.getTags()))); + if (storagePools.isEmpty()) { LOGGER.debug(String.format("Could not find any zone wide storage pool that matched with any of the following tags [%s].", Arrays.toString(dskCh.getTags()))); storagePools = new ArrayList<>(); } @@ -82,7 +82,7 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { storagePools.addAll(anyHypervisorStoragePools); // add remaining pools in zone, that did not match tags, to avoid set - List allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null); + List allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null, false); allPools.removeAll(storagePools); for (StoragePoolVO pool : allPools) { avoid.addPool(pool.getId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index cd525ae0ef7..757623e3d04 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.springframework.stereotype.Component; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; @@ -40,6 +41,8 @@ public class DataStoreManagerImpl implements DataStoreManager { PrimaryDataStoreProviderManager primaryStoreMgr; @Inject ImageStoreProviderManager imageDataStoreMgr; + @Inject + ObjectStoreProviderManager objectStoreProviderMgr; @Override public DataStore getDataStore(long storeId, DataStoreRole role) { @@ -50,6 +53,8 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.getImageStore(storeId); } else if (role == DataStoreRole.ImageCache) { return imageDataStoreMgr.getImageStore(storeId); + } else if (role == DataStoreRole.Object) { + return objectStoreProviderMgr.getObjectStore(storeId); } } catch (CloudRuntimeException e) { throw e; @@ -63,6 +68,8 @@ public class DataStoreManagerImpl implements DataStoreManager { return primaryStoreMgr.getPrimaryDataStore(uuid); } else if (role == DataStoreRole.Image) { return imageDataStoreMgr.getImageStore(uuid); + } else if (role == DataStoreRole.Object) { + return objectStoreProviderMgr.getObjectStore(uuid); } throw new CloudRuntimeException("un recognized type" + role); } @@ -77,6 +84,16 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.listImageStoresByScopeExcludingReadOnly(scope); } + @Override + public List getImageStoresByZoneIds(Long... zoneIds) { + return imageDataStoreMgr.listImageStoresFilteringByZoneIds(zoneIds); + } + + @Override + public DataStore getImageStoreByUuid(String uuid) { + return imageDataStoreMgr.getImageStore(uuid); + } + @Override public DataStore getRandomImageStore(long zoneId) { List stores = getImageStoresByScope(new ZoneScope(zoneId)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java index 98eeb6b4405..35e758accca 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java @@ -30,6 +30,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -56,6 +58,8 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto PrimaryDataStoreProviderManager primaryDataStoreProviderMgr; @Inject ImageStoreProviderManager imageStoreProviderMgr; + @Inject + ObjectStoreProviderManager objectStoreProviderMgr; @Override public DataStoreProvider getDataStoreProvider(String name) { @@ -144,6 +148,8 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto primaryDataStoreProviderMgr.registerHostListener(provider.getName(), provider.getHostListener()); } else if (types.contains(DataStoreProviderType.IMAGE)) { imageStoreProviderMgr.registerDriver(provider.getName(), (ImageStoreDriver)provider.getDataStoreDriver()); + } else if (types.contains(DataStoreProviderType.OBJECT)) { + objectStoreProviderMgr.registerDriver(provider.getName(), (ObjectStoreDriver)provider.getDataStoreDriver()); } } catch (Exception e) { s_logger.debug("configure provider failed", e); @@ -169,6 +175,11 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return this.getDataStoreProvider(DataStoreProvider.NFS_IMAGE); } + @Override + public DataStoreProvider getDefaultObjectStoreProvider() { + return this.getDataStoreProvider(DataStoreProvider.S3_IMAGE); + } + @Override public List getDataStoreProviders(String type) { if (type == null) { @@ -180,7 +191,9 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return this.getImageDataStoreProviders(); } else if (type.equalsIgnoreCase(DataStoreProvider.DataStoreProviderType.ImageCache.toString())) { return this.getCacheDataStoreProviders(); - } else { + } else if (type.equalsIgnoreCase(DataStoreProviderType.OBJECT.toString())) { + return this.getObjectStoreProviders(); + }else { throw new InvalidParameterValueException("Invalid parameter: " + type); } } @@ -223,4 +236,16 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return providers; } + public List getObjectStoreProviders() { + List providers = new ArrayList(); + for (DataStoreProvider provider : providerMap.values()) { + if (provider.getTypes().contains(DataStoreProviderType.OBJECT)) { + StorageProviderResponse response = new StorageProviderResponse(); + response.setName(provider.getName()); + response.setType(DataStoreProviderType.OBJECT.toString()); + providers.add(response); + } + } + return providers; + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 47e2ee38307..39f42e842c1 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -80,5 +80,8 @@ public interface ImageStoreProviderManager { List listImageStoresWithFreeCapacity(List imageStores); List orderImageStoresOnFreeCapacity(List imageStores); + + List listImageStoresFilteringByZoneIds(Long... zoneIds); + long getImageStoreZoneId(long dataStoreId); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java new file mode 100644 index 00000000000..e6027a1959f --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.cloud.agent.api.to.DataTO; +import com.cloud.host.Host; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.log4j.Logger; + +import java.util.Map; + +public abstract class BaseObjectStoreDriverImpl implements ObjectStoreDriver { + private static final Logger LOGGER = Logger.getLogger(BaseObjectStoreDriverImpl.class); + + @Override + public Map getCapabilities() { + return null; + } + + @Override + public DataTO getTO(DataObject data) { + return null; + } + + protected class CreateContext extends AsyncRpcContext { + final DataObject data; + + public CreateContext(AsyncCompletionCallback callback, DataObject data) { + super(callback); + this.data = data; + } + } + + @Override + public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + } + + @Override + public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + } + + @Override + public void copyAsync(DataObject srcdata, DataObject destData, AsyncCompletionCallback callback) { + } + + @Override + public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback callback) { + copyAsync(srcData, destData, callback); + } + + @Override + public boolean canCopy(DataObject srcData, DataObject destData) { + return false; + } + + @Override + public void resize(DataObject data, AsyncCompletionCallback callback) { + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java new file mode 100644 index 00000000000..4953b9b0cdf --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; + +import java.util.List; +import java.util.Map; + +public interface ObjectStoreDriver extends DataStoreDriver { + Bucket createBucket(Bucket bucket, boolean objectLock); + + List listBuckets(long storeId); + + boolean deleteBucket(String bucketName, long storeId); + + AccessControlList getBucketAcl(String bucketName, long storeId); + + void setBucketAcl(String bucketName, AccessControlList acl, long storeId); + + void setBucketPolicy(String bucketName, String policyType, long storeId); + + BucketPolicy getBucketPolicy(String bucketName, long storeId); + + void deleteBucketPolicy(String bucketName, long storeId); + + boolean createUser(long accountId, long storeId); + + boolean setBucketEncryption(String bucketName, long storeId); + + boolean deleteBucketEncryption(String bucketName, long storeId); + + + boolean setBucketVersioning(String bucketName, long storeId); + + boolean deleteBucketVersioning(String bucketName, long storeId); + + void setBucketQuota(String bucketName, long storeId, long size); + + Map getAllBucketsUsage(long storeId); +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java new file mode 100644 index 00000000000..c58d801e40e --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.datastore; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailVO; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; + +@Component +public class ObjectStoreHelper { + @Inject + ObjectStoreDao ObjectStoreDao; + @Inject + ObjectStoreDetailsDao ObjectStoreDetailsDao; + + public ObjectStoreVO createObjectStore(Map params, Map details) { + ObjectStoreVO store = new ObjectStoreVO(); + + store.setProviderName((String)params.get("providerName")); + store.setUuid(UUID.randomUUID().toString()); + store.setUrl((String)params.get("url")); + store.setName((String)params.get("name")); + + store = ObjectStoreDao.persist(store); + + // persist details + if (details != null) { + Iterator keyIter = details.keySet().iterator(); + while (keyIter.hasNext()) { + String key = keyIter.next().toString(); + String value = details.get(key); + ObjectStoreDetailVO detail = new ObjectStoreDetailVO(store.getId(), key, value); + ObjectStoreDetailsDao.persist(detail); + } + } + return store; + } + + public boolean deleteObjectStore(long id) { + ObjectStoreVO store = ObjectStoreDao.findById(id); + if (store == null) { + throw new CloudRuntimeException("can't find Object store:" + id); + } + + ObjectStoreDao.remove(id); + return true; + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java new file mode 100644 index 00000000000..b23f3194139 --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.datastore; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; + +import java.util.List; + +public interface ObjectStoreProviderManager { + ObjectStoreEntity getObjectStore(String uuid); + + List listObjectStores(); + + List listObjectStoreByProvider(String provider); + + ObjectStoreEntity getObjectStore(long objectStoreId); + + boolean registerDriver(String uuid, ObjectStoreDriver driver); + +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index c3379ad316b..fbb4a6e1618 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -136,7 +136,7 @@ public class PrimaryDataStoreHelper { storageTags.add(tag); } } - dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags); + dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags, params.isTagARule()); return dataStoreMgr.getDataStore(dataStoreVO.getId(), DataStoreRole.Primary); } diff --git a/framework/db/src/main/java/com/cloud/utils/db/Filter.java b/framework/db/src/main/java/com/cloud/utils/db/Filter.java index 59dc8c1477e..15161ab058f 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/Filter.java +++ b/framework/db/src/main/java/com/cloud/utils/db/Filter.java @@ -51,6 +51,10 @@ public class Filter { addOrderBy(clazz, field, ascending); } + public Filter(Class clazz, String field, boolean ascending) { + this(clazz, field, ascending, null, null); + } + public Filter(long limit) { _orderBy = " ORDER BY RAND() LIMIT " + limit; } diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java index 1eae0edd9c3..b2a9e6f733a 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java @@ -258,6 +258,8 @@ public interface GenericDao { public T findOneBy(final SearchCriteria sc); + T findOneBy(SearchCriteria sc, Filter filter); + /** * @return */ diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index 5fd9580342c..6bf36df0941 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -422,7 +422,7 @@ public abstract class GenericDaoBase extends Compone return result; } catch (final SQLException e) { throw new CloudRuntimeException("DB Exception on: " + pstmt, e); - } catch (final Throwable e) { + } catch (final Exception e) { throw new CloudRuntimeException("Caught: " + pstmt, e); } } @@ -499,7 +499,7 @@ public abstract class GenericDaoBase extends Compone return results; } catch (final SQLException e) { throw new CloudRuntimeException("DB Exception on: " + pstmt, e); - } catch (final Throwable e) { + } catch (final Exception e) { throw new CloudRuntimeException("Caught: " + pstmt, e); } } @@ -907,6 +907,15 @@ public abstract class GenericDaoBase extends Compone return findOneIncludingRemovedBy(sc); } + @Override + @DB() + public T findOneBy(SearchCriteria sc, final Filter filter) { + sc = checkAndSetRemovedIsNull(sc); + filter.setLimit(1L); + List results = searchIncludingRemoved(sc, filter, null, false); + return results.isEmpty() ? null : results.get(0); + } + @DB() protected List listBy(SearchCriteria sc, final Filter filter) { sc = checkAndSetRemovedIsNull(sc); @@ -1145,7 +1154,7 @@ public abstract class GenericDaoBase extends Compone return result; } catch (final SQLException e) { throw new CloudRuntimeException("DB Exception on: " + pstmt, e); - } catch (final Throwable e) { + } catch (final Exception e) { throw new CloudRuntimeException("Caught: " + pstmt, e); } } @@ -1227,7 +1236,7 @@ public abstract class GenericDaoBase extends Compone return pstmt.executeUpdate(); } catch (final SQLException e) { throw new CloudRuntimeException("DB Exception on: " + pstmt, e); - } catch (final Throwable e) { + } catch (final Exception e) { throw new CloudRuntimeException("Caught: " + pstmt, e); } } @@ -2050,7 +2059,7 @@ public abstract class GenericDaoBase extends Compone return 0; } catch (final SQLException e) { throw new CloudRuntimeException("DB Exception on: " + pstmt, e); - } catch (final Throwable e) { + } catch (final Exception e) { throw new CloudRuntimeException("Caught: " + pstmt, e); } } @@ -2101,7 +2110,7 @@ public abstract class GenericDaoBase extends Compone return 0; } catch (final SQLException e) { throw new CloudRuntimeException("DB Exception in executing: " + sql, e); - } catch (final Throwable e) { + } catch (final Exception e) { throw new CloudRuntimeException("Caught exception in : " + sql, e); } } @@ -2158,7 +2167,7 @@ public abstract class GenericDaoBase extends Compone return 0; } catch (final SQLException e) { throw new CloudRuntimeException("DB Exception on: " + pstmt, e); - } catch (final Throwable e) { + } catch (final Exception e) { throw new CloudRuntimeException("Caught: " + pstmt, e); } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index b17d27b0adf..56a6edf5db4 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -89,11 +89,11 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { private TimeZone _usageTimezone; private int _aggregationDuration = 0; - - static final BigDecimal s_hoursInMonth = BigDecimal.valueOf(DateUtil.HOURS_IN_A_MONTH); static final BigDecimal GiB_DECIMAL = BigDecimal.valueOf(ByteScaleUtils.GiB); List lockablesAccountTypes = Arrays.asList(Account.Type.NORMAL, Account.Type.DOMAIN_ADMIN); + static BigDecimal hoursInCurrentMonth; + public QuotaManagerImpl() { super(); } @@ -455,7 +455,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { } - jsInterpreter.injectVariable("resourceType", presetVariables.getResourceType()); + jsInterpreter.injectStringVariable("resourceType", presetVariables.getResourceType()); jsInterpreter.injectVariable("value", presetVariables.getValue().toString()); jsInterpreter.injectVariable("zone", presetVariables.getZone().toString()); } @@ -533,7 +533,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { protected BigDecimal getUsageValueAccordingToUsageUnitType(UsageVO usageRecord, BigDecimal aggregatedQuotaTariffsValue, String quotaUnit) { BigDecimal rawUsage = BigDecimal.valueOf(usageRecord.getRawUsage()); - BigDecimal costPerHour = aggregatedQuotaTariffsValue.divide(s_hoursInMonth, 8, RoundingMode.HALF_EVEN); + BigDecimal costPerHour = getCostPerHour(aggregatedQuotaTariffsValue, usageRecord.getStartDate()); switch (UsageUnitTypes.getByDescription(quotaUnit)) { case COMPUTE_MONTH: @@ -558,6 +558,12 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { } } + protected BigDecimal getCostPerHour(BigDecimal costPerMonth, Date date) { + BigDecimal hoursInCurrentMonth = BigDecimal.valueOf(DateUtil.getHoursInCurrentMonth(date)); + s_logger.trace(String.format("Dividing tariff cost per month [%s] by [%s] to get the tariffs cost per hour.", costPerMonth, hoursInCurrentMonth)); + return costPerMonth.divide(hoursInCurrentMonth, 8, RoundingMode.HALF_EVEN); + } + @Override public boolean isLockable(AccountVO account) { return lockablesAccountTypes.contains(account.getType()); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java index 337131038e8..fef3e4376dc 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java @@ -22,6 +22,8 @@ import java.util.List; public class Host extends GenericPresetVariable { private List tags; + private Boolean isTagARule; + public List getTags() { return tags; } @@ -31,4 +33,12 @@ public class Host extends GenericPresetVariable { fieldNamesToIncludeInToString.add("tags"); } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + fieldNamesToIncludeInToString.add("isTagARule"); + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 1cf4f864ab9..9723d3e5899 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -25,8 +25,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.cloud.host.HostTagVO; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -65,6 +68,7 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOSVO; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; @@ -318,6 +322,10 @@ public class PresetVariableHelper { value.setTags(getPresetVariableValueResourceTags(vmId, ResourceObjectType.UserVm)); value.setTemplate(getPresetVariableValueTemplate(vmVo.getTemplateId())); + Hypervisor.HypervisorType hypervisorType = vmVo.getHypervisorType(); + if (hypervisorType != null) { + value.setHypervisorType(hypervisorType.name()); + } } protected void logNotLoadingMessageInTrace(String resource, int usageType) { @@ -345,7 +353,17 @@ public class PresetVariableHelper { Host host = new Host(); host.setId(hostVo.getUuid()); host.setName(hostVo.getName()); - host.setTags(hostTagsDao.getHostTags(hostId)); + List hostTagVOList = hostTagsDao.getHostTags(hostId); + List hostTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(hostTagVOList)) { + isTagARule = hostTagVOList.get(0).getIsTagARule(); + if (!isTagARule) { + hostTags = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); + } + } + host.setTags(hostTags); + host.setIsTagARule(isTagARule); return host; } @@ -470,6 +488,11 @@ public class PresetVariableHelper { value.setTags(getPresetVariableValueResourceTags(volumeId, ResourceObjectType.Volume)); value.setSize(ByteScaleUtils.bytesToMebibytes(volumeVo.getSize())); + + ImageFormat format = volumeVo.getFormat(); + if (format != null) { + value.setVolumeFormat(format.name()); + } } protected GenericPresetVariable getPresetVariableValueDiskOffering(Long diskOfferingId) { @@ -497,7 +520,17 @@ public class PresetVariableHelper { storage.setId(storagePoolVo.getUuid()); storage.setName(storagePoolVo.getName()); storage.setScope(storagePoolVo.getScope()); - storage.setTags(storagePoolTagsDao.getStoragePoolTags(storageId)); + List storagePoolTagVOList = storagePoolTagsDao.findStoragePoolTags(storageId); + List storageTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(storagePoolTagVOList)) { + isTagARule = storagePoolTagVOList.get(0).isTagARule(); + if (!isTagARule) { + storageTags = storagePoolTagVOList.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()); + } + } + storage.setTags(storageTags); + storage.setIsTagARule(isTagARule); return storage; } @@ -558,6 +591,10 @@ public class PresetVariableHelper { value.setSnapshotType(Snapshot.Type.values()[snapshotVo.getSnapshotType()]); value.setStorage(getPresetVariableValueStorage(getSnapshotDataStoreId(snapshotId, usageRecord.getZoneId()), usageType)); value.setTags(getPresetVariableValueResourceTags(snapshotId, ResourceObjectType.Snapshot)); + Hypervisor.HypervisorType hypervisorType = snapshotVo.getHypervisorType(); + if (hypervisorType != null) { + value.setHypervisorType(hypervisorType.name()); + } } protected SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { @@ -621,6 +658,11 @@ public class PresetVariableHelper { value.setName(vmSnapshotVo.getName()); value.setTags(getPresetVariableValueResourceTags(vmSnapshotId, ResourceObjectType.VMSnapshot)); value.setVmSnapshotType(vmSnapshotVo.getType()); + + VMInstanceVO vmVo = vmInstanceDao.findByIdIncludingRemoved(vmSnapshotVo.getVmId()); + if (vmVo != null && vmVo.getHypervisorType() != null) { + value.setHypervisorType(vmVo.getHypervisorType().name()); + } } protected void loadPresetVariableValueForBackup(UsageVO usageRecord, Value value) { diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java index 3533c5d45c1..6be1dfb025a 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java @@ -23,6 +23,8 @@ import com.cloud.storage.ScopeType; public class Storage extends GenericPresetVariable { private List tags; + + private Boolean isTagARule; private ScopeType scope; public List getTags() { @@ -34,6 +36,15 @@ public class Storage extends GenericPresetVariable { fieldNamesToIncludeInToString.add("tags"); } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + fieldNamesToIncludeInToString.add("isTagARule"); + } + public ScopeType getScope() { return scope; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index 87fefe3a363..f3accd83be8 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -41,6 +41,8 @@ public class Value extends GenericPresetVariable { private Storage storage; private ComputingResources computingResources; private BackupOffering backupOffering; + private String hypervisorType; + private String volumeFormat; public Host getHost() { return host; @@ -185,4 +187,22 @@ public class Value extends GenericPresetVariable { this.backupOffering = backupOffering; fieldNamesToIncludeInToString.add("backupOffering"); } + + public void setHypervisorType(String hypervisorType) { + this.hypervisorType = hypervisorType; + fieldNamesToIncludeInToString.add("hypervisorType"); + } + + public String getHypervisorType() { + return hypervisorType; + } + + public void setVolumeFormat(String volumeFormat) { + this.volumeFormat = volumeFormat; + fieldNamesToIncludeInToString.add("volumeFormat"); + } + + public String getVolumeFormat() { + return volumeFormat; + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 553dc840c0d..6f19fa95f1b 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -55,6 +55,7 @@ public class QuotaTypes extends UsageTypes { quotaTypeList.put(VOLUME_SECONDARY, new QuotaTypes(VOLUME_SECONDARY, "VOLUME_SECONDARY", UsageUnitTypes.GB_MONTH.toString(), "Volume secondary storage usage")); quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot primary storage usage")); quotaTypeList.put(BACKUP, new QuotaTypes(BACKUP, "BACKUP", UsageUnitTypes.GB_MONTH.toString(), "Backup storage usage")); + quotaTypeList.put(BUCKET, new QuotaTypes(BUCKET, "BUCKET", UsageUnitTypes.GB_MONTH.toString(), "Object Store bucket usage")); quotaTypeMap = Collections.unmodifiableMap(quotaTypeList); } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java index 5978dcf5fc4..e53051f2925 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java @@ -142,8 +142,10 @@ public class QuotaManagerImplTest { public void getUsageValueAccordingToUsageUnitTypeTestAllTypes() { Mockito.doReturn(10.0).when(usageVoMock).getRawUsage(); Mockito.doReturn(ByteScaleUtils.GiB).when(usageVoMock).getSize(); + Mockito.doReturn(new Date(0, 8, 10)).when(usageVoMock).getStartDate(); BigDecimal aggregatedQuotaTariffsValue = new BigDecimal(400); + Arrays.asList(UsageUnitTypes.values()).forEach(type -> { BigDecimal result = quotaManagerImplSpy.getUsageValueAccordingToUsageUnitType(usageVoMock, aggregatedQuotaTariffsValue, type.toString()); Double expected = null; @@ -267,7 +269,7 @@ public class QuotaManagerImplTest { Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString()); Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("project"), Mockito.anyString()); - Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString()); + Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString()); } @@ -288,7 +290,7 @@ public class QuotaManagerImplTest { Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("project"), Mockito.anyString()); - Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("resourceType"), Mockito.anyString()); + Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString()); } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index cf1a680f2bb..b973d1145c3 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -27,6 +27,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.cloud.host.HostTagVO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -70,6 +73,7 @@ import com.cloud.storage.GuestOSVO; import com.cloud.storage.ScopeType; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; @@ -239,6 +243,14 @@ public class PresetVariableHelperTest { return host; } + private List getHostTagsForTests() { + return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); + } + + private List getHostRuleTagsForTests() { + return List.of(new HostTagVO(1, "tagrule", true)); + } + private Storage getStorageForTests() { Storage storage = new Storage(); storage.setId("storage_id"); @@ -248,6 +260,14 @@ public class PresetVariableHelperTest { return storage; } + private List getStorageTagsForTests() { + return Arrays.asList(new StoragePoolTagVO(1, "tag1", false), new StoragePoolTagVO(1, "tag2", false)); + } + + private List getStorageRuleTagsForTests() { + return List.of(new StoragePoolTagVO(1, "tagrule", true)); + } + private Set> getQuotaTypesForTests(Integer... typesToRemove) { Map quotaTypesMap = new LinkedHashMap<>(QuotaTypes.listQuotaTypes()); @@ -485,38 +505,42 @@ public class PresetVariableHelperTest { @Test public void loadPresetVariableValueForRunningAndAllocatedVmTestRecordIsRunningOrAllocatedVmSetFields() { - Value expected = getValueForTests(); + for (Hypervisor.HypervisorType hypervisorType : Hypervisor.HypervisorType.values()) { + Value expected = getValueForTests(); - Mockito.doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + Mockito.doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); - mockMethodValidateIfObjectIsNull(); + mockMethodValidateIfObjectIsNull(); - Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableHostInValueIfUsageTypeIsRunningVm(Mockito.any(Value.class), Mockito.anyInt(), - Mockito.any(VMInstanceVO.class)); + Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableHostInValueIfUsageTypeIsRunningVm(Mockito.any(Value.class), Mockito.anyInt(), + Mockito.any(VMInstanceVO.class)); - Mockito.doReturn(expected.getId()).when(vmInstanceVoMock).getUuid(); - Mockito.doReturn(expected.getName()).when(vmInstanceVoMock).getHostName(); - Mockito.doReturn(expected.getOsName()).when(presetVariableHelperSpy).getPresetVariableValueOsName(Mockito.anyLong()); - Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableValueServiceOfferingAndComputingResources(Mockito.any(), Mockito.anyInt(), Mockito.any()); - Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); - Mockito.doReturn(expected.getTemplate()).when(presetVariableHelperSpy).getPresetVariableValueTemplate(Mockito.anyLong()); + Mockito.doReturn(expected.getId()).when(vmInstanceVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(vmInstanceVoMock).getHostName(); + Mockito.doReturn(expected.getOsName()).when(presetVariableHelperSpy).getPresetVariableValueOsName(Mockito.anyLong()); + Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableValueServiceOfferingAndComputingResources(Mockito.any(), Mockito.anyInt(), Mockito.any()); + Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); + Mockito.doReturn(expected.getTemplate()).when(presetVariableHelperSpy).getPresetVariableValueTemplate(Mockito.anyLong()); + Mockito.doReturn(hypervisorType).when(vmInstanceVoMock).getHypervisorType(); - runningAndAllocatedVmUsageTypes.forEach(type -> { - Mockito.doReturn(type).when(usageVoMock).getUsageType(); + runningAndAllocatedVmUsageTypes.forEach(type -> { + Mockito.doReturn(type).when(usageVoMock).getUsageType(); - Value result = new Value(); - presetVariableHelperSpy.loadPresetVariableValueForRunningAndAllocatedVm(usageVoMock, result); + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForRunningAndAllocatedVm(usageVoMock, result); - assertPresetVariableIdAndName(expected, result); - Assert.assertEquals(expected.getOsName(), result.getOsName()); - Assert.assertEquals(expected.getTags(), result.getTags()); - Assert.assertEquals(expected.getTemplate(), result.getTemplate()); + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getOsName(), result.getOsName()); + Assert.assertEquals(expected.getTags(), result.getTags()); + Assert.assertEquals(expected.getTemplate(), result.getTemplate()); + Assert.assertEquals(hypervisorType.name(), result.getHypervisorType()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "osName", "tags", "template"), result); - }); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "osName", "tags", "template", "hypervisorType"), result); + }); + } - Mockito.verify(presetVariableHelperSpy, Mockito.times(runningAndAllocatedVmUsageTypes.size())).getPresetVariableValueResourceTags(Mockito.anyLong(), - Mockito.eq(ResourceObjectType.UserVm)); + Mockito.verify(presetVariableHelperSpy, Mockito.times(runningAndAllocatedVmUsageTypes.size() * Hypervisor.HypervisorType.values().length)) + .getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.UserVm)); } @Test @@ -533,15 +557,16 @@ public class PresetVariableHelperTest { @Test public void setPresetVariableHostInValueIfUsageTypeIsRunningVmTestQuotaTypeIsRunningVmSetHost() { Value result = new Value(); - Host expected = getHostForTests(); + Host expectedHost = getHostForTests(); + List expectedHostTags = getHostTagsForTests(); - Mockito.doReturn(expected).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong()); + Mockito.doReturn(expectedHost).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong()); presetVariableHelperSpy.setPresetVariableHostInValueIfUsageTypeIsRunningVm(result, UsageTypes.RUNNING_VM, vmInstanceVoMock); Assert.assertNotNull(result.getHost()); - assertPresetVariableIdAndName(expected, result.getHost()); - Assert.assertEquals(expected.getTags(), result.getHost().getTags()); + assertPresetVariableIdAndName(expectedHost, result.getHost()); + Assert.assertEquals(expectedHost.getTags(), result.getHost().getTags()); validateFieldNamesToIncludeInToString(Arrays.asList("host"), result); } @@ -549,18 +574,39 @@ public class PresetVariableHelperTest { public void getPresetVariableValueHostTestSetFieldsAndReturnObject() { Host expected = getHostForTests(); HostVO hostVoMock = Mockito.mock(HostVO.class); + List hostTagVOListMock = getHostTagsForTests(); Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); mockMethodValidateIfObjectIsNull(); Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(hostVoMock).getName(); - Mockito.doReturn(expected.getTags()).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); + Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l); assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getTags(), result.getTags()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "tags"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result); + } + + @Test + public void getPresetVariableValueHostTestSetFieldsWithRuleTagAndReturnObject() { + Host expected = getHostForTests(); + HostVO hostVoMock = Mockito.mock(HostVO.class); + List hostTagVOListMock = getHostRuleTagsForTests(); + + Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(hostVoMock).getName(); + Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); + + Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(new ArrayList<>(), result.getTags()); + Assert.assertTrue(result.getIsTagARule()); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result); } @Test @@ -636,75 +682,85 @@ public class PresetVariableHelperTest { @Test public void loadPresetVariableValueForVolumeTestRecordIsVolumeAndHasStorageSetFields() { - Value expected = getValueForTests(); + for (ImageFormat imageFormat : ImageFormat.values()) { + Value expected = getValueForTests(); - VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class); - Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); - Mockito.doReturn(1l).when(volumeVoMock).getPoolId(); + VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class); + Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + Mockito.doReturn(1l).when(volumeVoMock).getPoolId(); - mockMethodValidateIfObjectIsNull(); + mockMethodValidateIfObjectIsNull(); - Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid(); - Mockito.doReturn(expected.getName()).when(volumeVoMock).getName(); - Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong()); - Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType(); - Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt()); - Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); - Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize(); + Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(volumeVoMock).getName(); + Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong()); + Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType(); + Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt()); + Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); + Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize(); + Mockito.doReturn(imageFormat).when(volumeVoMock).getFormat(); - Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType(); + Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType(); - Value result = new Value(); - presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result); + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result); - Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize()); + Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize()); - assertPresetVariableIdAndName(expected, result); - Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering()); - Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType()); - Assert.assertEquals(expected.getStorage(), result.getStorage()); - Assert.assertEquals(expected.getTags(), result.getTags()); - Assert.assertEquals(expectedSize, result.getSize()); + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering()); + Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType()); + Assert.assertEquals(expected.getStorage(), result.getStorage()); + Assert.assertEquals(expected.getTags(), result.getTags()); + Assert.assertEquals(expectedSize, result.getSize()); + Assert.assertEquals(imageFormat.name(), result.getVolumeFormat()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "storage", "tags", "size"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "storage", "tags", "size", "volumeFormat"), result); + } - Mockito.verify(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.Volume)); + Mockito.verify(presetVariableHelperSpy, Mockito.times(ImageFormat.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(), + Mockito.eq(ResourceObjectType.Volume)); } @Test public void loadPresetVariableValueForVolumeTestRecordIsVolumeAndDoesNotHaveStorageSetFields() { - Value expected = getValueForTests(); + for (ImageFormat imageFormat : ImageFormat.values()) { + Value expected = getValueForTests(); - VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class); - Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); - Mockito.doReturn(null).when(volumeVoMock).getPoolId(); + VolumeVO volumeVoMock = Mockito.mock(VolumeVO.class); + Mockito.doReturn(volumeVoMock).when(volumeDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + Mockito.doReturn(null).when(volumeVoMock).getPoolId(); - mockMethodValidateIfObjectIsNull(); + mockMethodValidateIfObjectIsNull(); - Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid(); - Mockito.doReturn(expected.getName()).when(volumeVoMock).getName(); - Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong()); - Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType(); - Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); - Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize(); + Mockito.doReturn(expected.getId()).when(volumeVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(volumeVoMock).getName(); + Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong()); + Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType(); + Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); + Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize(); + Mockito.doReturn(imageFormat).when(volumeVoMock).getFormat(); - Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType(); + Mockito.doReturn(UsageTypes.VOLUME).when(usageVoMock).getUsageType(); - Value result = new Value(); - presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result); + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForVolume(usageVoMock, result); - Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize()); + Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize()); - assertPresetVariableIdAndName(expected, result); - Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering()); - Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType()); - Assert.assertEquals(null, result.getStorage()); - Assert.assertEquals(expected.getTags(), result.getTags()); - Assert.assertEquals(expectedSize, result.getSize()); + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering()); + Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType()); + Assert.assertNull(result.getStorage()); + Assert.assertEquals(expected.getTags(), result.getTags()); + Assert.assertEquals(expectedSize, result.getSize()); + Assert.assertEquals(imageFormat.name(), result.getVolumeFormat()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "tags", "size"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "tags", "size", "volumeFormat"), result); + } - Mockito.verify(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.Volume)); + Mockito.verify(presetVariableHelperSpy, Mockito.times(ImageFormat.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(), + Mockito.eq(ResourceObjectType.Volume)); } @Test @@ -739,13 +795,15 @@ public class PresetVariableHelperTest { Storage expected = getStorageForTests(); Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt()); + List storageTagVOListMock = getStorageTagsForTests(); + StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class); Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName(); Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope(); - Mockito.doReturn(expected.getTags()).when(storagePoolTagsDaoMock).getStoragePoolTags(Mockito.anyLong()); + Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong()); Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2); @@ -753,7 +811,32 @@ public class PresetVariableHelperTest { Assert.assertEquals(expected.getScope(), result.getScope()); Assert.assertEquals(expected.getTags(), result.getTags()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "scope", "tags"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "scope", "tags"), result); + } + + @Test + public void getPresetVariableValueStorageTestGetSecondaryStorageForSnapshotReturnsNullWithRuleTag() { + Storage expected = getStorageForTests(); + Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt()); + + List storageTagVOListMock = getStorageRuleTagsForTests(); + + StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class); + Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName(); + Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope(); + Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong()); + + Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getScope(), result.getScope()); + Assert.assertEquals(new ArrayList<>(), result.getTags()); + Assert.assertTrue(result.getIsTagARule()); + + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "scope", "tags"), result); } @Test @@ -852,37 +935,42 @@ public class PresetVariableHelperTest { @Test public void loadPresetVariableValueForSnapshotTestRecordIsSnapshotSetFields() { - Value expected = getValueForTests(); + for (Hypervisor.HypervisorType hypervisorType : Hypervisor.HypervisorType.values()) { + Value expected = getValueForTests(); - SnapshotVO snapshotVoMock = Mockito.mock(SnapshotVO.class); - Mockito.doReturn(snapshotVoMock).when(snapshotDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + SnapshotVO snapshotVoMock = Mockito.mock(SnapshotVO.class); + Mockito.doReturn(snapshotVoMock).when(snapshotDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); - mockMethodValidateIfObjectIsNull(); + mockMethodValidateIfObjectIsNull(); - Mockito.doReturn(expected.getId()).when(snapshotVoMock).getUuid(); - Mockito.doReturn(expected.getName()).when(snapshotVoMock).getName(); - Mockito.doReturn(expected.getSize()).when(snapshotVoMock).getSize(); - Mockito.doReturn((short) 3).when(snapshotVoMock).getSnapshotType(); - Mockito.doReturn(1l).when(presetVariableHelperSpy).getSnapshotDataStoreId(Mockito.anyLong(), Mockito.anyLong()); - Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt()); - Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); + Mockito.doReturn(expected.getId()).when(snapshotVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(snapshotVoMock).getName(); + Mockito.doReturn(expected.getSize()).when(snapshotVoMock).getSize(); + Mockito.doReturn((short) 3).when(snapshotVoMock).getSnapshotType(); + Mockito.doReturn(1l).when(presetVariableHelperSpy).getSnapshotDataStoreId(Mockito.anyLong(), Mockito.anyLong()); + Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt()); + Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); + Mockito.doReturn(hypervisorType).when(snapshotVoMock).getHypervisorType(); - Mockito.doReturn(UsageTypes.SNAPSHOT).when(usageVoMock).getUsageType(); + Mockito.doReturn(UsageTypes.SNAPSHOT).when(usageVoMock).getUsageType(); - Value result = new Value(); - presetVariableHelperSpy.loadPresetVariableValueForSnapshot(usageVoMock, result); + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForSnapshot(usageVoMock, result); - Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize()); + Long expectedSize = ByteScaleUtils.bytesToMebibytes(expected.getSize()); - assertPresetVariableIdAndName(expected, result); - Assert.assertEquals(expected.getSnapshotType(), result.getSnapshotType()); - Assert.assertEquals(expected.getStorage(), result.getStorage()); - Assert.assertEquals(expected.getTags(), result.getTags()); - Assert.assertEquals(expectedSize, result.getSize()); + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getSnapshotType(), result.getSnapshotType()); + Assert.assertEquals(expected.getStorage(), result.getStorage()); + Assert.assertEquals(expected.getTags(), result.getTags()); + Assert.assertEquals(expectedSize, result.getSize()); + Assert.assertEquals(hypervisorType.name(), result.getHypervisorType()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "snapshotType", "storage", "tags", "size"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "snapshotType", "storage", "tags", "size", "hypervisorType"), result); + } - Mockito.verify(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.eq(ResourceObjectType.Snapshot)); + Mockito.verify(presetVariableHelperSpy, Mockito.times(Hypervisor.HypervisorType.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(), + Mockito.eq(ResourceObjectType.Snapshot)); } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java index c9d14401b3f..9e65de754c8 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/ValueTest.java @@ -136,4 +136,18 @@ public class ValueTest { variable.setBackupOffering(null); Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("backupOffering")); } + + @Test + public void setHypervisorTypeTestAddFieldHypervisorTypeToCollection() { + Value variable = new Value(); + variable.setHypervisorType(null); + Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("hypervisorType")); + } + + @Test + public void setVolumeFormatTestAddFieldVolumeFormatToCollection() { + Value variable = new Value(); + variable.setVolumeFormat(null); + Assert.assertTrue(variable.fieldNamesToIncludeInToString.contains("volumeFormat")); + } } diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index 8a46d10a7b5..70920df5eb5 100644 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -22,7 +22,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.ListUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -72,7 +74,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { } String hostTag = offering.getHostTag(); if (hostTag != null) { - s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId + " having host tag:" + hostTag); + s_logger.debug(String.format("Looking for hosts in dc [%s], pod [%s], cluster [%s] and complying with host tag [%s].", dcId, podId, clusterId, hostTag)); } else { s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); } @@ -82,7 +84,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { if (hostTag != null) { hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag)); } else { - hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId)); + hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); } } else { // list all computing hosts, regardless of whether they support routing...it's random after all @@ -90,9 +92,16 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { if (hostTag != null) { hostsCopy = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag); } else { - hostsCopy = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); } } + hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTag)); + + if (hostsCopy.isEmpty()) { + s_logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTag)); + throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile)); + } + s_logger.debug("Random Allocator found " + hostsCopy.size() + " hosts"); if (hostsCopy.size() == 0) { return suitableHosts; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 093070fddd6..b9abea4f0bc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -28,15 +28,17 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.OutputInterpreter.AllLinesParser; import com.cloud.utils.script.Script; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; public class KVMHABase { private static final Logger s_logger = Logger.getLogger(KVMHABase.class); private long _timeout = 60000; /* 1 minutes */ protected static String s_heartBeatPath; - protected long _heartBeatUpdateTimeout = 60000; - protected long _heartBeatUpdateFreq = 60000; - protected long _heartBeatUpdateMaxTries = 5; - protected long _heartBeatUpdateRetrySleep = 10000; + protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); + protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); + protected long _heartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); public static enum PoolType { PrimaryStorage, SecondaryStorage diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index 72944b54e92..eb09408c14e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -48,7 +48,6 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { hostPrivateIp = host; configureHeartBeatPath(scriptPath); - _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 98e779253cc..43a09ccf2bf 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -19,6 +19,8 @@ package com.cloud.hypervisor.kvm.storage; import java.util.List; import java.util.Map; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.joda.time.Duration; @@ -29,11 +31,11 @@ import com.cloud.storage.Storage.StoragePoolType; public interface KVMStoragePool { - public static final long HeartBeatUpdateTimeout = 60000; - public static final long HeartBeatUpdateFreq = 60000; - public static final long HeartBeatUpdateMaxTries = 5; - public static final long HeartBeatUpdateRetrySleep = 10000; - public static final long HeartBeatCheckerTimeout = 360000; // 6 minutes + public static final long HeartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); + public static final long HeartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + public static final long HeartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); + public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); + public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 987825921b8..b1842f38da2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -294,12 +294,14 @@ public class KVMStoragePoolManager { String uuid = null; String sourceHost = ""; StoragePoolType protocol = null; - if (storageUri.getScheme().equalsIgnoreCase("nfs") || storageUri.getScheme().equalsIgnoreCase("NetworkFilesystem")) { + final String scheme = storageUri.getScheme().toLowerCase(); + List acceptedSchemes = List.of("nfs", "networkfilesystem", "filesystem"); + if (acceptedSchemes.contains(scheme)) { sourcePath = storageUri.getPath(); sourcePath = sourcePath.replace("//", "/"); sourceHost = storageUri.getHost(); uuid = UUID.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString(); - protocol = StoragePoolType.NetworkFilesystem; + protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem; } // secondary storage registers itself through here diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index f7ec09ca50f..dd31025d35f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -25,23 +25,26 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; import javax.naming.ConfigurationException; -import org.apache.cloudstack.direct.download.DirectDownloadHelper; -import org.apache.cloudstack.direct.download.DirectTemplateDownloader; -import com.cloud.storage.ScopeType; -import com.cloud.storage.Volume; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; +import org.apache.cloudstack.direct.download.DirectDownloadHelper; +import org.apache.cloudstack.direct.download.DirectTemplateDownloader; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; @@ -71,7 +74,10 @@ import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FileUtils; - +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.Domain; @@ -110,9 +116,11 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef.DiskProtocol; import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper; import com.cloud.storage.JavaStorageLayer; import com.cloud.storage.MigrationOptions; +import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; +import com.cloud.storage.Volume; import com.cloud.storage.resource.StorageProcessor; import com.cloud.storage.template.Processor; import com.cloud.storage.template.Processor.FormatInfo; @@ -126,16 +134,6 @@ import com.cloud.utils.script.Script; import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.VmDetailConstants; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.builder.ToStringBuilder; -import org.apache.commons.lang3.builder.ToStringStyle; - public class KVMStorageProcessor implements StorageProcessor { private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class); private final KVMStoragePoolManager storagePoolMgr; @@ -1074,7 +1072,8 @@ public class KVMStorageProcessor implements StorageProcessor { s_logger.debug(String.format("This backup is temporary, not deleting snapshot [%s] on primary storage [%s]", snapshotPath, primaryPool.getUuid())); } } - protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map params) throws + + protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map params, DataStoreTO store) throws LibvirtException, InternalErrorException { DiskDef iso = new DiskDef(); boolean isUefiEnabled = MapUtils.isNotEmpty(params) && params.containsKey("UEFI"); @@ -1082,8 +1081,14 @@ public class KVMStorageProcessor implements StorageProcessor { final int index = isoPath.lastIndexOf("/"); final String path = isoPath.substring(0, index); final String name = isoPath.substring(index + 1); - final KVMStoragePool secondaryPool = storagePoolMgr.getStoragePoolByURI(path); - final KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name); + KVMStoragePool storagePool; + if (store instanceof PrimaryDataStoreTO) { + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO)store; + storagePool = storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), store.getUuid()); + } else { + storagePool = storagePoolMgr.getStoragePoolByURI(path); + } + final KVMPhysicalDisk isoVol = storagePool.getPhysicalDisk(name); isoPath = isoVol.getPath(); iso.defISODisk(isoPath, isUefiEnabled); @@ -1112,7 +1117,7 @@ public class KVMStorageProcessor implements StorageProcessor { try { String dataStoreUrl = getDataStoreUrlFromStore(store); final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName()); - attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo()); + attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), true, cmd.getControllerInfo(), store); } catch (final LibvirtException e) { return new Answer(cmd, false, e.toString()); } catch (final InternalErrorException e) { @@ -1133,7 +1138,7 @@ public class KVMStorageProcessor implements StorageProcessor { try { String dataStoreUrl = getDataStoreUrlFromStore(store); final Connect conn = LibvirtConnection.getConnectionByVmName(cmd.getVmName()); - attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams()); + attachOrDetachISO(conn, cmd.getVmName(), dataStoreUrl + File.separator + isoTO.getPath(), false, cmd.getParams(), store); } catch (final LibvirtException e) { return new Answer(cmd, false, e.toString()); } catch (final InternalErrorException e) { @@ -1149,19 +1154,25 @@ public class KVMStorageProcessor implements StorageProcessor { * Return data store URL from store */ private String getDataStoreUrlFromStore(DataStoreTO store) { - if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) || - store instanceof PrimaryDataStoreTO && !((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem))) { + List supportedPoolType = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.Filesystem); + if (!(store instanceof NfsTO) && (!(store instanceof PrimaryDataStoreTO) || !supportedPoolType.contains(((PrimaryDataStoreTO) store).getPoolType()))) { + s_logger.error(String.format("Unsupported protocol, store: %s", store.getUuid())); throw new InvalidParameterValueException("unsupported protocol"); } if (store instanceof NfsTO) { NfsTO nfsStore = (NfsTO)store; return nfsStore.getUrl(); - } else if (store instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO) store).getPoolType().equals(StoragePoolType.NetworkFilesystem)) { + } else if (store instanceof PrimaryDataStoreTO) { //In order to support directly downloaded ISOs + StoragePoolType poolType = ((PrimaryDataStoreTO)store).getPoolType(); String psHost = ((PrimaryDataStoreTO) store).getHost(); String psPath = ((PrimaryDataStoreTO) store).getPath(); - return "nfs://" + psHost + File.separator + psPath; + if (StoragePoolType.NetworkFilesystem.equals(poolType)) { + return "nfs://" + psHost + File.separator + psPath; + } else if (StoragePoolType.Filesystem.equals(poolType)) { + return StoragePoolType.Filesystem.toString().toLowerCase() + "://" + psHost + File.separator + psPath; + } } return store.getUrl(); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 990a1875a57..d60dc99a429 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -129,15 +129,15 @@ class VmwareVmImplementer { } } } else { - // for user-VM, use E1000 as default if (nicDeviceType == null) { - details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString()); + details.put(VmDetailConstants.NIC_ADAPTER, vmwareMgr.VmwareUserVmNicDeviceType.value()); } else { try { VirtualEthernetCardType.valueOf(nicDeviceType); } catch (Exception e) { - LOGGER.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); - details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString()); + LOGGER.warn(String.format("Invalid NIC device type [%s] specified in VM details, switching to value [%s] of configuration [%s].", + nicDeviceType, vmwareMgr.VmwareUserVmNicDeviceType.value(), vmwareMgr.VmwareUserVmNicDeviceType.toString())); + details.put(VmDetailConstants.NIC_ADAPTER, vmwareMgr.VmwareUserVmNicDeviceType.value()); } } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java index c2cdbccdae7..d64a2d38ece 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -53,6 +53,14 @@ public interface VmwareManager { "VMware interval window (in seconds) to collect metrics. If this is set to less than 20, then default (300 seconds) will be used. The interval used must be enabled in vCenter for this change to work, " + "otherwise the collection of metrics will result in an error. Check VMWare docs to know how to enable metrics interval.", true); + static final ConfigKey VmwareUserVmNicDeviceType = new ConfigKey( + String.class, + "vmware.uservm.nic.device.type", + "Advanced", + "E1000", + "Specify the default network device type for user VMs, valid values are E1000, PCNet32, Vmxnet2, Vmxnet3", + true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "E1000,PCNet32,Vmxnet2,Vmxnet3"); + String composeWorkerName(); String getSystemVMIsoFileNameOnDatastore(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 2a41d2c1956..f88c89307cc 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -296,7 +296,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout, s_vmwareCleanupPortGroups, VMWARE_STATS_TIME_WINDOW}; + return new ConfigKey[] {s_vmwareNicHotplugWaitTimeout, s_vmwareCleanOldWorderVMs, templateCleanupInterval, s_vmwareSearchExcludeFolder, s_vmwareOVAPackageTimeout, s_vmwareCleanupPortGroups, VMWARE_STATS_TIME_WINDOW, VmwareUserVmNicDeviceType}; } @Override public boolean configure(String name, Map params) throws ConfigurationException { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index f72115abd8b..6fb3ead006b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -128,6 +128,7 @@ import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.network.router.NetworkHelper; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.security.SecurityGroupManager; @@ -256,6 +257,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne private SecurityGroupManager securityGroupManager; @Inject public SecurityGroupService securityGroupService; + @Inject + public NetworkHelper networkHelper; @Inject private UserVmService userVmService; @@ -363,8 +366,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) { VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType); + if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) { + LOGGER.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter)); + template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId())); + } if (template == null) { - throw new CloudRuntimeException("Not able to find the System templates or not downloaded in zone " + dataCenter.getId()); + throw new CloudRuntimeException("Not able to find the System or Routing template in ready state for the zone " + dataCenter.getUuid()); } return template; } @@ -631,7 +638,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } - private DataCenter validateAndGetZoneForKubernetesCreateParameters(Long zoneId) { + private DataCenter validateAndGetZoneForKubernetesCreateParameters(Long zoneId, Long networkId) { DataCenter zone = dataCenterDao.findById(zoneId); if (zone == null) { throw new InvalidParameterValueException("Unable to find zone by ID: " + zoneId); @@ -639,6 +646,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (zone.getAllocationState() == Grouping.AllocationState.Disabled) { throw new PermissionDeniedException(String.format("Cannot perform this operation, zone ID: %s is currently disabled", zone.getUuid())); } + if (DataCenter.Type.Edge.equals(zone.getType()) && networkId == null) { + throw new PermissionDeniedException("Kubernetes clusters cannot be created on an edge zone without an existing network"); + } return zone; } @@ -678,7 +688,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name: " + name); } - validateAndGetZoneForKubernetesCreateParameters(zoneId); + validateAndGetZoneForKubernetesCreateParameters(zoneId, networkId); validateSshKeyPairForKubernetesCreateParameters(sshKeyPair, owner); if (nodeRootDiskSize != null && nodeRootDiskSize <= 0) { @@ -754,7 +764,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne String.format("Maximum cluster size can not exceed %d. Please contact your administrator", maxClusterSize)); } - DataCenter zone = validateAndGetZoneForKubernetesCreateParameters(zoneId); + DataCenter zone = validateAndGetZoneForKubernetesCreateParameters(zoneId, networkId); if (!isKubernetesServiceConfigured(zone)) { throw new CloudRuntimeException("Kubernetes service has not been configured properly to provision Kubernetes clusters"); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 643044f78d8..3ea30291f43 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -31,10 +31,12 @@ import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesSupportedVersionsCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; +import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; @@ -56,7 +58,6 @@ import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.lang3.StringUtils; public class KubernetesVersionManagerImpl extends ManagerBase implements KubernetesVersionService { public static final Logger LOGGER = Logger.getLogger(KubernetesVersionManagerImpl.class.getName()); @@ -104,6 +105,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne response.setIsoId(template.getUuid()); response.setIsoName(template.getName()); response.setIsoState(template.getState().toString()); + response.setDirectDownload(template.isDirectDownload()); } response.setCreated(kubernetesSupportedVersion.getCreated()); return response; @@ -147,7 +149,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne return versions; } - private VirtualMachineTemplate registerKubernetesVersionIso(final Long zoneId, final String versionName, final String isoUrl, final String isoChecksum)throws IllegalAccessException, NoSuchFieldException, + private VirtualMachineTemplate registerKubernetesVersionIso(final Long zoneId, final String versionName, final String isoUrl, final String isoChecksum, final boolean directDownload) throws IllegalAccessException, NoSuchFieldException, IllegalArgumentException, ResourceAllocationException { String isoName = String.format("%s-Kubernetes-Binaries-ISO", versionName); RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); @@ -163,6 +165,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne if (StringUtils.isNotEmpty(isoChecksum)) { registerIsoCmd.setChecksum(isoChecksum); } + registerIsoCmd.setDirectDownload(directDownload); registerIsoCmd.setAccountName(accountManager.getSystemAccount().getAccountName()); registerIsoCmd.setDomainId(accountManager.getSystemAccount().getDomainId()); return templateService.registerIso(registerIsoCmd); @@ -288,6 +291,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne final String isoChecksum = cmd.getChecksum(); final Integer minimumCpu = cmd.getMinimumCpu(); final Integer minimumRamSize = cmd.getMinimumRamSize(); + final boolean isDirectDownload = cmd.isDirectDownload(); if (minimumCpu == null || minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) { throw new InvalidParameterValueException(String.format("Invalid value for %s parameter. Minimum %d vCPUs required.", ApiConstants.MIN_CPU_NUMBER, KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU)); } @@ -297,8 +301,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); } - if (zoneId != null && dataCenterDao.findById(zoneId) == null) { - throw new InvalidParameterValueException("Invalid zone specified"); + if (zoneId != null) { + DataCenter zone = dataCenterDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Invalid zone specified"); + } + if (DataCenter.Type.Edge.equals(zone.getType()) && !isDirectDownload) { + throw new InvalidParameterValueException(String.format("Zone: %s supports only direct download Kubernetes versions", zone.getName())); + } } if (StringUtils.isEmpty(isoUrl)) { throw new InvalidParameterValueException(String.format("Invalid URL for ISO specified, %s", isoUrl)); @@ -312,7 +322,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne VMTemplateVO template = null; try { - VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(zoneId, name, isoUrl, isoChecksum); + VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(zoneId, name, isoUrl, isoChecksum, isDirectDownload); template = templateDao.findById(vmTemplate.getId()); } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException | ResourceAllocationException ex) { LOGGER.error(String.format("Unable to register binaries ISO for supported kubernetes version, %s, with url: %s", name, isoUrl), ex); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index 48bb26c8d44..380c93cca20 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; @@ -38,7 +39,6 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesVersionService; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.commons.lang3.StringUtils; @APICommand(name = "addKubernetesSupportedVersion", description = "Add a supported Kubernetes version", @@ -84,6 +84,10 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCm description = "the minimum RAM size in MB to be set with the Kubernetes version") private Integer minimumRamSize; + @Parameter(name=ApiConstants.DIRECT_DOWNLOAD, type = CommandType.BOOLEAN, since="4.18.2", + description = "If set to true the Kubernetes supported version ISO will bypass Secondary Storage and be downloaded to Primary Storage on deployment. Default is false") + private Boolean directDownload; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -123,6 +127,10 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCm return minimumRamSize; } + public boolean isDirectDownload() { + return (directDownload != null) ? directDownload : false; + } + @Override public long getEntityOwnerId() { return CallContext.current().getCallingAccountId(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index 72a52682364..188328ac008 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -86,6 +86,10 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "the date when this Kubernetes supported version was created") private Date created; + @SerializedName(ApiConstants.DIRECT_DOWNLOAD) + @Param(description = "KVM Only: true if the ISO for the Kubernetes supported version is directly downloaded to Primary Storage bypassing Secondary Storage", since = "4.18.2") + private Boolean directDownload; + public String getId() { return id; } @@ -193,4 +197,8 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { public void setCreated(Date created) { this.created = created; } + + public void setDirectDownload(Boolean directDownload) { + this.directDownload = directDownload; + } } diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index ee48252534f..3b111da5961 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -29,6 +29,7 @@ import com.cloud.configuration.dao.ResourceCountDao; import com.cloud.dc.DedicatedResourceVO; import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.host.HostStats; +import com.cloud.host.HostTagVO; import com.cloud.user.Account; import com.cloud.user.dao.AccountDao; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; @@ -234,7 +235,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp } private String markTagMaps(HostVO host, Map totalHosts, Map upHosts, Map downHosts) { - List hostTags = _hostTagsDao.getHostTags(host.getId()); + List hostTagVOS = _hostTagsDao.getHostTags(host.getId()); + List hostTags = new ArrayList<>(); + hostTagVOS.forEach(hostTagVO -> hostTags.add(hostTagVO.getTag())); markTags(hostTags,totalHosts); if (host.getStatus() == Status.Up && !host.isInMaintenanceStates()) { markTags(hostTags, upHosts); @@ -277,10 +280,12 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag)); }); - List allHostTags = hostDao.listAll().stream() + List allHostTagVOS = hostDao.listAll().stream() .flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream()) .distinct() .collect(Collectors.toList()); + List allHostTags = new ArrayList<>(); + allHostTagVOS.forEach(hostTagVO -> allHostTags.add(hostTagVO.getTag())); for (final State state : State.values()) { for (final String hostTag : allHostTags) { diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java index b5ac137dadd..cc3b7d55653 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java @@ -28,6 +28,7 @@ import javax.inject.Inject; import java.io.IOException; import java.io.OutputStream; import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.util.Arrays; public class PrometheusExporterServerImpl extends ManagerBase implements PrometheusExporterServer, Configurable { @@ -57,11 +58,21 @@ public class PrometheusExporterServerImpl extends ManagerBase implements Prometh response = prometheusExporter.getMetrics(); responseCode = 200; } - httpExchange.getResponseHeaders().set("content-type", "text/plain"); - httpExchange.sendResponseHeaders(responseCode, response.length()); + byte[] bytesToOutput = response.getBytes(StandardCharsets.UTF_8); + httpExchange.getResponseHeaders().set("content-type", "text/plain; charset=UTF-8"); + httpExchange.sendResponseHeaders(responseCode, bytesToOutput.length); final OutputStream os = httpExchange.getResponseBody(); - os.write(response.getBytes()); - os.close(); + try { + os.write(bytesToOutput); + } catch (IOException e) { + LOG.error(String.format("could not export Prometheus data due to %s", e.getLocalizedMessage())); + if (LOG.isDebugEnabled()) { + LOG.debug("Error during Prometheus export: ", e); + } + os.write("The system could not export Prometheus due to an internal error. Contact your operator to learn about the reason.".getBytes()); + } finally { + os.close(); + } } } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index a27896aa58d..51c020fc237 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -71,6 +71,7 @@ import org.apache.cloudstack.response.VolumeMetricsResponse; import org.apache.cloudstack.response.VolumeMetricsStatsResponse; import org.apache.cloudstack.response.ZoneMetricsResponse; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.commons.beanutils.BeanUtils; @@ -176,6 +177,9 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements @Inject private VolumeStatsDao volumeStatsDao; + @Inject + private ObjectStoreDao objectStoreDao; + private static Gson gson = new Gson(); protected MetricsServiceImpl() { @@ -557,6 +561,7 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements response.setHosts(hostDao.countAllByType(Host.Type.Routing)); response.setStoragePools(storagePoolDao.countAll()); response.setImageStores(imageStoreDao.countAllImageStores()); + response.setObjectStores(objectStoreDao.countAllObjectStores()); response.setSystemvms(vmInstanceDao.listByTypes(VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm).size()); response.setRouters(domainRouterDao.countAllByRole(VirtualRouter.Role.VIRTUAL_ROUTER)); response.setInternalLbs(domainRouterDao.countAllByRole(VirtualRouter.Role.INTERNAL_LB_VM)); diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java b/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java index 280b7998fbb..cb1faf237b7 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java @@ -47,6 +47,10 @@ public class InfrastructureResponse extends BaseResponse { @Param(description = "Number of images stores") private Integer imageStores; + @SerializedName("objectstores") + @Param(description = "Number of object stores") + private Integer objectStores; + @SerializedName("systemvms") @Param(description = "Number of systemvms") private Integer systemvms; @@ -118,4 +122,8 @@ public class InfrastructureResponse extends BaseResponse { public void setAlerts(Integer alerts) { this.alerts = alerts; } public void setInternalLbs(Integer internalLbs) { this.internalLbs = internalLbs; } + + public void setObjectStores(Integer objectStores) { + this.objectStores = objectStores; + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeletedNsxDistributedFirewallRulesCommand.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxDistributedFirewallRulesCommand.java similarity index 79% rename from plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeletedNsxDistributedFirewallRulesCommand.java rename to plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxDistributedFirewallRulesCommand.java index d0ae6dd9c21..ad88f23b3b1 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeletedNsxDistributedFirewallRulesCommand.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/agent/api/DeleteNsxDistributedFirewallRulesCommand.java @@ -20,8 +20,8 @@ import org.apache.cloudstack.resource.NsxNetworkRule; import java.util.List; -public class DeletedNsxDistributedFirewallRulesCommand extends CreateNsxDistributedFirewallRulesCommand { - public DeletedNsxDistributedFirewallRulesCommand(long domainId, long accountId, long zoneId, Long vpcId, long networkId, List rules) { +public class DeleteNsxDistributedFirewallRulesCommand extends CreateNsxDistributedFirewallRulesCommand { + public DeleteNsxDistributedFirewallRulesCommand(long domainId, long accountId, long zoneId, Long vpcId, long networkId, List rules) { super(domainId, accountId, zoneId, vpcId, networkId, rules); } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java index b115aa802a8..bceee68389c 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxNetworkRule.java @@ -21,6 +21,11 @@ import com.cloud.network.Network; import java.util.List; public class NsxNetworkRule { + + public enum NsxRuleAction { + ALLOW, DROP + } + private long domainId; private long accountId; private long zoneId; @@ -36,7 +41,7 @@ public class NsxNetworkRule { private String protocol; private String algorithm; private List memberList; - private String aclAction; + private NsxRuleAction aclAction; private List sourceCidrList; private List destinationCidrList; private Integer icmpCode; @@ -165,11 +170,11 @@ public class NsxNetworkRule { this.memberList = memberList; } - public String getAclAction() { + public NsxRuleAction getAclAction() { return aclAction; } - public void setAclAction(String aclAction) { + public void setAclAction(NsxRuleAction aclAction) { this.aclAction = aclAction; } @@ -238,7 +243,7 @@ public class NsxNetworkRule { private String protocol; private String algorithm; private List memberList; - private String aclAction; + private NsxRuleAction aclAction; private List sourceCidrList; private List destinationidrList; private String trafficType; @@ -325,11 +330,17 @@ public class NsxNetworkRule { return this; } - public Builder setAclAction(String aclAction) { + + public Builder setAclAction(NsxRuleAction aclAction) { this.aclAction = aclAction; return this; } + public Builder setTrafficType(String trafficType) { + this.trafficType = trafficType; + return this; + } + public Builder setIcmpType(Integer icmpType) { this.icmpType = icmpType; return this; @@ -350,11 +361,6 @@ public class NsxNetworkRule { return this; } - public Builder setTrafficType(String trafficType) { - this.trafficType = trafficType; - return this; - } - public Builder setService(Network.Service service) { this.service = service; return this; diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java index ce0d1fb2960..35017ed27ba 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/resource/NsxResource.java @@ -43,11 +43,11 @@ import org.apache.cloudstack.agent.api.CreateNsxSegmentCommand; import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; +import org.apache.cloudstack.agent.api.DeleteNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.agent.api.DeleteNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; -import org.apache.cloudstack.agent.api.DeletedNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.service.NsxApiClient; import org.apache.cloudstack.utils.NsxControllerUtils; import org.apache.commons.collections.CollectionUtils; @@ -125,8 +125,8 @@ public class NsxResource implements ServerResource { return executeRequest((CreateNsxLoadBalancerRuleCommand) cmd); } else if (cmd instanceof DeleteNsxLoadBalancerRuleCommand) { return executeRequest((DeleteNsxLoadBalancerRuleCommand) cmd); - } else if (cmd instanceof DeletedNsxDistributedFirewallRulesCommand) { - return executeRequest((DeletedNsxDistributedFirewallRulesCommand) cmd); + } else if (cmd instanceof DeleteNsxDistributedFirewallRulesCommand) { + return executeRequest((DeleteNsxDistributedFirewallRulesCommand) cmd); } else if (cmd instanceof CreateNsxDistributedFirewallRulesCommand) { return executeRequest((CreateNsxDistributedFirewallRulesCommand) cmd); } else { @@ -474,7 +474,7 @@ public class NsxResource implements ServerResource { return new NsxAnswer(cmd, true, null); } - private NsxAnswer executeRequest(DeletedNsxDistributedFirewallRulesCommand cmd) { + private NsxAnswer executeRequest(DeleteNsxDistributedFirewallRulesCommand cmd) { String segmentName = NsxControllerUtils.getNsxSegmentId(cmd.getDomainId(), cmd.getAccountId(), cmd.getZoneId(), cmd.getVpcId(), cmd.getNetworkId()); List rules = cmd.getRules(); @@ -487,7 +487,6 @@ public class NsxResource implements ServerResource { return new NsxAnswer(cmd, true, null); } - @Override public boolean start() { return true; diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java index 3139167f56e..24ef915d101 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java @@ -39,6 +39,7 @@ import com.vmware.nsx_policy.model.ApiError; import com.vmware.nsx_policy.model.DhcpRelayConfig; import com.vmware.nsx_policy.model.EnforcementPointListResult; import com.vmware.nsx_policy.model.Group; +import com.vmware.nsx_policy.model.GroupListResult; import com.vmware.nsx_policy.model.ICMPTypeServiceEntry; import com.vmware.nsx_policy.model.L4PortSetServiceEntry; import com.vmware.nsx_policy.model.LBAppProfileListResult; @@ -451,7 +452,7 @@ public class NsxApiClient { // delete NAT rule natService.delete(tier1GatewayName, NatId.USER.name(), ruleName); if (service == Network.Service.PortForwarding) { - String svcName = getServiceName(ruleName, privatePort, protocol); + String svcName = getServiceName(ruleName, privatePort, protocol, null, null); // Delete service Services services = (Services) nsxService.apply(Services.class); services.delete(svcName); @@ -698,23 +699,36 @@ public class NsxApiClient { } } + + private com.vmware.nsx_policy.model.Service getInfraService(String ruleName, String port, String protocol, Integer icmpType, Integer icmpCode) { + Services service = (Services) nsxService.apply(Services.class); + String serviceName = getServiceName(ruleName, port, protocol, icmpType, icmpCode); + createNsxInfraService(service, serviceName, ruleName, port, protocol, icmpType, icmpCode); + return service.get(serviceName); + } + + public String getServicePath(String ruleName, String port, String protocol, Integer icmpType, Integer icmpCode) { + com.vmware.nsx_policy.model.Service svc = getInfraService(ruleName, port, protocol, icmpType, icmpCode); + return svc.getServiceEntries().get(0)._getDataValue().getField("parent_path").toString(); + } + public void createNsxInfraService(Services service, String serviceName, String ruleName, String port, String protocol, - Integer icmpCode, Integer icmpType) { + Integer icmpType, Integer icmpCode) { try { List serviceEntries = new ArrayList<>(); - protocol = "ICMP".equals(protocol) ? "ICMP4" : protocol; + protocol = "ICMP".equalsIgnoreCase(protocol) ? "ICMPv4" : protocol; String serviceEntryName = getServiceEntryName(ruleName, port, protocol); - if (protocol.equals("ICMP4")) { + if (protocol.equals("ICMPv4")) { serviceEntries.add(new ICMPTypeServiceEntry.Builder() - .setDisplayName(serviceEntryName) - .setDisplayName(serviceEntryName) - .setIcmpCode(Long.valueOf(icmpCode)) + .setId(serviceEntryName) + .setDisplayName(serviceEntryName) +// .setIcmpCode(Long.valueOf(icmpCode)) .setIcmpType(Long.valueOf(icmpType)) .setProtocol(protocol) - .build() + .build() ); } else { - serviceEntries.add( new L4PortSetServiceEntry.Builder() + serviceEntries.add(new L4PortSetServiceEntry.Builder() .setId(serviceEntryName) .setDisplayName(serviceEntryName) .setDestinationPorts(List.of(port)) @@ -735,18 +749,6 @@ public class NsxApiClient { } } - private com.vmware.nsx_policy.model.Service getInfraService(String ruleName, String port, String protocol, Integer icmpType, Integer icmpCode) { - Services service = (Services) nsxService.apply(Services.class); - String serviceName = getServiceName(ruleName, port, protocol); - createNsxInfraService(service, serviceName, ruleName, port, protocol, icmpType, icmpCode); - return service.get(serviceName); - } - - public String getServicePath(String ruleName, String port, String protocol, Integer icmpType, Integer icmpCode) { //com.vmware.nsx_policy.model.Service svc) { - com.vmware.nsx_policy.model.Service svc = getInfraService(ruleName, port, protocol, icmpType, icmpCode); - return svc.getServiceEntries().get(0)._getDataValue().getField("parent_path").toString(); - } - private String getServiceById(String ruleName) { try { Services service = (Services) nsxService.apply(Services.class); @@ -802,6 +804,10 @@ public class NsxApiClient { public void createSegmentDistributedFirewall(String segmentName, List nsxRules) { try { + String groupPath = getGroupPath(segmentName); + if (Objects.isNull(groupPath)) { + throw new CloudRuntimeException(String.format("Failed to find group for segment %s", segmentName)); + } SecurityPolicies services = (SecurityPolicies) nsxService.apply(SecurityPolicies.class); List rules = getRulesForDistributedFirewall(segmentName, nsxRules); SecurityPolicy policy = new SecurityPolicy.Builder() @@ -809,6 +815,7 @@ public class NsxApiClient { .setId(segmentName) .setCategory("Application") .setRules(rules) + .setScope(List.of(groupPath)) .build(); services.patch(DEFAULT_DOMAIN, segmentName, policy); } catch (Error error) { @@ -822,7 +829,7 @@ public class NsxApiClient { public void deleteDistributedFirewallRules(String segmentName, List nsxRules) { for(NsxNetworkRule rule : nsxRules) { String ruleId = NsxControllerUtils.getNsxDistributedFirewallPolicyRuleId(segmentName, rule.getRuleId()); - String svcName = getServiceName(ruleId, rule.getPrivatePort(), rule.getProtocol()); + String svcName = getServiceName(ruleId, rule.getPrivatePort(), rule.getProtocol(), rule.getIcmpType(), rule.getIcmpCode()); // delete rules Rules rules = (Rules) nsxService.apply(Rules.class); rules.delete(DEFAULT_DOMAIN, segmentName, ruleId); @@ -834,27 +841,38 @@ public class NsxApiClient { private List getRulesForDistributedFirewall(String segmentName, List nsxRules) { List rules = new ArrayList<>(); - for (NsxNetworkRule rule: nsxRules) { + String groupPath = getGroupPath(segmentName); + if (Objects.isNull(groupPath)) { + throw new CloudRuntimeException(String.format("Failed to find group for segment %s", segmentName)); + } + for (NsxNetworkRule rule : nsxRules) { String ruleId = NsxControllerUtils.getNsxDistributedFirewallPolicyRuleId(segmentName, rule.getRuleId()); Rule ruleToAdd = new Rule.Builder() - .setAction(Objects.nonNull(rule.getAclAction()) ? actionMap.get(rule.getAclAction()) : FirewallActions.ALLOW.name()) + .setAction(rule.getAclAction().toString()) .setId(ruleId) .setDisplayName(ruleId) .setResourceType("SecurityPolicy") .setSourceGroups(getGroupsForTraffic(rule, segmentName, true)) .setDestinationGroups(getGroupsForTraffic(rule, segmentName, false)) - .setServices(Objects.nonNull(rule.getPrivatePort()) && !rule.getPrivatePort().equals("null") ? List.of(getServicePath(ruleId, rule.getPrivatePort(), - rule.getProtocol(), rule.getIcmpType(), rule.getIcmpCode())) : List.of("ANY")) - .setScope(List.of("ANY")) + .setServices(getServicesListForDistributedFirewallRule(rule, segmentName)) + .setScope(List.of(groupPath)) .build(); - if (Objects.nonNull(rule.getPrivatePort())) { - - } rules.add(ruleToAdd); } return rules; } + private List getServicesListForDistributedFirewallRule(NsxNetworkRule rule, String segmentName) { + List services = List.of("ANY"); + if (!rule.getProtocol().equalsIgnoreCase("all")) { + String ruleName = String.format("%s-R%s", segmentName, rule.getRuleId()); + String serviceName = getNsxInfraServices(ruleName, rule.getPrivatePort(), rule.getProtocol(), + rule.getIcmpType(), rule.getIcmpCode()); + services = List.of(serviceName); + } + return services; + } + protected List getGroupsForTraffic(NsxNetworkRule rule, String segmentName, boolean source) { List segmentGroup = List.of(String.format("%s/%s", GROUPS_PATH_PREFIX, segmentName)); @@ -866,10 +884,30 @@ public class NsxApiClient { return source ? sourceCidrList : (rule.getService() == Network.Service.NetworkACL ? segmentGroup : destCidrList); } else if (trafficType.equalsIgnoreCase("egress")) { return source ? segmentGroup : (rule.getService() == Network.Service.NetworkACL ? sourceCidrList : destCidrList); - } + } String err = String.format("Unsupported traffic type %s", trafficType); LOGGER.error(err); throw new CloudRuntimeException(err); } + + private List listNsxGroups() { + try { + Groups groups = (Groups) nsxService.apply(Groups.class); + GroupListResult result = groups.list(DEFAULT_DOMAIN, null, false, null, null, null, null, null); + return result.getResults(); + } catch (Error error) { + ApiError ae = error.getData()._convertTo(ApiError.class); + String msg = String.format("Failed to list NSX groups, due to: %s", ae.getErrorMessage()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + } + + private String getGroupPath(String segmentName) { + List groups = listNsxGroups(); + Optional matchingGroup = groups.stream().filter(group -> group.getDisplayName().equals(segmentName)).findFirst(); + return matchingGroup.map(Group::getPath).orElse(null); + + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java index a9d1a512975..e5a157e6189 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java @@ -591,6 +591,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd())); } + private static String getPrivatePortRangeForACLRule(NetworkACLItem rule) { + return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ? + String.valueOf(rule.getSourcePortStart()) : + String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd())); + } + @Override public boolean applyLBRules(Network network, List rules) throws ResourceUnavailableException { for (LoadBalancingRule loadBalancingRule : rules) { @@ -669,33 +675,36 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns if (!canHandle(network, Network.Service.NetworkACL)) { return false; } - List nsxNetworkRules = new ArrayList<>(); + List nsxAddNetworkRules = new ArrayList<>(); + List nsxDelNetworkRules = new ArrayList<>(); for (NetworkACLItem rule : rules) { + String privatePort = getPrivatePortRangeForACLRule(rule); NsxNetworkRule networkRule = new NsxNetworkRule.Builder() .setRuleId(rule.getId()) .setSourceCidrList(Objects.nonNull(rule.getSourceCidrList()) ? transformCidrListValues(rule.getSourceCidrList()) : List.of("ANY")) - .setAclAction(rule.getAction().toString()) + .setAclAction(transformActionValue(rule.getAction())) .setTrafficType(rule.getTrafficType().toString()) + .setProtocol(rule.getProtocol().toUpperCase()) + .setPublicPort(String.valueOf(rule.getSourcePortStart())) + .setPrivatePort(privatePort) + .setIcmpCode(rule.getIcmpCode()) + .setIcmpType(rule.getIcmpType()) .setService(Network.Service.NetworkACL) .build(); - nsxNetworkRules.add(networkRule); - } - return nsxService.addFirewallRules(network, nsxNetworkRules); - } - - /** - * Replace 0.0.0.0/0 to ANY on each occurrence - */ - private List transformCidrListValues(List sourceCidrList) { - List list = new ArrayList<>(); - for (String cidr : sourceCidrList) { - if (cidr == null || cidr.equals("0.0.0.0/0")) { - list.add("ANY"); - } else { - list.add(cidr); + if (NetworkACLItem.State.Add == rule.getState()) { + nsxAddNetworkRules.add(networkRule); + } else if (NetworkACLItem.State.Revoke == rule.getState()) { + nsxDelNetworkRules.add(networkRule); } } - return list; + boolean success = true; + if (!nsxDelNetworkRules.isEmpty()) { + success = nsxService.deleteFirewallRules(network, nsxDelNetworkRules); + if (!success) { + LOGGER.warn("Not all firewall rules were successfully deleted"); + } + } + return success && nsxService.addFirewallRules(network, nsxAddNetworkRules); } @Override @@ -735,4 +744,32 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns } return success && nsxService.addFirewallRules(network, nsxAddNetworkRules); } + + protected NsxNetworkRule.NsxRuleAction transformActionValue(NetworkACLItem.Action action) { + if (action == NetworkACLItem.Action.Allow) { + return NsxNetworkRule.NsxRuleAction.ALLOW; + } else if (action == NetworkACLItem.Action.Deny) { + return NsxNetworkRule.NsxRuleAction.DROP; + } + String err = String.format("Unsupported action %s", action.toString()); + LOGGER.error(err); + throw new CloudRuntimeException(err); + } + + /** + * Replace 0.0.0.0/0 to ANY on each occurrence + */ + protected List transformCidrListValues(List sourceCidrList) { + List list = new ArrayList<>(); + if (org.apache.commons.collections.CollectionUtils.isNotEmpty(sourceCidrList)) { + for (String cidr : sourceCidrList) { + if (cidr.equals("0.0.0.0/0")) { + list.add("ANY"); + } else { + list.add(cidr); + } + } + } + return list; + } } diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java index 8227084bfc3..481b68f9840 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxServiceImpl.java @@ -32,11 +32,11 @@ import org.apache.cloudstack.agent.api.CreateNsxPortForwardRuleCommand; import org.apache.cloudstack.agent.api.CreateNsxStaticNatCommand; import org.apache.cloudstack.agent.api.CreateNsxTier1GatewayCommand; import org.apache.cloudstack.agent.api.CreateOrUpdateNsxTier1NatRuleCommand; +import org.apache.cloudstack.agent.api.DeleteNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.agent.api.DeleteNsxLoadBalancerRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxSegmentCommand; import org.apache.cloudstack.agent.api.DeleteNsxNatRuleCommand; import org.apache.cloudstack.agent.api.DeleteNsxTier1GatewayCommand; -import org.apache.cloudstack.agent.api.DeletedNsxDistributedFirewallRulesCommand; import org.apache.cloudstack.resource.NsxNetworkRule; import org.apache.cloudstack.utils.NsxControllerUtils; import org.apache.cloudstack.utils.NsxHelper; @@ -184,7 +184,7 @@ public class NsxServiceImpl implements NsxService { } public boolean deleteFirewallRules(Network network, List netRules) { - DeletedNsxDistributedFirewallRulesCommand command = new DeletedNsxDistributedFirewallRulesCommand(network.getDomainId(), + DeleteNsxDistributedFirewallRulesCommand command = new DeleteNsxDistributedFirewallRulesCommand(network.getDomainId(), network.getAccountId(), network.getDataCenterId(), network.getVpcId(), network.getId(), netRules); NsxAnswer result = nsxControllerUtils.sendNsxCommand(command, network.getDataCenterId()); return result.getResult(); diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java index ae356a40f6d..eec19e3e075 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/utils/NsxControllerUtils.java @@ -46,7 +46,7 @@ public class NsxControllerUtils { } public static String getNsxDistributedFirewallPolicyRuleId(String segmentName, long ruleId) { - return String.format("%s-P%s", segmentName, ruleId); + return String.format("%s-R%s", segmentName, ruleId); } public NsxAnswer sendNsxCommand(NsxCommand cmd, long zoneId) throws IllegalArgumentException { @@ -101,8 +101,10 @@ public class NsxControllerUtils { return getTier1GatewayName(domainId, accountId, zoneId, networkResourceId, isVpcResource) + suffix + ruleId; } - public static String getServiceName(String ruleName, String port, String protocol) { - return ruleName + "-SVC-" + port + "-" +protocol; + public static String getServiceName(String ruleName, String port, String protocol, Integer icmpType, Integer icmpCode) { + return protocol.equalsIgnoreCase("icmp") ? + String.format("%s-SVC-%s-%s-%s", ruleName, icmpType, icmpCode, protocol) : + String.format("%s-SVC-%s-%s", ruleName, port, protocol); } public static String getServiceEntryName(String ruleName, String port, String protocol) { diff --git a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java index dc5bea4cf42..e614421954d 100644 --- a/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java +++ b/plugins/network-elements/nsx/src/test/java/org/apache/cloudstack/service/NsxElementTest.java @@ -28,12 +28,15 @@ import com.cloud.network.Networks; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao; import com.cloud.resource.ResourceManager; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.vm.ReservationContext; +import org.apache.cloudstack.resource.NsxNetworkRule; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -129,6 +132,24 @@ public class NsxElementTest { assertTrue(nsxElement.shutdownVpc(vpc, reservationContext)); } + @Test + public void testTransformActionValue() { + NsxNetworkRule.NsxRuleAction action = nsxElement.transformActionValue(NetworkACLItem.Action.Deny); + Assert.assertEquals(NsxNetworkRule.NsxRuleAction.DROP, action); + } + @Test + public void testTransformCidrListValuesEmptyList() { + List values = nsxElement.transformCidrListValues(null); + Assert.assertNotNull(values); + Assert.assertTrue(values.isEmpty()); + } + + @Test + public void testTransformCidrListValuesList() { + List values = nsxElement.transformCidrListValues(List.of("0.0.0.0/0")); + Assert.assertEquals(1, values.size()); + Assert.assertEquals("ANY", values.get(0)); + } } diff --git a/plugins/network-elements/tungsten/src/test/java/org/apache/cloudstack/network/tungsten/service/TungstenApiTest.java b/plugins/network-elements/tungsten/src/test/java/org/apache/cloudstack/network/tungsten/service/TungstenApiTest.java index 3c12bfb5cba..580bea057d8 100644 --- a/plugins/network-elements/tungsten/src/test/java/org/apache/cloudstack/network/tungsten/service/TungstenApiTest.java +++ b/plugins/network-elements/tungsten/src/test/java/org/apache/cloudstack/network/tungsten/service/TungstenApiTest.java @@ -1360,6 +1360,7 @@ public class TungstenApiTest { s_logger.debug("Check if policy was listed all in Tungsten-Fabric"); List policyList3 = tungstenApi.listTungstenPolicy(projectUuid, null); + policyList3.sort(comparator); assertEquals(policyList1, policyList3); s_logger.debug("Check if policy was listed with uuid in Tungsten-Fabric"); @@ -1383,6 +1384,7 @@ public class TungstenApiTest { s_logger.debug("Check if network was listed all in Tungsten-Fabric"); List networkList3 = tungstenApi.listTungstenNetwork(projectUuid, null); + networkList3.sort(comparator); assertEquals(networkList1, networkList3); s_logger.debug("Check if network policy was listed with uuid in Tungsten-Fabric"); diff --git a/plugins/pom.xml b/plugins/pom.xml index 5db7234a2e9..09713993e1a 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -134,6 +134,8 @@ storage/volume/scaleio storage/volume/linstor storage/volume/storpool + storage/object/minio + storage/object/simulator storage-allocators/random @@ -188,6 +190,30 @@ test-jar test + + org.apache.cloudstack + cloud-engine-storage + 4.19.0.0-SNAPSHOT + compile + + + io.minio + minio + 8.5.2 + compile + + + io.minio + minio-admin + 8.5.2 + compile + + + org.apache.cloudstack + cloud-engine-storage-object + 4.19.0.0-SNAPSHOT + compile + diff --git a/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java b/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java index 19ded7844db..9f75251c93f 100644 --- a/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java +++ b/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java @@ -18,12 +18,15 @@ package org.apache.cloudstack.shutdown; import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -39,6 +42,12 @@ public class ShutdownManagerImplTest { @Mock AsyncJobManager jobManagerMock; + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } private long prepareCountPendingJobs() { long expectedCount = 1L; @@ -75,4 +84,9 @@ public class ShutdownManagerImplTest { spy.cancelShutdown(); Mockito.verify(jobManagerMock).enableAsyncJobs(); } + + @After + public void tearDown() throws Exception { + closeable.close(); + } } diff --git a/plugins/storage/object/minio/pom.xml b/plugins/storage/object/minio/pom.xml new file mode 100644 index 00000000000..704772a8ab4 --- /dev/null +++ b/plugins/storage/object/minio/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + cloud-plugin-storage-object-minio + Apache CloudStack Plugin - MinIO object storage provider + + org.apache.cloudstack + cloudstack-plugins + 4.19.0.0-SNAPSHOT + ../../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + io.minio + minio + 8.5.2 + + + io.minio + minio-admin + 8.5.2 + + + diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java new file mode 100644 index 00000000000..15df5df56dc --- /dev/null +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import io.minio.BucketExistsArgs; +import io.minio.DeleteBucketEncryptionArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.RemoveBucketArgs; +import io.minio.SetBucketEncryptionArgs; +import io.minio.SetBucketPolicyArgs; +import io.minio.SetBucketVersioningArgs; +import io.minio.admin.MinioAdminClient; +import io.minio.admin.QuotaUnit; +import io.minio.admin.UserInfo; +import io.minio.admin.messages.DataUsageInfo; +import io.minio.messages.SseConfiguration; +import io.minio.messages.VersioningConfiguration; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.cloudstack.storage.object.BucketObject; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.inject.Inject; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = Logger.getLogger(MinIOObjectStoreDriverImpl.class); + + @Inject + AccountDao _accountDao; + + @Inject + AccountDetailsDao _accountDetailsDao; + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + @Inject + ObjectStoreDetailsDao _storeDetailsDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + private static final String MINIO_ACCESS_KEY = "minio-accesskey"; + private static final String MINIO_SECRET_KEY = "minio-secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + //ToDo Client pool mgmt + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + long accountId = bucket.getAccountId(); + MinioClient minioClient = getMinIOClient(storeId); + Account account = _accountDao.findById(accountId); + + if ((_accountDetailsDao.findDetail(accountId, MINIO_ACCESS_KEY) == null) + || (_accountDetailsDao.findDetail(accountId, MINIO_SECRET_KEY) == null)) { + throw new CloudRuntimeException("Bucket access credentials unavailable for account: "+account.getAccountName()); + } + + try { + if(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket already exists with name "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + try { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(objectLock).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + + List buckets = _bucketDao.listByObjectStoreIdAndAccountId(storeId, accountId); + StringBuilder resources_builder = new StringBuilder(); + for(BucketVO exitingBucket : buckets) { + resources_builder.append("\"arn:aws:s3:::"+exitingBucket.getName()+"/*\",\n"); + } + resources_builder.append("\"arn:aws:s3:::"+bucketName+"/*\"\n"); + + String policy = " {\n" + + " \"Statement\": [\n" + + " {\n" + + " \"Action\": \"s3:*\",\n" + + " \"Effect\": \"Allow\",\n" + + " \"Principal\": \"*\",\n" + + " \"Resource\": ["+resources_builder+"]" + + " }\n" + + " ],\n" + + " \"Version\": \"2012-10-17\"\n" + + " }"; + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String policyName = "acs-"+account.getAccountName()+"-policy"; + String userName = "acs-"+account.getAccountName(); + try { + minioAdminClient.addCannedPolicy(policyName, policy); + minioAdminClient.setPolicy(userName, false, policyName); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + String accessKey = _accountDetailsDao.findDetail(accountId, MINIO_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, MINIO_SECRET_KEY).getValue(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(accessKey); + bucketVO.setSecretKey(secretKey); + bucketVO.setBucketURL(store.getUrl()+"/"+bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucket; + } + + @Override + public List listBuckets(long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + List bucketsList = new ArrayList<>(); + try { + List minIOBuckets = minioClient.listBuckets(); + for(io.minio.messages.Bucket minIObucket : minIOBuckets) { + Bucket bucket = new BucketObject(); + bucket.setName(minIObucket.name()); + bucketsList.add(bucket); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return bucketsList; + } + + @Override + public boolean deleteBucket(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + if(!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket doesn't exist: "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + //ToDo: check bucket empty + try { + minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public AccessControlList getBucketAcl(String bucketName, long storeId) { + return null; + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(String bucketName, String policy, long storeId) { + String privatePolicy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + builder.append(" \"Statement\": [\n"); + builder.append(" {\n"); + builder.append(" \"Action\": [\n"); + builder.append(" \"s3:GetBucketLocation\",\n"); + builder.append(" \"s3:ListBucket\"\n"); + builder.append(" ],\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"\"\n"); + builder.append(" },\n"); + builder.append(" {\n"); + builder.append(" \"Action\": \"s3:GetObject\",\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"/*\"\n"); + builder.append(" }\n"); + builder.append(" ],\n"); + builder.append(" \"Version\": \"2012-10-17\"\n"); + builder.append("}\n"); + + String publicPolicy = builder.toString(); + + //ToDo Support custom policy + String policyConfig = (policy.equalsIgnoreCase("public"))? publicPolicy : privatePolicy; + + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketPolicy( + SetBucketPolicyArgs.builder().bucket(bucketName).config(policyConfig).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + return null; + } + + @Override + public void deleteBucketPolicy(String bucketName, long storeId) { + + } + + @Override + public boolean createUser(long accountId, long storeId) { + Account account = _accountDao.findById(accountId); + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String accessKey = "acs-"+account.getAccountName(); + // Check user exists + try { + UserInfo userInfo = minioAdminClient.getUserInfo(accessKey); + if(userInfo != null) { + s_logger.debug("User already exists in MinIO store: "+accessKey); + return true; + } + } catch (Exception e) { + s_logger.debug("User does not exist. Creating user: "+accessKey); + } + + KeyGenerator generator = null; + try { + generator = KeyGenerator.getInstance("HmacSHA1"); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException(e); + } + SecretKey key = generator.generateKey(); + String secretKey = Base64.encodeBase64URLSafeString(key.getEncoded()); + try { + minioAdminClient.addUser(accessKey, UserInfo.Status.ENABLED, secretKey, "", new ArrayList()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + // Store user credentials + Map details = new HashMap<>(); + details.put(MINIO_ACCESS_KEY, accessKey); + details.put(MINIO_SECRET_KEY, secretKey); + _accountDetailsDao.persist(accountId, details); + return true; + } + + @Override + public boolean setBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketEncryption(SetBucketEncryptionArgs.builder() + .bucket(bucketName) + .config(SseConfiguration.newConfigWithSseS3Rule()) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.deleteBucketEncryption(DeleteBucketEncryptionArgs.builder() + .bucket(bucketName) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean setBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.ENABLED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.SUSPENDED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public void setBucketQuota(String bucketName, long storeId, long size) { + + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + try { + minioAdminClient.setBucketQuota(bucketName, size, QuotaUnit.GB); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public Map getAllBucketsUsage(long storeId) { + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + try { + DataUsageInfo dataUsageInfo = minioAdminClient.getDataUsageInfo(); + return dataUsageInfo.bucketsSizes(); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + protected MinioClient getMinIOClient(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String url = store.getUrl(); + String accessKey = storeDetails.get(ACCESS_KEY); + String secretKey = storeDetails.get(SECRET_KEY); + MinioClient minioClient = + MinioClient.builder() + .endpoint(url) + .credentials(accessKey,secretKey) + .build(); + if(minioClient == null){ + throw new CloudRuntimeException("Error while creating MinIO client"); + } + return minioClient; + } + + protected MinioAdminClient getMinIOAdminClient(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String url = store.getUrl(); + String accessKey = storeDetails.get(ACCESS_KEY); + String secretKey = storeDetails.get(SECRET_KEY); + MinioAdminClient minioAdminClient = + MinioAdminClient.builder() + .endpoint(url) + .credentials(accessKey,secretKey) + .build(); + if(minioAdminClient == null){ + throw new CloudRuntimeException("Error while creating MinIO client"); + } + return minioAdminClient; + } +} diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java new file mode 100644 index 00000000000..fb7d1a652fc --- /dev/null +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.lifecycle; + +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.exception.CloudRuntimeException; +import io.minio.MinioClient; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +public class MinIOObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { + + private static final Logger s_logger = Logger.getLogger(MinIOObjectStoreLifeCycleImpl.class); + + @Inject + ObjectStoreHelper objectStoreHelper; + @Inject + ObjectStoreProviderManager objectStoreMgr; + + public MinIOObjectStoreLifeCycleImpl() { + } + + @SuppressWarnings("unchecked") + @Override + public DataStore initialize(Map dsInfos) { + + String url = (String)dsInfos.get("url"); + String name = (String)dsInfos.get("name"); + String providerName = (String)dsInfos.get("providerName"); + Map details = (Map)dsInfos.get("details"); + if(details == null){ + throw new CloudRuntimeException("MinIO credentials are missing"); + } + String accessKey = details.get("accesskey"); + String secretKey = details.get("secretkey"); + + + Map objectStoreParameters = new HashMap(); + objectStoreParameters.put("name", name); + objectStoreParameters.put("url", url); + + objectStoreParameters.put("providerName", providerName); + objectStoreParameters.put("accesskey", accessKey); + objectStoreParameters.put("secretkey", secretKey); + + //check credentials + MinioClient minioClient = + MinioClient.builder() + .endpoint(url) + .credentials(accessKey,secretKey) + .build(); + try { + // Test connection by listing buckets + minioClient.listBuckets(); + s_logger.debug("Successfully connected to MinIO EndPoint: "+url); + } catch (Exception e) { + s_logger.debug("Error while initializing MinIO Object Store: "+e.getMessage()); + throw new RuntimeException("Error while initializing MinIO Object Store. Invalid credentials or URL"); + } + + ObjectStoreVO objectStore = objectStoreHelper.createObjectStore(objectStoreParameters, details); + return objectStoreMgr.getObjectStore(objectStore.getId()); + } + + @Override + public boolean attachCluster(DataStore store, ClusterScope scope) { + return false; + } + + @Override + public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { + return false; + } + + @Override + public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { + return false; + } + + @Override + public boolean maintain(DataStore store) { + return false; + } + + @Override + public boolean cancelMaintain(DataStore store) { + return false; + } + + @Override + public boolean deleteDataStore(DataStore store) { + return false; + } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + +} diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImpl.java new file mode 100644 index 00000000000..abcc4996b88 --- /dev/null +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.driver.MinIOObjectStoreDriverImpl; +import org.apache.cloudstack.storage.datastore.lifecycle.MinIOObjectStoreLifeCycleImpl; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Component +public class MinIOObjectStoreProviderImpl implements ObjectStoreProvider { + + @Inject + ObjectStoreProviderManager storeMgr; + @Inject + ObjectStoreHelper helper; + + private final String providerName = "MinIO"; + protected ObjectStoreLifeCycle lifeCycle; + protected ObjectStoreDriver driver; + + @Override + public DataStoreLifeCycle getDataStoreLifeCycle() { + return lifeCycle; + } + + @Override + public String getName() { + return this.providerName; + } + + @Override + public boolean configure(Map params) { + lifeCycle = ComponentContext.inject(MinIOObjectStoreLifeCycleImpl.class); + driver = ComponentContext.inject(MinIOObjectStoreDriverImpl.class); + storeMgr.registerDriver(this.getName(), driver); + return true; + } + + @Override + public DataStoreDriver getDataStoreDriver() { + return this.driver; + } + + @Override + public HypervisorHostListener getHostListener() { + return null; + } + + @Override + public Set getTypes() { + Set types = new HashSet(); + types.add(DataStoreProviderType.OBJECT); + return types; + } +} diff --git a/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/module.properties b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/module.properties new file mode 100644 index 00000000000..828454e4634 --- /dev/null +++ b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=storage-object-minio +parent=storage diff --git a/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/spring-storage-object-minio-context.xml b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/spring-storage-object-minio-context.xml new file mode 100644 index 00000000000..30876de2e0a --- /dev/null +++ b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/spring-storage-object-minio-context.xml @@ -0,0 +1,31 @@ + + + + diff --git a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java new file mode 100644 index 00000000000..3041a5b232b --- /dev/null +++ b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.driver; + +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.AccountDetailVO; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import io.minio.BucketExistsArgs; +import io.minio.MinioClient; +import io.minio.RemoveBucketArgs; +import io.minio.admin.MinioAdminClient; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.Bucket; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class MinIOObjectStoreDriverImplTest { + + @Spy + MinIOObjectStoreDriverImpl minioObjectStoreDriverImpl = new MinIOObjectStoreDriverImpl(); + + @Mock + MinioClient minioClient; + @Mock + MinioAdminClient minioAdminClient; + @Mock + ObjectStoreDao objectStoreDao; + @Mock + ObjectStoreVO objectStoreVO; + @Mock + ObjectStoreDetailsDao objectStoreDetailsDao; + @Mock + AccountDao accountDao; + @Mock + BucketDao bucketDao; + @Mock + AccountVO account; + @Mock + AccountDetailsDao accountDetailsDao; + + Bucket bucket; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + minioObjectStoreDriverImpl._storeDao = objectStoreDao; + minioObjectStoreDriverImpl._storeDetailsDao = objectStoreDetailsDao; + minioObjectStoreDriverImpl._accountDao = accountDao; + minioObjectStoreDriverImpl._bucketDao = bucketDao; + minioObjectStoreDriverImpl._accountDetailsDao = accountDetailsDao; + bucket = new BucketVO(); + bucket.setName("test-bucket"); + when(objectStoreVO.getUrl()).thenReturn("http://localhost:9000"); + when(objectStoreDao.findById(any())).thenReturn(objectStoreVO); + } + + @Test + public void testCreateBucket() throws Exception { + doReturn(minioClient).when(minioObjectStoreDriverImpl).getMinIOClient(anyLong()); + doReturn(minioAdminClient).when(minioObjectStoreDriverImpl).getMinIOAdminClient(anyLong()); + when(bucketDao.listByObjectStoreIdAndAccountId(anyLong(), anyLong())).thenReturn(new ArrayList()); + when(account.getAccountName()).thenReturn("admin"); + when(accountDao.findById(anyLong())).thenReturn(account); + when(accountDetailsDao.findDetail(anyLong(),anyString())). + thenReturn(new AccountDetailVO(1L, "abc","def")); + when(bucketDao.findById(anyLong())).thenReturn(new BucketVO()); + Bucket bucketRet = minioObjectStoreDriverImpl.createBucket(bucket, false); + assertEquals(bucketRet.getName(), bucket.getName()); + verify(minioClient, times(1)).bucketExists(any()); + verify(minioClient, times(1)).makeBucket(any()); + } + + @Test + public void testDeleteBucket() throws Exception { + String bucketName = "test-bucket"; + doReturn(minioClient).when(minioObjectStoreDriverImpl).getMinIOClient(anyLong()); + when(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())).thenReturn(true); + doNothing().when(minioClient).removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); + boolean success = minioObjectStoreDriverImpl.deleteBucket(bucketName, 1L); + assertTrue(success); + verify(minioClient, times(1)).bucketExists(any()); + verify(minioClient, times(1)).removeBucket(any()); + } +} diff --git a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImplTest.java b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImplTest.java new file mode 100644 index 00000000000..8651e00b6ad --- /dev/null +++ b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImplTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.provider; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider.DataStoreProviderType; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class MinIOObjectStoreProviderImplTest { + + private MinIOObjectStoreProviderImpl minioObjectStoreProviderImpl; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + minioObjectStoreProviderImpl = new MinIOObjectStoreProviderImpl(); + } + + @Test + public void testGetName() { + String name = minioObjectStoreProviderImpl.getName(); + assertEquals("MinIO", name); + } + + @Test + public void testGetTypes() { + Set types = minioObjectStoreProviderImpl.getTypes(); + assertEquals(1, types.size()); + assertEquals("OBJECT", types.toArray()[0].toString()); + } +} diff --git a/plugins/storage/object/simulator/pom.xml b/plugins/storage/object/simulator/pom.xml new file mode 100644 index 00000000000..36ba0755fe9 --- /dev/null +++ b/plugins/storage/object/simulator/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + cloud-plugin-storage-object-simulator + Apache CloudStack Plugin - Simulator object storage provider + + org.apache.cloudstack + cloudstack-plugins + 4.19.0.0-SNAPSHOT + ../../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + io.minio + minio + 8.5.2 + + + io.minio + minio-admin + 8.5.2 + + + diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java new file mode 100644 index 00000000000..5f25a6061a7 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SimulatorObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = Logger.getLogger(SimulatorObjectStoreDriverImpl.class); + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(ACCESS_KEY); + bucketVO.setSecretKey(SECRET_KEY); + bucketVO.setBucketURL(store.getUrl()+"/"+bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucket; + } + + @Override + public List listBuckets(long storeId) { + List bucketsList = new ArrayList<>(); + return bucketsList; + } + + @Override + public boolean deleteBucket(String bucketName, long storeId) { + return true; + } + + @Override + public AccessControlList getBucketAcl(String bucketName, long storeId) { + return null; + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(String bucketName, String policy, long storeId) { + + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + return null; + } + + @Override + public void deleteBucketPolicy(String bucketName, long storeId) { + + } + + @Override + public boolean createUser(long accountId, long storeId) { + return true; + } + + @Override + public boolean setBucketEncryption(String bucketName, long storeId) { + return true; + } + + @Override + public boolean deleteBucketEncryption(String bucketName, long storeId) { + return true; + } + + @Override + public boolean setBucketVersioning(String bucketName, long storeId) { + return true; + } + + @Override + public boolean deleteBucketVersioning(String bucketName, long storeId) { + return true; + } + + @Override + public void setBucketQuota(String bucketName, long storeId, long size) { + + } + + @Override + public Map getAllBucketsUsage(long storeId) { + return new HashMap(); + } +} diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java new file mode 100644 index 00000000000..34e928ced30 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.lifecycle; + +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.resource.Discoverer; +import com.cloud.resource.ResourceManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SimulatorObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { + + private static final Logger s_logger = Logger.getLogger(SimulatorObjectStoreLifeCycleImpl.class); + @Inject + protected ResourceManager _resourceMgr; + @Inject + protected ObjectStoreDao objectStoreDao; + @Inject + ObjectStoreHelper objectStoreHelper; + @Inject + ObjectStoreProviderManager objectStoreMgr; + + protected List _discoverers; + + public List getDiscoverers() { + return _discoverers; + } + + public void setDiscoverers(List discoverers) { + this._discoverers = discoverers; + } + + public SimulatorObjectStoreLifeCycleImpl() { + } + + @SuppressWarnings("unchecked") + @Override + public DataStore initialize(Map dsInfos) { + + String url = (String)dsInfos.get("url"); + String name = (String)dsInfos.get("name"); + String providerName = (String)dsInfos.get("providerName"); + Map details = (Map)dsInfos.get("details"); + + Map objectStoreParameters = new HashMap(); + objectStoreParameters.put("name", name); + objectStoreParameters.put("url", url); + objectStoreParameters.put("providerName", providerName); + + ObjectStoreVO ids = objectStoreHelper.createObjectStore(objectStoreParameters, details); + return objectStoreMgr.getObjectStore(ids.getId()); + } + + @Override + public boolean attachCluster(DataStore store, ClusterScope scope) { + return false; + } + + @Override + public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { + return false; + } + + @Override + public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { + return false; + } + + @Override + public boolean maintain(DataStore store) { + return false; + } + + @Override + public boolean cancelMaintain(DataStore store) { + return false; + } + + @Override + public boolean deleteDataStore(DataStore store) { + return false; + } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + +} diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImpl.java new file mode 100644 index 00000000000..651fac4d1f1 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.driver.SimulatorObjectStoreDriverImpl; +import org.apache.cloudstack.storage.datastore.lifecycle.SimulatorObjectStoreLifeCycleImpl; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Component +public class SimulatorObjectStoreProviderImpl implements ObjectStoreProvider { + + @Inject + ObjectStoreProviderManager storeMgr; + @Inject + ObjectStoreHelper helper; + + private final String providerName = "Simulator"; + protected ObjectStoreLifeCycle lifeCycle; + protected ObjectStoreDriver driver; + + @Override + public DataStoreLifeCycle getDataStoreLifeCycle() { + return lifeCycle; + } + + @Override + public String getName() { + return this.providerName; + } + + @Override + public boolean configure(Map params) { + lifeCycle = ComponentContext.inject(SimulatorObjectStoreLifeCycleImpl.class); + driver = ComponentContext.inject(SimulatorObjectStoreDriverImpl.class); + storeMgr.registerDriver(this.getName(), driver); + return true; + } + + @Override + public DataStoreDriver getDataStoreDriver() { + return this.driver; + } + + @Override + public HypervisorHostListener getHostListener() { + return null; + } + + @Override + public Set getTypes() { + Set types = new HashSet(); + types.add(DataStoreProviderType.OBJECT); + return types; + } +} diff --git a/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/module.properties b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/module.properties new file mode 100644 index 00000000000..552f08c7a5c --- /dev/null +++ b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=storage-object-simulator +parent=storage diff --git a/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/spring-storage-object-simulator-context.xml b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/spring-storage-object-simulator-context.xml new file mode 100644 index 00000000000..863b881d7f1 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/spring-storage-object-simulator-context.xml @@ -0,0 +1,31 @@ + + + + diff --git a/plugins/storage/object/simulator/src/test/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImplTest.java b/plugins/storage/object/simulator/src/test/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImplTest.java new file mode 100644 index 00000000000..57c7eee5cfb --- /dev/null +++ b/plugins/storage/object/simulator/src/test/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImplTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.provider; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class SimulatorObjectStoreProviderImplTest { + + private SimulatorObjectStoreProviderImpl simulatorObjectStoreProviderImpl; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + simulatorObjectStoreProviderImpl = new SimulatorObjectStoreProviderImpl(); + } + + @Test + public void testGetName() { + String name = simulatorObjectStoreProviderImpl.getName(); + assertEquals("Simulator", name); + } + + @Test + public void testGetTypes() { + Set types = simulatorObjectStoreProviderImpl.getTypes(); + assertEquals(1, types.size()); + assertEquals("OBJECT", types.toArray()[0].toString()); + } +} diff --git a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java index d6a67b447c5..0798f9f2cd2 100644 --- a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java @@ -107,6 +107,7 @@ public class ElastistorPrimaryDataStoreLifeCycle implements PrimaryDataStoreLife Long capacityBytes = (Long) dsInfos.get("capacityBytes"); Long capacityIops = (Long) dsInfos.get("capacityIops"); String tags = (String) dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); boolean managed = (Boolean) dsInfos.get("managed"); Map details = (Map) dsInfos.get("details"); String domainName = details.get("domainname"); @@ -196,6 +197,7 @@ public class ElastistorPrimaryDataStoreLifeCycle implements PrimaryDataStoreLife parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(HypervisorType.Any); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); parameters.setClusterId(clusterId); diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java index 0906e645b66..6fd42009125 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java @@ -100,6 +100,7 @@ public class DateraPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCycl Long capacityBytes = (Long) dsInfos.get("capacityBytes"); Long capacityIops = (Long) dsInfos.get("capacityIops"); String tags = (String) dsInfos.get("tags"); + boolean isTagARule = (Boolean)dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map) dsInfos.get("details"); String domainName = details.get("domainname"); @@ -181,6 +182,7 @@ public class DateraPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCycl parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(HypervisorType.Any); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); String managementVip = DateraUtil.getManagementVip(url); diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 213e5620553..685565d73b0 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -27,6 +27,7 @@ import com.cloud.agent.api.ValidateVcenterDetailsCommand; import com.cloud.alert.AlertManager; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.StorageConflictException; +import com.cloud.exception.StorageUnavailableException; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; @@ -44,7 +45,6 @@ import com.cloud.storage.dao.StoragePoolWorkDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.UriUtils; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; @@ -67,10 +67,6 @@ import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; import org.apache.log4j.Logger; import javax.inject.Inject; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLDecoder; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -138,64 +134,27 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore PrimaryDataStoreParameters parameters = new PrimaryDataStoreParameters(); - UriUtils.UriInfo uriInfo = UriUtils.getUriInfo(url); - - String scheme = uriInfo.getScheme(); - String storageHost = uriInfo.getStorageHost(); - String storagePath = uriInfo.getStoragePath(); - try { - if (scheme == null) { - throw new InvalidParameterValueException("scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); - } else if (scheme.equalsIgnoreCase("nfs")) { - if (storageHost == null || storagePath == null || storageHost.trim().isEmpty() || storagePath.trim().isEmpty()) { - throw new InvalidParameterValueException("host or path is null, should be nfs://hostname/path"); - } - } else if (scheme.equalsIgnoreCase("cifs")) { - // Don't validate against a URI encoded URI. - URI cifsUri = new URI(url); - String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri); - if (warnMsg != null) { - throw new InvalidParameterValueException(warnMsg); - } - } else if (scheme.equalsIgnoreCase("sharedMountPoint")) { - if (storagePath == null) { - throw new InvalidParameterValueException("host or path is null, should be sharedmountpoint://localhost/path"); - } - } else if (scheme.equalsIgnoreCase("rbd")) { - if (storagePath == null) { - throw new InvalidParameterValueException("host or path is null, should be rbd://hostname/pool"); - } - } else if (scheme.equalsIgnoreCase("gluster")) { - if (storageHost == null || storagePath == null || storageHost.trim().isEmpty() || storagePath.trim().isEmpty()) { - throw new InvalidParameterValueException("host or path is null, should be gluster://hostname/volume"); - } - } - } catch (URISyntaxException e) { - throw new InvalidParameterValueException(url + " is not a valid uri"); - } - String tags = (String)dsInfos.get("tags"); Map details = (Map)dsInfos.get("details"); parameters.setTags(tags); + parameters.setIsTagARule((Boolean)dsInfos.get("isTagARule")); parameters.setDetails(details); - String hostPath = null; - try { - hostPath = URLDecoder.decode(storagePath, "UTF-8"); - } catch (UnsupportedEncodingException e) { - s_logger.error("[ignored] we are on a platform not supporting \"UTF-8\"!?!", e); - } - if (hostPath == null) { // if decoding fails, use getPath() anyway - hostPath = storagePath; - } + String scheme = dsInfos.get("scheme").toString(); + String storageHost = dsInfos.get("host").toString(); + String hostPath = dsInfos.get("hostPath").toString(); + String uri = String.format("%s://%s%s", scheme, storageHost, hostPath); + Object localStorage = dsInfos.get("localStorage"); - if (localStorage != null) { - hostPath = hostPath.replaceFirst("/", ""); + if (localStorage != null) { + hostPath = hostPath.contains("//") ? hostPath.replaceFirst("/", "") : hostPath; hostPath = hostPath.replace("+", " "); } - String userInfo = uriInfo.getUserInfo(); - int port = uriInfo.getPort(); + + String userInfo = dsInfos.get("userInfo") != null ? dsInfos.get("userInfo").toString() : null; + int port = dsInfos.get("port") != null ? Integer.parseInt(dsInfos.get("port").toString()) : -1; + if (s_logger.isDebugEnabled()) { s_logger.debug("createPool Params @ scheme - " + scheme + " storageHost - " + storageHost + " hostPath - " + hostPath + " port - " + port); } @@ -312,8 +271,8 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore parameters.setPort(0); parameters.setPath(hostPath); } else { - s_logger.warn("Unable to figure out the scheme for URI: " + uriInfo); - throw new IllegalArgumentException("Unable to figure out the scheme for URI: " + uriInfo); + s_logger.warn("Unable to figure out the scheme for URI: " + scheme); + throw new IllegalArgumentException("Unable to figure out the scheme for URI: " + scheme); } } @@ -321,7 +280,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore List pools = primaryDataStoreDao.listPoolByHostPath(storageHost, hostPath); if (!pools.isEmpty() && !scheme.equalsIgnoreCase("sharedmountpoint")) { Long oldPodId = pools.get(0).getPodId(); - throw new CloudRuntimeException("Storage pool " + uriInfo + " already in use by another pod (id=" + oldPodId + ")"); + throw new CloudRuntimeException("Storage pool " + hostPath + " already in use by another pod (id=" + oldPodId + ")"); } } @@ -550,7 +509,16 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore @Override public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { - dataStoreHelper.attachHost(store, scope, existingInfo); + DataStore dataStore = dataStoreHelper.attachHost(store, scope, existingInfo); + if(existingInfo.getCapacityBytes() == 0){ + try { + storageMgr.connectHostToSharedPool(scope.getScopeId(), dataStore.getId()); + } catch (StorageUnavailableException ex) { + s_logger.error("Storage unavailable ",ex); + } catch (StorageConflictException ex) { + s_logger.error("Storage already exists ",ex); + } + } return true; } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index c0f3cb4b459..9b493ff01b9 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -527,6 +527,16 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + private ResourceDefinitionCreate createResourceDefinitionCreate(String rscName, String rscGrpName) + throws ApiException { + ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate(); + ResourceDefinition rd = new ResourceDefinition(); + rd.setName(rscName); + rd.setResourceGroupName(rscGrpName); + rdCreate.setResourceDefinition(rd); + return rdCreate; + } + private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) { final String rscGrp = getRscGrp(storagePoolVO); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); @@ -539,11 +549,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver try { s_logger.debug("Create new resource definition: " + rscName); - ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate(); - ResourceDefinition rd = new ResourceDefinition(); - rd.setName(rscName); - rd.setResourceGroupName(rscGrp); - rdCreate.setResourceDefinition(rd); + ResourceDefinitionCreate rdCreate = createResourceDefinitionCreate(rscName, rscGrp); ApiCallRcList answers = linstorApi.resourceDefinitionCreate(rdCreate); checkLinstorAnswersThrow(answers); @@ -712,6 +718,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver VirtualMachineManager.ExecuteInSequence.value()); Optional optEP = getDiskfullEP(linstorApi, rscName); + if (optEP.isEmpty()) { + optEP = getLinstorEP(linstorApi, rscName); + } + if (optEP.isPresent()) { Answer answer = optEP.get().sendMessage(cmd); if (!answer.getResult()) { @@ -840,6 +850,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver callback.complete(res); } + /** + * Tries to get a Linstor cloudstack end point, that is at least diskless. + * + * @param api Linstor java api object + * @param rscName resource name to make available on node + * @return Optional RemoteHostEndPoint if one could get found. + * @throws ApiException + */ private Optional getLinstorEP(DevelopersApi api, String rscName) throws ApiException { List linstorNodeNames = LinstorUtil.getLinstorNodeNames(api); Collections.shuffle(linstorNodeNames); // do not always pick the first linstor node @@ -892,6 +910,25 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return Optional.empty(); } + private String restoreResourceFromSnapshot( + DevelopersApi api, + StoragePoolVO storagePoolVO, + String rscName, + String snapshotName, + String restoredName) throws ApiException { + final String rscGrp = getRscGrp(storagePoolVO); + ResourceDefinitionCreate rdc = createResourceDefinitionCreate(restoredName, rscGrp); + api.resourceDefinitionCreate(rdc); + + SnapshotRestore sr = new SnapshotRestore(); + sr.toResource(restoredName); + api.resourceSnapshotsRestoreVolumeDefinition(rscName, snapshotName, sr); + + api.resourceSnapshotRestore(rscName, snapshotName, sr); + + return getDeviceName(api, restoredName); + } + private Answer copyTemplate(DataObject srcData, DataObject dstData) { TemplateInfo tInfo = (TemplateInfo) dstData; final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); @@ -929,6 +966,39 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return answer; } + /** + * Create a temporary resource from the snapshot to backup, so we can copy the data on a diskless agent + * @param api Linstor Developer api object + * @param pool StoragePool this resource resides on + * @param rscName rscName of the snapshotted resource + * @param snapshotInfo snapshot info of the snapshot + * @param origCmd original LinstorBackupSnapshotCommand that needs to have a patched path + * @return answer from agent operation + * @throws ApiException if any Linstor api operation fails + */ + private Answer copyFromTemporaryResource( + DevelopersApi api, StoragePoolVO pool, String rscName, SnapshotInfo snapshotInfo, CopyCommand origCmd) + throws ApiException { + Answer answer; + String restoreName = rscName + "-rst"; + String snapshotName = LinstorUtil.RSC_PREFIX + snapshotInfo.getUuid(); + String devName = restoreResourceFromSnapshot(api, pool, rscName, snapshotName, restoreName); + + Optional optEPAny = getLinstorEP(api, restoreName); + if (optEPAny.isPresent()) { + // patch the src device path to the temporary linstor resource + SnapshotObjectTO soTO = (SnapshotObjectTO)snapshotInfo.getTO(); + soTO.setPath(devName); + origCmd.setSrcTO(soTO); + answer = optEPAny.get().sendMessage(origCmd); + } else{ + answer = new Answer(origCmd, false, "Unable to get matching Linstor endpoint."); + } + // delete the temporary resource, noop if already gone + api.resourceDefinitionDelete(restoreName); + return answer; + } + protected Answer copySnapshot(DataObject srcData, DataObject destData) { String value = _configDao.getValue(Config.BackupSnapshotWait.toString()); int _backupsnapshotwait = NumbersUtil.parseInt( @@ -956,13 +1026,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver VirtualMachineManager.ExecuteInSequence.value()); cmd.setOptions(options); - Optional optEP = getDiskfullEP( - api, LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid()); + String rscName = LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid(); + Optional optEP = getDiskfullEP(api, rscName); Answer answer; if (optEP.isPresent()) { answer = optEP.get().sendMessage(cmd); } else { - answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + s_logger.debug("No diskfull endpoint found to copy image, creating diskless endpoint"); + answer = copyFromTemporaryResource(api, pool, rscName, snapshotInfo, cmd); } return answer; } catch (Exception e) { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java index a7d0c7fc0a3..efc69438e75 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java @@ -91,6 +91,7 @@ public class LinstorPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLif String providerName = (String) dsInfos.get("providerName"); Long capacityIops = (Long) dsInfos.get("capacityIops"); String tags = (String) dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map) dsInfos.get("details"); @@ -168,6 +169,7 @@ public class LinstorPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLif parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(HypervisorType.KVM); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); parameters.setUserInfo(resourceGroup); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java index 16cc24a78d4..90ebf30f7cd 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java @@ -21,8 +21,8 @@ import org.apache.cloudstack.framework.config.Configurable; public class LinstorConfigurationManager implements Configurable { - public static final ConfigKey BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "false", - "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot), only works on hyperconverged setups.", true, ConfigKey.Scope.Global, null); + public static final ConfigKey BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "true", + "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot)", true, ConfigKey.Scope.Global, null); public static final ConfigKey[] CONFIG_KEYS = new ConfigKey[] { BackupSnapshots }; diff --git a/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java index 32735664a25..507189edc14 100644 --- a/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java @@ -69,6 +69,7 @@ public class NexentaPrimaryDataStoreLifeCycle Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get("capacityIops"); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); Map details = (Map) dsInfos.get("details"); NexentaUtil.NexentaPluginParameters params = NexentaUtil.parseNexentaPluginUrl(url); DataCenterVO zone = zoneDao.findById(zoneId); @@ -98,6 +99,7 @@ public class NexentaPrimaryDataStoreLifeCycle parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(Hypervisor.HypervisorType.Any); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); details.put(NexentaUtil.NMS_URL, params.getNmsUrl().toString()); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java index 65831e4ec3d..17150699923 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java @@ -139,6 +139,7 @@ public class ScaleIOPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCyc Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get("capacityIops"); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); Map details = (Map) dsInfos.get("details"); if (zoneId == null) { @@ -224,6 +225,7 @@ public class ScaleIOPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCyc parameters.setHypervisorType(Hypervisor.HypervisorType.KVM); parameters.setUuid(UUID.randomUUID().toString()); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); StoragePoolStatistics poolStatistics = scaleIOPool.getStatistics(); if (poolStatistics != null) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java index 38c8c240094..7a2767c32e6 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java @@ -91,6 +91,7 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get("capacityIops"); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map)dsInfos.get("details"); @@ -142,6 +143,7 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC } parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); String managementVip = SolidFireUtil.getManagementVip(url); diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java index 9cc746d4ee8..557cc3f60f6 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java @@ -104,6 +104,7 @@ public class SolidFireSharedPrimaryDataStoreLifeCycle implements PrimaryDataStor Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get(CAPACITY_IOPS); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map)dsInfos.get("details"); @@ -152,6 +153,7 @@ public class SolidFireSharedPrimaryDataStoreLifeCycle implements PrimaryDataStor parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(hypervisorType); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); String managementVip = SolidFireUtil.getManagementVip(url); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index df6ea74881e..862e8acc9e0 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -182,7 +183,6 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (hasSvcOfferingTag && hasTemplateTag) { hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); if (s_logger.isDebugEnabled()) { s_logger.debug("Found " + hostsMatchingOfferingTag.size() + " Hosts satisfying both tags, host ids are:" + hostsMatchingOfferingTag); } @@ -202,6 +202,13 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { clusterHosts.retainAll(hostsMatchingUefiTag); } + clusterHosts.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); + + + if (clusterHosts.isEmpty()) { + s_logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTagOnOffering)); + throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile)); + } // add all hosts that we are not considering to the avoid list List allhostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); allhostsInCluster.removeAll(clusterHosts); @@ -267,6 +274,8 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { } } + hostsCopy.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); + if (!hostsCopy.isEmpty()) { suitableHosts = allocateTo(plan, offering, template, avoid, hostsCopy, returnUpTo, considerReservedCapacity, account); } diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 026c8f03407..a3532a79af4 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -294,6 +294,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeStats; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; @@ -349,6 +350,10 @@ import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; + public class ApiDBUtils { private static ManagementServer s_ms; static AsyncJobManager s_asyncMgr; @@ -481,6 +486,9 @@ public class ApiDBUtils { static NicDao s_nicDao; static ResourceManagerUtil s_resourceManagerUtil; static SnapshotPolicyDetailsDao s_snapshotPolicyDetailsDao; + static ObjectStoreDao s_objectStoreDao; + + static BucketDao s_bucketDao; @Inject private ManagementServer ms; @@ -740,6 +748,11 @@ public class ApiDBUtils { @Inject SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; + @Inject + private ObjectStoreDao objectStoreDao; + @Inject + private BucketDao bucketDao; + @PostConstruct void init() { s_ms = ms; @@ -871,6 +884,8 @@ public class ApiDBUtils { s_backupOfferingDao = backupOfferingDao; s_resourceIconDao = resourceIconDao; s_resourceManagerUtil = resourceManagerUtil; + s_objectStoreDao = objectStoreDao; + s_bucketDao = bucketDao; } // /////////////////////////////////////////////////////////// @@ -2208,4 +2223,12 @@ public class ApiDBUtils { public static NicSecondaryIpVO findSecondaryIpByIp4AddressAndNetworkId(String ip4Address, long networkId) { return s_nicSecondaryIpDao.findByIp4AddressAndNetworkId(ip4Address, networkId); } + + public static ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store) { + return s_objectStoreDao.newObjectStoreResponse(store); + } + + public static ObjectStoreResponse fillObjectStoreDetails(ObjectStoreResponse storeData, ObjectStoreVO store) { + return s_objectStoreDao.setObjectStoreResponse(storeData, store); + } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 98a4ab17f7c..3791de5995f 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -41,6 +41,8 @@ import javax.inject.Inject; import com.cloud.dc.VlanDetailsVO; import com.cloud.dc.dao.VlanDetailsDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.BucketVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -66,6 +68,7 @@ import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.CapabilityResponse; import org.apache.cloudstack.api.response.CapacityResponse; import org.apache.cloudstack.api.response.ClusterResponse; @@ -103,6 +106,7 @@ import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; import org.apache.cloudstack.api.response.InternalLoadBalancerElementResponse; import org.apache.cloudstack.api.response.IpForwardingRuleResponse; +import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.IpRangeResponse; import org.apache.cloudstack.api.response.Ipv6RouteResponse; import org.apache.cloudstack.api.response.IsolationMethodResponse; @@ -121,6 +125,7 @@ import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.NicExtraDhcpOptionResponse; import org.apache.cloudstack.api.response.NicResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.OvsProviderResponse; import org.apache.cloudstack.api.response.PhysicalNetworkResponse; import org.apache.cloudstack.api.response.PodResponse; @@ -142,6 +147,7 @@ import org.apache.cloudstack.api.response.RollingMaintenanceHostUpdatedResponse; import org.apache.cloudstack.api.response.RollingMaintenanceResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SSHKeyPairResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.SecurityGroupRuleResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; @@ -198,10 +204,15 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.usage.UsageService; import org.apache.cloudstack.usage.UsageTypes; @@ -265,7 +276,6 @@ import com.cloud.gpu.GPU; import com.cloud.host.ControlState; import com.cloud.host.Host; import com.cloud.host.HostVO; -import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorCapabilities; import com.cloud.network.GuestVlan; import com.cloud.network.GuestVlanRange; @@ -285,6 +295,7 @@ import com.cloud.network.OvsProvider; import com.cloud.network.PhysicalNetwork; import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.PhysicalNetworkTrafficType; +import com.cloud.network.PublicIpQuarantine; import com.cloud.network.RemoteAccessVpn; import com.cloud.network.RouterHealthCheckResult; import com.cloud.network.Site2SiteCustomerGateway; @@ -473,6 +484,9 @@ public class ApiResponseHelper implements ResponseGenerator { @Inject VlanDetailsDao vlanDetailsDao; + @Inject + ObjectStoreDao _objectStoreDao; + @Override public UserResponse createUserResponse(User user) { UserAccountJoinVO vUser = ApiDBUtils.newUserView(user); @@ -4317,6 +4331,10 @@ public class ApiResponseHelper implements ResponseGenerator { } usageRecResponse.setDescription(builder.toString()); } + } else if (usageRecord.getUsageType() == UsageTypes.BUCKET) { + BucketVO bucket = _entityMgr.findByIdIncludingRemoved(BucketVO.class, usageRecord.getUsageId().toString()); + usageRecResponse.setUsageId(bucket.getUuid()); + usageRecResponse.setResourceName(bucket.getName()); } if(resourceTagResponseMap != null && resourceTagResponseMap.get(resourceId + ":" + resourceType) != null) { usageRecResponse.setTags(resourceTagResponseMap.get(resourceId + ":" + resourceType)); @@ -5098,4 +5116,69 @@ public class ApiResponseHelper implements ResponseGenerator { response.setObjectName("firewallrule"); return response; } + + @Override + public SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic) { + String zoneUuid = ApiDBUtils.findZoneById(heuristic.getZoneId()).getUuid(); + SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = new SecondaryStorageHeuristicsResponse(heuristic.getUuid(), heuristic.getName(), + heuristic.getDescription(), zoneUuid, heuristic.getType(), heuristic.getHeuristicRule(), heuristic.getCreated(), heuristic.getRemoved()); + secondaryStorageHeuristicsResponse.setResponseName("secondarystorageheuristics"); + + return secondaryStorageHeuristicsResponse; + } + + @Override + public IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine quarantinedIp) { + IpQuarantineResponse quarantinedIpsResponse = new IpQuarantineResponse(); + String ipAddress = userIpAddressDao.findById(quarantinedIp.getPublicIpAddressId()).getAddress().toString(); + Account previousOwner = _accountMgr.getAccount(quarantinedIp.getPreviousOwnerId()); + + quarantinedIpsResponse.setId(quarantinedIp.getUuid()); + quarantinedIpsResponse.setPublicIpAddress(ipAddress); + quarantinedIpsResponse.setPreviousOwnerId(previousOwner.getUuid()); + quarantinedIpsResponse.setPreviousOwnerName(previousOwner.getName()); + quarantinedIpsResponse.setCreated(quarantinedIp.getCreated()); + quarantinedIpsResponse.setRemoved(quarantinedIp.getRemoved()); + quarantinedIpsResponse.setEndDate(quarantinedIp.getEndDate()); + quarantinedIpsResponse.setRemovalReason(quarantinedIp.getRemovalReason()); + quarantinedIpsResponse.setResponseName("quarantinedip"); + + return quarantinedIpsResponse; + } + + public ObjectStoreResponse createObjectStoreResponse(ObjectStore os) { + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + objectStoreResponse.setId(os.getUuid()); + objectStoreResponse.setName(os.getName()); + objectStoreResponse.setProviderName(os.getProviderName()); + objectStoreResponse.setObjectName("objectstore"); + return objectStoreResponse; + } + + @Override + public BucketResponse createBucketResponse(Bucket bucket) { + BucketResponse bucketResponse = new BucketResponse(); + bucketResponse.setName(bucket.getName()); + bucketResponse.setId(bucket.getUuid()); + bucketResponse.setCreated(bucket.getCreated()); + bucketResponse.setState(bucket.getState()); + bucketResponse.setSize(bucket.getSize()); + if(bucket.getQuota() != null) { + bucketResponse.setQuota(bucket.getQuota()); + } + bucketResponse.setVersioning(bucket.isVersioning()); + bucketResponse.setEncryption(bucket.isEncryption()); + bucketResponse.setObjectLock(bucket.isObjectLock()); + bucketResponse.setPolicy(bucket.getPolicy()); + bucketResponse.setBucketURL(bucket.getBucketURL()); + bucketResponse.setAccessKey(bucket.getAccessKey()); + bucketResponse.setSecretKey(bucket.getSecretKey()); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); + bucketResponse.setObjectStoragePoolId(objectStoreVO.getUuid()); + bucketResponse.setObjectStoragePool(objectStoreVO.getName()); + bucketResponse.setObjectName("bucket"); + bucketResponse.setProvider(objectStoreVO.getProviderName()); + populateAccount(bucketResponse, bucket.getAccountId()); + return bucketResponse; + } } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index c99aa293234..6b0144045a2 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -43,6 +43,9 @@ import com.cloud.network.as.dao.AutoScaleVmGroupDao; import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PublicIpQuarantineDao; +import com.cloud.network.PublicIpQuarantine; +import com.cloud.network.vo.PublicIpQuarantineVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.SSHKeyPairVO; import com.cloud.user.dao.SSHKeyPairDao; @@ -80,14 +83,17 @@ import org.apache.cloudstack.api.command.admin.router.GetRouterHealthCheckResult import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; import org.apache.cloudstack.api.command.admin.snapshot.ListSnapshotsCmdByAdmin; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; import org.apache.cloudstack.api.command.admin.template.ListTemplatesCmdByAdmin; import org.apache.cloudstack.api.command.admin.user.ListUsersCmd; import org.apache.cloudstack.api.command.admin.zone.ListZonesCmdByAdmin; import org.apache.cloudstack.api.command.user.account.ListAccountsCmd; import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; +import org.apache.cloudstack.api.command.user.address.ListQuarantinedIpsCmd; import org.apache.cloudstack.api.command.user.affinitygroup.ListAffinityGroupsCmd; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; import org.apache.cloudstack.api.command.user.iso.ListIsosCmd; @@ -119,8 +125,10 @@ import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; +import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -128,6 +136,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; @@ -151,6 +160,11 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; @@ -237,6 +251,7 @@ import com.cloud.network.router.VirtualNetworkApplianceManager; import com.cloud.network.security.SecurityGroupVMMapVO; import com.cloud.network.security.dao.SecurityGroupVMMapDao; import com.cloud.offering.DiskOffering; +import com.cloud.offering.ServiceOffering; import com.cloud.org.Grouping; import com.cloud.projects.Project; import com.cloud.projects.Project.ListProjectResourcesCriteria; @@ -255,6 +270,7 @@ import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; @@ -269,6 +285,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; @@ -304,6 +321,8 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; +import org.apache.cloudstack.api.response.BucketResponse; import static com.cloud.vm.VmDetailConstants.SSH_PUBLIC_KEY; @@ -495,6 +514,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private VirtualMachineManager virtualMachineManager; + @Inject private VolumeDao volumeDao; @@ -504,6 +524,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ManagementServerHostDao msHostDao; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + @Inject private NetworkDao networkDao; @@ -537,9 +560,18 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private SnapshotJoinDao snapshotJoinDao; + @Inject + private ObjectStoreDao objectStoreDao; + + @Inject + private BucketDao bucketDao; + @Inject EntityManager entityManager; + @Inject + private PublicIpQuarantineDao publicIpQuarantineDao; + private SearchCriteria getMinimumCpuServiceOfferingJoinSearchCriteria(int cpu) { SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); SearchCriteria sc1 = _srvOfferingJoinDao.createSearchCriteria(); @@ -2784,10 +2816,29 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForStoragePools(ListStoragePoolsCmd cmd) { - Pair, Integer> result = searchForStoragePoolsInternal(cmd); - ListResponse response = new ListResponse(); + Pair, Integer> result = cmd.getHostId() != null ? searchForLocalStorages(cmd) : searchForStoragePoolsInternal(cmd); + return createStoragesPoolResponse(result); + } - List poolResponses = ViewResponseHelper.createStoragePoolResponse(result.first().toArray(new StoragePoolJoinVO[result.first().size()])); + private Pair, Integer> searchForLocalStorages(ListStoragePoolsCmd cmd) { + long id = cmd.getHostId(); + String scope = ScopeType.HOST.toString(); + Pair, Integer> localStorages; + + ListHostsCmd listHostsCmd = new ListHostsCmd(); + listHostsCmd.setId(id); + Pair, Integer> hosts = searchForServersInternal(listHostsCmd); + + cmd.setScope(scope); + localStorages = searchForStoragePoolsInternal(cmd); + + return localStorages; + } + + private ListResponse createStoragesPoolResponse(Pair, Integer> storagePools) { + ListResponse response = new ListResponse<>(); + + List poolResponses = ViewResponseHelper.createStoragePoolResponse(storagePools.first().toArray(new StoragePoolJoinVO[storagePools.first().size()])); for (StoragePoolResponse poolResponse : poolResponses) { DataStore store = dataStoreManager.getPrimaryDataStore(poolResponse.getId()); if (store != null) { @@ -2807,7 +2858,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } - response.setResponses(poolResponses, result.second()); + response.setResponses(poolResponses, storagePools.second()); return response; } @@ -3125,11 +3176,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Object keyword = cmd.getKeyword(); Long domainId = cmd.getDomainId(); Boolean isRootAdmin = accountMgr.isRootAdmin(account.getAccountId()); + Long projectId = cmd.getProjectId(); + String accountName = cmd.getAccountName(); Boolean isRecursive = cmd.isRecursive(); Long zoneId = cmd.getZoneId(); Long volumeId = cmd.getVolumeId(); Long storagePoolId = cmd.getStoragePoolId(); Boolean encrypt = cmd.getEncrypt(); + String storageType = cmd.getStorageType(); + // Keeping this logic consistent with domain specific zones // if a domainId is provided, we just return the disk offering // associated with this domain @@ -3181,6 +3236,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.addAnd("encrypt", SearchCriteria.Op.EQ, encrypt); } + useStorageType(sc, storageType); + if (zoneId != null) { SearchBuilder sb = _diskOfferingJoinDao.createSearchBuilder(); sb.and("zoneId", sb.entity().getZoneId(), Op.FIND_IN_SET); @@ -3214,9 +3271,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // Filter offerings that are not associated with caller's domain // Fetch the offering ids from the details table since theres no smart way to filter them in the join ... yet! - Account caller = CallContext.current().getCallingAccount(); - if (caller.getType() != Account.Type.ADMIN) { - Domain callerDomain = _domainDao.findById(caller.getDomainId()); + account = accountMgr.finalizeOwner(account, accountName, domainId, projectId); + if (!Account.Type.ADMIN.equals(account.getType())) { + Domain callerDomain = _domainDao.findById(account.getDomainId()); List domainIds = findRelatedDomainIds(callerDomain, isRecursive); List ids = _diskOfferingDetailsDao.findOfferingIdsByDomainIds(domainIds); @@ -3260,6 +3317,17 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return new Pair<>(result.first(), result.second()); } + private void useStorageType(SearchCriteria sc, String storageType) { + if (storageType != null) { + if (storageType.equalsIgnoreCase(ServiceOffering.StorageType.local.toString())) { + sc.addAnd("useLocalStorage", Op.EQ, true); + + } else if (storageType.equalsIgnoreCase(ServiceOffering.StorageType.shared.toString())) { + sc.addAnd("useLocalStorage", Op.EQ, false); + } + } + } + private List findRelatedDomainIds(Domain domain, boolean isRecursive) { List domainIds = _domainDao.getDomainParentIds(domain.getId()) .stream().collect(Collectors.toList()); @@ -3295,6 +3363,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q searchFilter.addOrderBy(ServiceOfferingJoinVO.class, "id", true); Account caller = CallContext.current().getCallingAccount(); + Long projectId = cmd.getProjectId(); + String accountName = cmd.getAccountName(); Object name = cmd.getServiceOfferingName(); Object id = cmd.getId(); Object keyword = cmd.getKeyword(); @@ -3309,7 +3379,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Integer memory = cmd.getMemory(); Integer cpuSpeed = cmd.getCpuSpeed(); Boolean encryptRoot = cmd.getEncryptRoot(); + String storageType = cmd.getStorageType(); + final Account owner = accountMgr.finalizeOwner(caller, accountName, domainId, projectId); SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); if (!accountMgr.isRootAdmin(caller.getId()) && isSystem) { throw new InvalidParameterValueException("Only ROOT admins can access system's offering"); @@ -3321,8 +3393,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (domainId != null && !accountMgr.isRootAdmin(caller.getId())) { // check if the user's domain == so's domain || user's domain is a // child of so's domain - if (!isPermissible(caller.getDomainId(), domainId)) { - throw new PermissionDeniedException("The account:" + caller.getAccountName() + " does not fall in the same domain hierarchy as the service offering"); + if (!isPermissible(owner.getDomainId(), domainId)) { + throw new PermissionDeniedException("The account:" + owner.getAccountName() + " does not fall in the same domain hierarchy as the service offering"); } } @@ -3388,16 +3460,16 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q throw new InvalidParameterValueException("Only root admins can access system's offering"); } if (isRecursive) { // domain + all sub-domains - if (caller.getType() == Account.Type.NORMAL) { + if (owner.getType() == Account.Type.NORMAL) { throw new InvalidParameterValueException("Only ROOT admins and Domain admins can list service offerings with isrecursive=true"); } } } else { // for root users - if (caller.getDomainId() != 1 && isSystem) { // NON ROOT admin - throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering"); + if (owner.getDomainId() != 1 && isSystem) { // NON ROOT admin + throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering."); } - if (domainId != null) { + if (domainId != null && accountName == null) { sc.addAnd("domainId", Op.FIND_IN_SET, String.valueOf(domainId)); } } @@ -3432,6 +3504,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.addAnd("vmType", SearchCriteria.Op.EQ, vmTypeStr); } + useStorageType(sc, storageType); + if (zoneId != null) { SearchBuilder sb = _srvOfferingJoinDao.createSearchBuilder(); sb.and("zoneId", sb.entity().getZoneId(), Op.FIND_IN_SET); @@ -3481,8 +3555,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // Filter offerings that are not associated with caller's domain // Fetch the offering ids from the details table since theres no smart way to filter them in the join ... yet! - if (caller.getType() != Account.Type.ADMIN) { - Domain callerDomain = _domainDao.findById(caller.getDomainId()); + if (owner.getType() != Account.Type.ADMIN) { + Domain callerDomain = _domainDao.findById(owner.getDomainId()); List domainIds = findRelatedDomainIds(callerDomain, isRecursive); List ids = _srvOfferingDetailsDao.findOfferingIdsByDomainIds(domainIds); @@ -4735,6 +4809,79 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } @Override + public ListResponse listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd) { + ListResponse response = new ListResponse<>(); + Pair, Integer> result = listSecondaryStorageSelectorsInternal(cmd.getZoneId(), cmd.getType(), cmd.isShowRemoved()); + List listOfSecondaryStorageHeuristicsResponses = new ArrayList<>(); + + for (Heuristic heuristic : result.first()) { + SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + listOfSecondaryStorageHeuristicsResponses.add(secondaryStorageHeuristicsResponse); + } + + response.setResponses(listOfSecondaryStorageHeuristicsResponses); + return response; + } + + private Pair, Integer> listSecondaryStorageSelectorsInternal(Long zoneId, String type, boolean showRemoved) { + SearchBuilder searchBuilder = secondaryStorageHeuristicDao.createSearchBuilder(); + + searchBuilder.and("zoneId", searchBuilder.entity().getZoneId(), SearchCriteria.Op.EQ); + searchBuilder.and("type", searchBuilder.entity().getType(), SearchCriteria.Op.EQ); + + searchBuilder.done(); + + SearchCriteria searchCriteria = searchBuilder.create(); + searchCriteria.setParameters("zoneId", zoneId); + searchCriteria.setParametersIfNotNull("type", type); + + return secondaryStorageHeuristicDao.searchAndCount(searchCriteria, null, showRemoved); + } + + @Override + public ListResponse listQuarantinedIps(ListQuarantinedIpsCmd cmd) { + ListResponse response = new ListResponse<>(); + Pair, Integer> result = listQuarantinedIpsInternal(cmd.isShowRemoved(), cmd.isShowInactive()); + List ipsQuarantinedResponses = new ArrayList<>(); + + for (PublicIpQuarantine quarantinedIp : result.first()) { + IpQuarantineResponse ipsInQuarantineResponse = responseGenerator.createQuarantinedIpsResponse(quarantinedIp); + ipsQuarantinedResponses.add(ipsInQuarantineResponse); + } + + response.setResponses(ipsQuarantinedResponses); + return response; + } + + /** + * It lists the quarantine IPs that the caller account is allowed to see by filtering the domain path of the caller account. + * Furthermore, it lists inactive and removed quarantined IPs according to the command parameters. + */ + private Pair, Integer> listQuarantinedIpsInternal(boolean showRemoved, boolean showInactive) { + String callingAccountDomainPath = _domainDao.findById(CallContext.current().getCallingAccount().getDomainId()).getPath(); + + SearchBuilder filterAllowedOnly = _accountJoinDao.createSearchBuilder(); + filterAllowedOnly.and("path", filterAllowedOnly.entity().getDomainPath(), SearchCriteria.Op.LIKE); + + SearchBuilder listAllPublicIpsInQuarantineAllowedToTheCaller = publicIpQuarantineDao.createSearchBuilder(); + listAllPublicIpsInQuarantineAllowedToTheCaller.join("listQuarantinedJoin", filterAllowedOnly, + listAllPublicIpsInQuarantineAllowedToTheCaller.entity().getPreviousOwnerId(), + filterAllowedOnly.entity().getId(), JoinBuilder.JoinType.INNER); + + if (!showInactive) { + listAllPublicIpsInQuarantineAllowedToTheCaller.and("endDate", listAllPublicIpsInQuarantineAllowedToTheCaller.entity().getEndDate(), SearchCriteria.Op.GT); + } + + filterAllowedOnly.done(); + listAllPublicIpsInQuarantineAllowedToTheCaller.done(); + + SearchCriteria searchCriteria = listAllPublicIpsInQuarantineAllowedToTheCaller.create(); + searchCriteria.setJoinParameters("listQuarantinedJoin", "path", callingAccountDomainPath + "%"); + searchCriteria.setParametersIfNotNull("endDate", new Date()); + + return publicIpQuarantineDao.searchAndCount(searchCriteria, null, showRemoved); + } + public ListResponse listSnapshots(ListSnapshotsCmd cmd) { Account caller = CallContext.current().getCallingAccount(); Pair, Integer> result = searchForSnapshotsWithParams(cmd.getId(), cmd.getIds(), @@ -4936,6 +5083,158 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return new Pair<>(snapshots, count); } + public ListResponse searchForObjectStores(ListObjectStoragePoolsCmd cmd) { + Pair, Integer> result = searchForObjectStoresInternal(cmd); + ListResponse response = new ListResponse(); + + List poolResponses = ViewResponseHelper.createObjectStoreResponse(result.first().toArray(new ObjectStoreVO[result.first().size()])); + response.setResponses(poolResponses, result.second()); + return response; + } + + private Pair, Integer> searchForObjectStoresInternal(ListObjectStoragePoolsCmd cmd) { + + Object id = cmd.getId(); + Object name = cmd.getStoreName(); + String provider = cmd.getProvider(); + Object keyword = cmd.getKeyword(); + Long startIndex = cmd.getStartIndex(); + Long pageSize = cmd.getPageSizeVal(); + + Filter searchFilter = new Filter(ObjectStoreVO.class, "id", Boolean.TRUE, startIndex, pageSize); + + SearchBuilder sb = objectStoreDao.createSearchBuilder(); + sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct + // ids + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("provider", sb.entity().getProviderName(), SearchCriteria.Op.EQ); + + SearchCriteria sc = sb.create(); + + if (keyword != null) { + SearchCriteria ssc = objectStoreDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + ssc.addOr("providerName", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + if (id != null) { + sc.setParameters("id", id); + } + + if (name != null) { + sc.setParameters("name", name); + } + + if (provider != null) { + sc.setParameters("provider", provider); + } + + // search Store details by ids + Pair, Integer> uniqueStorePair = objectStoreDao.searchAndCount(sc, searchFilter); + Integer count = uniqueStorePair.second(); + if (count.intValue() == 0) { + // empty result + return uniqueStorePair; + } + List uniqueStores = uniqueStorePair.first(); + Long[] osIds = new Long[uniqueStores.size()]; + int i = 0; + for (ObjectStoreVO v : uniqueStores) { + osIds[i++] = v.getId(); + } + List objectStores = objectStoreDao.searchByIds(osIds); + return new Pair<>(objectStores, count); + } + + + @Override + public ListResponse searchForBuckets(ListBucketsCmd listBucketsCmd) { + List buckets = searchForBucketsInternal(listBucketsCmd); + List bucketResponses = new ArrayList<>(); + for (BucketVO bucket : buckets) { + bucketResponses.add(responseGenerator.createBucketResponse(bucket)); + } + ListResponse response = new ListResponse<>(); + response.setResponses(bucketResponses, bucketResponses.size()); + return response; + } + + private List searchForBucketsInternal(ListBucketsCmd cmd) { + + Long id = cmd.getId(); + String name = cmd.getBucketName(); + Long storeId = cmd.getObjectStorageId(); + String keyword = cmd.getKeyword(); + Long startIndex = cmd.getStartIndex(); + Long pageSize = cmd.getPageSizeVal(); + Account caller = CallContext.current().getCallingAccount(); + List permittedAccounts = new ArrayList(); + + // Verify parameters + if (id != null) { + BucketVO bucket = bucketDao.findById(id); + if (bucket != null) { + accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, bucket); + } + } + + List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); + + Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), + cmd.isRecursive(), null); + accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); + Long domainId = domainIdRecursiveListProject.first(); + Boolean isRecursive = domainIdRecursiveListProject.second(); + ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + + Filter searchFilter = new Filter(BucketVO.class, "id", Boolean.TRUE, startIndex, pageSize); + + SearchBuilder sb = bucketDao.createSearchBuilder(); + accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct + // ids + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + + SearchCriteria sc = sb.create(); + accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + + if (keyword != null) { + SearchCriteria ssc = bucketDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + ssc.addOr("state", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + if (id != null) { + sc.setParameters("id", id); + } + + if (name != null) { + sc.setParameters("name", name); + } + + setIdsListToSearchCriteria(sc, ids); + + // search Volume details by ids + Pair, Integer> uniqueBktPair = bucketDao.searchAndCount(sc, searchFilter); + Integer count = uniqueBktPair.second(); + if (count.intValue() == 0) { + // empty result + return uniqueBktPair.first(); + } + List uniqueBkts = uniqueBktPair.first(); + Long[] bktIds = new Long[uniqueBkts.size()]; + int i = 0; + for (BucketVO b : uniqueBkts) { + bktIds[i++] = b.getId(); + } + + return bucketDao.searchByIds(bktIds); + } + @Override public String getConfigComponentName() { return QueryService.class.getSimpleName(); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index b909d69e80e..44096a799b7 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.Hashtable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,6 +44,7 @@ import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -58,6 +60,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; @@ -659,4 +662,20 @@ public class ViewResponseHelper { return new ArrayList(vrDataList.values()); } + public static List createObjectStoreResponse(ObjectStoreVO[] stores) { + Hashtable storeList = new Hashtable(); + // Initialise the storeList with the input data + for (ObjectStoreVO store : stores) { + ObjectStoreResponse storeData = storeList.get(store.getId()); + if (storeData == null) { + // first time encountering this store + storeData = ApiDBUtils.newObjectStoreResponse(store); + } else { + // update tags + storeData = ApiDBUtils.fillObjectStoreDetails(storeData, store); + } + storeList.put(store.getId(), storeData); + } + return new ArrayList<>(storeList.values()); + } } diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java index bf3ded1fef0..da81f42b41d 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java @@ -205,6 +205,7 @@ public class HostJoinDaoImpl extends GenericDaoBase implements String hostTags = host.getTag(); hostResponse.setHostTags(hostTags); + hostResponse.setIsTagARule(host.getIsTagARule()); hostResponse.setHaHost(containsHostHATag(hostTags)); hostResponse.setHypervisorVersion(host.getHypervisorVersion()); diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java index 87659210ad7..9028f3418b3 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.response.StoragePoolResponse; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.storage.StoragePool; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; public interface StoragePoolJoinDao extends GenericDao { @@ -44,4 +45,6 @@ public interface StoragePoolJoinDao extends GenericDao Pair, Integer> searchAndCount(Long storagePoolId, String storagePoolName, Long zoneId, String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, String keyword, Filter searchFilter); + List findStoragePoolByScopeAndRuleTags(Long datacenterId, Long podId, Long clusterId, ScopeType scopeType, List tags); + } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index de469d21a11..e75e86108c7 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -25,6 +25,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StorageStats; +import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.user.AccountManager; import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; @@ -44,6 +45,7 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -71,10 +73,13 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase spSearch; private final SearchBuilder spIdSearch; + private final SearchBuilder findByDatacenterAndScopeSb; + protected StoragePoolJoinDaoImpl() { spSearch = createSearchBuilder(); @@ -85,6 +90,15 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase findStoragePoolByScopeAndRuleTags(Long datacenterId, Long podId, Long clusterId, ScopeType scopeType, List tags) { + SearchCriteria sc = findByDatacenterAndScopeSb.create(); + if (datacenterId != null) { + sc.setParameters("zoneId", datacenterId); + } + if (clusterId != null) { + sc.setParameters("clusterId", clusterId); + } + if (podId != null) { + sc.setParameters("podId", podId); + } + + sc.setParameters("scope", scopeType); + sc.setParameters("status", "Up"); + sc.setParameters("is_tag_a_rule", true); + List storagePools = search(sc, null, false, false); + + List filteredPools = new ArrayList<>(); + + StringBuilder injectableTagsBuilder = new StringBuilder(); + for (String tag : tags) { + injectableTagsBuilder.append(tag).append(","); + } + if (!tags.isEmpty()) { + injectableTagsBuilder.deleteCharAt(injectableTagsBuilder.length() - 1); + } + String injectableTag = injectableTagsBuilder.toString(); + + for (StoragePoolJoinVO storagePoolJoinVO : storagePools) { + if (TagAsRuleHelper.interpretTagAsRule(storagePoolJoinVO.getTag(), injectableTag, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value())) { + StoragePoolVO storagePoolVO = storagePoolDao.findById(storagePoolJoinVO.getId()); + if (storagePoolVO != null) { + filteredPools.add(storagePoolVO); + } else { + s_logger.warn(String.format("Unable to find Storage Pool [%s] in the DB.", storagePoolJoinVO.getUuid())); + } + } + } + return filteredPools; + } + } diff --git a/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java index bb3b6935389..78a45429463 100644 --- a/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java @@ -172,6 +172,9 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private Boolean isTagARule; + @Column(name = "memory_used_capacity") private long memUsedCapacity; @@ -388,6 +391,10 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity return tag; } + public Boolean getIsTagARule() { + return isTagARule; + } + public String getAnnotation() { return annotation; } diff --git a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java index 1831aaafac9..5eb04d2e00d 100644 --- a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java @@ -110,6 +110,9 @@ public class StoragePoolJoinVO extends BaseViewVO implements InternalIdentity, I @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + @Column(name = "disk_used_capacity") private long usedCapacity; @@ -243,6 +246,10 @@ public class StoragePoolJoinVO extends BaseViewVO implements InternalIdentity, I return tag; } + public boolean getIsTagARule() { + return isTagARule; + } + public long getUsedCapacity() { return usedCapacity; } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 927c174fc38..37df5c2de1d 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -52,6 +52,10 @@ import com.cloud.hypervisor.HypervisorGuru; import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.element.NsxProviderVO; import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.host.HostTagVO; +import com.cloud.storage.StoragePoolTagVO; +import com.cloud.storage.VolumeApiServiceImpl; +import com.googlecode.ipv6.IPv6Address; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -132,6 +136,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -298,7 +303,6 @@ import com.google.common.base.Enums; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; -import com.googlecode.ipv6.IPv6Address; import com.googlecode.ipv6.IPv6Network; public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable { @@ -4093,17 +4097,23 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (CollectionUtils.isNotEmpty(pools)) { List listOfTags = Arrays.asList(tags.split(",")); for (StoragePoolVO storagePoolVO : pools) { - List tagsOnPool = storagePoolTagDao.getStoragePoolTags(storagePoolVO.getId()); - if (CollectionUtils.isEmpty(tagsOnPool) || !tagsOnPool.containsAll(listOfTags)) { - DiskOfferingVO offeringToRetrieveInfo = _diskOfferingDao.findById(diskOffering.getId()); - List volumes = _volumeDao.findByDiskOfferingId(diskOffering.getId()); - String listOfVolumesNamesAndUuid = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(volumes, "name", "uuid"); - String diskOfferingInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(offeringToRetrieveInfo, "name", "uuid"); - String poolInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(storagePoolVO, "name", "uuid"); - throw new InvalidParameterValueException(String.format("There are active volumes using the disk offering %s, and the pool %s doesn't have the new tags. " + - "The following volumes are using the mentioned disk offering %s. Please first add the new tags to the mentioned storage pools before adding them" + - " to the disk offering.", diskOfferingInfo, poolInfo, listOfVolumesNamesAndUuid)); + List tagsOnPool = storagePoolTagDao.findStoragePoolTags(storagePoolVO.getId()); + List tagsAsString = tagsOnPool.stream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()); + + if ((CollectionUtils.isNotEmpty(tagsAsString) && tagsAsString.containsAll(listOfTags)) || + (tagsOnPool.size() == 1 && tagsOnPool.get(0).isTagARule() && + TagAsRuleHelper.interpretTagAsRule(tagsOnPool.get(0).getTag(), tags, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value()))) { + continue; } + + DiskOfferingVO offeringToRetrieveInfo = _diskOfferingDao.findById(diskOffering.getId()); + List volumes = _volumeDao.findByDiskOfferingId(diskOffering.getId()); + String listOfVolumesNamesAndUuid = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(volumes, "name", "uuid"); + String diskOfferingInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(offeringToRetrieveInfo, "name", "uuid"); + String poolInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(storagePoolVO, "name", "uuid"); + throw new InvalidParameterValueException(String.format("There are active volumes using the disk offering %s, and the pool %s doesn't have the new tags. " + + "The following volumes are using the mentioned disk offering %s. Please first add the new tags to the mentioned storage pools before adding them" + + " to the disk offering.", diskOfferingInfo, poolInfo, listOfVolumesNamesAndUuid)); } } diskOffering.setTags(tags); @@ -4130,10 +4140,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (CollectionUtils.isNotEmpty(hosts)) { List listOfHostTags = Arrays.asList(hostTags.split(",")); for (HostVO host : hosts) { - List tagsOnHost = hostTagDao.getHostTags(host.getId()); - if (CollectionUtils.isEmpty(tagsOnHost) || !tagsOnHost.containsAll(listOfHostTags)) { - throw new InvalidParameterValueException(String.format("There are active VMs using offering [%s], and the hosts [%s] don't have the new tags", offering.getId(), hosts)); + List tagsOnHost = hostTagDao.getHostTags(host.getId()); + List tagsAsString = tagsOnHost.stream().map(HostTagVO::getTag).collect(Collectors.toList()); + + if ((CollectionUtils.isNotEmpty(tagsAsString) && tagsAsString.containsAll(listOfHostTags)) || + (tagsOnHost.size() == 1 && tagsOnHost.get(0).getIsTagARule() && + TagAsRuleHelper.interpretTagAsRule(tagsOnHost.get(0).getTag(), hostTags, HostTagsDao.hostTagRuleExecutionTimeout.value()))) { + continue; } + + throw new InvalidParameterValueException(String.format("There are active VMs using offering [%s], and the hosts [%s] don't have the new tags", + offering.getId(), hosts)); } } offering.setHostTag(hostTags); diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index f71f46fdc03..cb22e81f366 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -1361,7 +1361,7 @@ StateListener, Configurable { if (vmRequiresSharedStorage) { // check shared pools - List allPoolsInCluster = _storagePoolDao.findPoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null); + List allPoolsInCluster = _storagePoolDao.findPoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null, false, 0); for (StoragePoolVO pool : allPoolsInCluster) { if (!allocatorAvoidOutput.shouldAvoid(pool)) { // there's some pool in the cluster that is not yet in avoid set @@ -1374,7 +1374,7 @@ StateListener, Configurable { if (vmRequiresLocalStorege) { // check local pools List allLocalPoolsInCluster = - _storagePoolDao.findLocalStoragePoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null); + _storagePoolDao.findLocalStoragePoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null, false); for (StoragePoolVO pool : allLocalPoolsInCluster) { if (!allocatorAvoidOutput.shouldAvoid(pool)) { // there's some pool in the cluster that is not yet @@ -1686,6 +1686,16 @@ StateListener, Configurable { for (VolumeVO toBeCreated : volumesTobeCreated) { s_logger.debug("Checking suitable pools for volume (Id, Type): (" + toBeCreated.getId() + "," + toBeCreated.getVolumeType().name() + ")"); + if (toBeCreated.getState() == Volume.State.Allocated && toBeCreated.getPoolId() != null) { + toBeCreated.setPoolId(null); + if (!_volsDao.update(toBeCreated.getId(), toBeCreated)) { + throw new CloudRuntimeException(String.format("Error updating volume [%s] to clear pool Id.", toBeCreated.getId())); + } + if (s_logger.isDebugEnabled()) { + String msg = String.format("Setting pool_id to NULL for volume id=%s as it is in Allocated state", toBeCreated.getId()); + s_logger.debug(msg); + } + } // If the plan specifies a poolId, it means that this VM's ROOT // volume is ready and the pool should be reused. // In this case, also check if rest of the volumes are ready and can diff --git a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java index 4e838510010..c2969ecce50 100644 --- a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java +++ b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java @@ -27,6 +27,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.capacity.CapacityVO; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -521,6 +522,12 @@ public class FirstFitPlanner extends AdapterBase implements DeploymentClusterPla private void removeClustersWithoutMatchingTag(List clusterListForVmAllocation, String hostTagOnOffering) { List matchingClusters = hostDao.listClustersByHostTag(hostTagOnOffering); + matchingClusters.addAll(hostDao.findClustersThatMatchHostTagRule(hostTagOnOffering)); + + if (matchingClusters.isEmpty()) { + s_logger.error(String.format("No suitable host found for the following compute offering tags [%s].", hostTagOnOffering)); + throw new CloudRuntimeException("No suitable host found."); + } clusterListForVmAllocation.retainAll(matchingClusters); diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index c85328807df..75ea572491e 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -18,11 +18,13 @@ package com.cloud.network; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.Set; import java.util.UUID; @@ -31,6 +33,8 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.network.dao.PublicIpQuarantineDao; +import com.cloud.network.vo.PublicIpQuarantineVO; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.annotation.AnnotationService; @@ -308,6 +312,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @Inject MessageBus messageBus; + @Inject + PublicIpQuarantineDao publicIpQuarantineDao; + SearchBuilder AssignIpAddressSearch; SearchBuilder AssignIpAddressFromPodVlanSearch; private static final Object allocatedLock = new Object(); @@ -318,6 +325,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage Boolean.class, "system.vm.public.ip.reservation.mode.strictness", "false", "If enabled, the use of System VMs public IP reservation is strict, preferred if not.", true, ConfigKey.Scope.Global); + public static final ConfigKey PUBLIC_IP_ADDRESS_QUARANTINE_DURATION = new ConfigKey<>("Network", Integer.class, "public.ip.address.quarantine.duration", + "0", "The duration (in minutes) for the public IP address to be quarantined when it is disassociated.", true, ConfigKey.Scope.Domain); + private Random rand = new Random(System.currentTimeMillis()); private List getIpv6SupportingVlanRangeIds(long dcId) throws InsufficientAddressCapacityException { @@ -523,8 +533,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage return true; } - private IpAddress allocateIP(Account ipOwner, boolean isSystem, long zoneId) throws ResourceAllocationException, InsufficientAddressCapacityException, - ConcurrentOperationException { + private IpAddress allocateIP(Account ipOwner, boolean isSystem, long zoneId) throws InsufficientAddressCapacityException, ConcurrentOperationException { Account caller = CallContext.current().getCallingAccount(); long callerUserId = CallContext.current().getCallingUserId(); // check permissions @@ -698,6 +707,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage public boolean disassociatePublicIpAddress(long addrId, long userId, Account caller) { boolean success = true; + IPAddressVO ipToBeDisassociated = _ipAddressDao.findById(addrId); + + PublicIpQuarantine publicIpQuarantine = null; // Cleanup all ip address resources - PF/LB/Static nat rules if (!cleanupIpResources(addrId, userId, caller)) { success = false; @@ -723,10 +735,9 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage } catch (ResourceUnavailableException e) { throw new CloudRuntimeException("We should never get to here because we used true when applyIpAssociations", e); } - } else { - if (ip.getState() == IpAddress.State.Releasing) { - _ipAddressDao.unassignIpAddress(ip.getId()); - } + } else if (ip.getState() == State.Releasing) { + publicIpQuarantine = addPublicIpAddressToQuarantine(ipToBeDisassociated, caller.getDomainId()); + _ipAddressDao.unassignIpAddress(ip.getId()); } annotationDao.removeByEntityType(AnnotationService.EntityType.PUBLIC_IP_ADDRESS.name(), ip.getUuid()); @@ -736,6 +747,8 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage releasePortableIpAddress(addrId); } s_logger.debug("Released a public ip id=" + addrId); + } else if (publicIpQuarantine != null) { + removePublicIpAddressFromQuarantine(publicIpQuarantine.getId(), "Public IP address removed from quarantine as there was an error while disassociating it."); } return success; @@ -972,6 +985,13 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage if (lockOneRow) { assert (addrs.size() == 1) : "Return size is incorrect: " + addrs.size(); + IpAddress ipAddress = addrs.get(0); + boolean ipCanBeAllocated = canPublicIpAddressBeAllocated(ipAddress, owner); + + if (!ipCanBeAllocated) { + throw new InsufficientAddressCapacityException(String.format("Failed to allocate public IP address [%s] as it is in quarantine.", ipAddress.getAddress()), + DataCenter.class, dcId); + } } if (assign && !fetchFromDedicatedRange && VlanType.VirtualNetwork.equals(vlanUse)) { @@ -1126,6 +1146,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage } else if (addr.getState() == IpAddress.State.Releasing) { // Cleanup all the resources for ip address if there are any, and only then un-assign ip in the system if (cleanupIpResources(addr.getId(), Account.ACCOUNT_ID_SYSTEM, _accountMgr.getSystemAccount())) { + addPublicIpAddressToQuarantine(addr, network.getDomainId()); _ipAddressDao.unassignIpAddress(addr.getId()); messageBus.publish(_name, MESSAGE_RELEASE_IPADDR_EVENT, PublishScope.LOCAL, addr); } else { @@ -1258,8 +1279,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @DB @Override public IpAddress allocateIp(final Account ipOwner, final boolean isSystem, Account caller, long callerUserId, final DataCenter zone, final Boolean displayIp, final String ipaddress) - throws ConcurrentOperationException, - ResourceAllocationException, InsufficientAddressCapacityException { + throws ConcurrentOperationException, InsufficientAddressCapacityException, CloudRuntimeException { final VlanType vlanType = VlanType.VirtualNetwork; final boolean assign = false; @@ -2347,7 +2367,8 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {UseSystemPublicIps, RulesContinueOnError, SystemVmPublicIpReservationModeStrictness, VrouterRedundantTiersPlacement, AllowUserListAvailableIpsOnSharedNetwork}; + return new ConfigKey[] {UseSystemPublicIps, RulesContinueOnError, SystemVmPublicIpReservationModeStrictness, VrouterRedundantTiersPlacement, AllowUserListAvailableIpsOnSharedNetwork, + PUBLIC_IP_ADDRESS_QUARANTINE_DURATION}; } /** @@ -2381,6 +2402,96 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage return SystemVmPublicIpReservationModeStrictness; } + @Override + public boolean canPublicIpAddressBeAllocated(IpAddress ip, Account newOwner) { + PublicIpQuarantineVO publicIpQuarantineVO = publicIpQuarantineDao.findByPublicIpAddressId(ip.getId()); + + if (publicIpQuarantineVO == null) { + s_logger.debug(String.format("Public IP address [%s] is not in quarantine; therefore, it is allowed to be allocated.", ip)); + return true; + } + + if (!isPublicIpAddressStillInQuarantine(publicIpQuarantineVO, new Date())) { + s_logger.debug(String.format("Public IP address [%s] is no longer in quarantine; therefore, it is allowed to be allocated.", ip)); + return true; + } + + Account previousOwner = _accountMgr.getAccount(publicIpQuarantineVO.getPreviousOwnerId()); + + if (Objects.equals(previousOwner.getUuid(), newOwner.getUuid())) { + s_logger.debug(String.format("Public IP address [%s] is in quarantine; however, the Public IP previous owner [%s] is the same as the new owner [%s]; therefore the IP" + + " can be allocated. The public IP address will be removed from quarantine.", ip, previousOwner, newOwner)); + removePublicIpAddressFromQuarantine(publicIpQuarantineVO.getId(), "IP was removed from quarantine because it has been allocated by the previous owner"); + return true; + } + + s_logger.error(String.format("Public IP address [%s] is in quarantine and the previous owner [%s] is different than the new owner [%s]; therefore, the IP cannot be " + + "allocated.", ip, previousOwner, newOwner)); + return false; + } + + public boolean isPublicIpAddressStillInQuarantine(PublicIpQuarantineVO publicIpQuarantineVO, Date currentDate) { + Date quarantineEndDate = publicIpQuarantineVO.getEndDate(); + Date removedDate = publicIpQuarantineVO.getRemoved(); + boolean hasQuarantineEndedEarly = removedDate != null; + + return hasQuarantineEndedEarly && currentDate.before(removedDate) || + !hasQuarantineEndedEarly && currentDate.before(quarantineEndDate); + } + + @Override + public PublicIpQuarantine addPublicIpAddressToQuarantine(IpAddress publicIpAddress, Long domainId) { + Integer quarantineDuration = PUBLIC_IP_ADDRESS_QUARANTINE_DURATION.valueInDomain(domainId); + if (quarantineDuration <= 0) { + s_logger.debug(String.format("Not adding IP [%s] to quarantine because configuration [%s] has value equal or less to 0.", publicIpAddress.getAddress(), + PUBLIC_IP_ADDRESS_QUARANTINE_DURATION.key())); + return null; + } + + long ipId = publicIpAddress.getId(); + long accountId = publicIpAddress.getAccountId(); + + if (accountId == Account.ACCOUNT_ID_SYSTEM) { + s_logger.debug(String.format("Not adding IP [%s] to quarantine because it belongs to the system account.", publicIpAddress.getAddress())); + return null; + } + + Date currentDate = new Date(); + Calendar quarantineEndDate = Calendar.getInstance(); + quarantineEndDate.setTime(currentDate); + quarantineEndDate.add(Calendar.MINUTE, quarantineDuration); + + PublicIpQuarantineVO publicIpQuarantine = new PublicIpQuarantineVO(ipId, accountId, currentDate, quarantineEndDate.getTime()); + s_logger.debug(String.format("Adding public IP Address [%s] to quarantine for the duration of [%s] minute(s).", publicIpAddress.getAddress(), quarantineDuration)); + return publicIpQuarantineDao.persist(publicIpQuarantine); + } + + @Override + public void removePublicIpAddressFromQuarantine(Long quarantineProcessId, String removalReason) { + PublicIpQuarantineVO publicIpQuarantineVO = publicIpQuarantineDao.findById(quarantineProcessId); + Ip ipAddress = _ipAddressDao.findById(publicIpQuarantineVO.getPublicIpAddressId()).getAddress(); + Date removedDate = new Date(); + + publicIpQuarantineVO.setRemoved(removedDate); + publicIpQuarantineVO.setRemovalReason(removalReason); + + s_logger.debug(String.format("Removing public IP Address [%s] from quarantine by updating the removed date to [%s].", ipAddress, removedDate)); + publicIpQuarantineDao.persist(publicIpQuarantineVO); + } + + @Override + public PublicIpQuarantine updatePublicIpAddressInQuarantine(Long quarantineProcessId, Date newEndDate) { + PublicIpQuarantineVO publicIpQuarantineVO = publicIpQuarantineDao.findById(quarantineProcessId); + Ip ipAddress = _ipAddressDao.findById(publicIpQuarantineVO.getPublicIpAddressId()).getAddress(); + Date currentEndDate = publicIpQuarantineVO.getEndDate(); + + publicIpQuarantineVO.setEndDate(newEndDate); + + s_logger.debug(String.format("Updating the end date for the quarantine of the public IP Address [%s] from [%s] to [%s].", ipAddress, currentEndDate, newEndDate)); + publicIpQuarantineDao.persist(publicIpQuarantineVO); + return publicIpQuarantineVO; + } + @Override public void updateSourceNatIpAddress(IPAddressVO requestedIp, List userIps) throws Exception{ Transaction.execute((TransactionCallbackWithException) status -> { diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 0dd4eb487d8..325cd42ca3f 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -2559,10 +2559,11 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi return false; } - //if the network has vms in Starting state (nics for those might not be allocated yet as Starting state also used when vm is being Created) - //don't GC - if (_nicDao.countNicsForStartingVms(networkId) > 0) { - s_logger.debug("Network id=" + networkId + " is not ready for GC as it has vms that are Starting at the moment"); + // if the network has user vms in Starting/Stopping/Migrating/Running state, or VRs in Starting/Stopping/Migrating state, don't GC + // The active nics count (nics_count in op_networks table) might be wrong due to some reasons, should check the state of vms as well. + // (nics for Starting VMs might not be allocated yet as Starting state also used when vm is being Created) + if (_nicDao.countNicsForNonStoppedVms(networkId) > 0 || _nicDao.countNicsForNonStoppedRunningVrs(networkId) > 0) { + s_logger.debug("Network id=" + networkId + " is not ready for GC as it has vms that are not Stopped at the moment"); return false; } diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 38a6a39c7a8..1b634d5b904 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -43,6 +43,7 @@ import javax.naming.ConfigurationException; import com.cloud.dc.VlanDetailsVO; import com.cloud.dc.dao.VlanDetailsDao; +import com.cloud.network.dao.PublicIpQuarantineDao; import com.cloud.offering.ServiceOffering; import com.cloud.service.dao.ServiceOfferingDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; @@ -57,6 +58,8 @@ import org.apache.cloudstack.api.command.admin.network.ListGuestVlansCmd; import org.apache.cloudstack.api.command.admin.network.ListNetworksCmdByAdmin; import org.apache.cloudstack.api.command.admin.network.UpdateNetworkCmdByAdmin; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd; +import org.apache.cloudstack.api.command.user.address.RemoveQuarantinedIpCmd; +import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkPermissionsCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkPermissionsCmd; @@ -405,6 +408,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C CommandSetupHelper commandSetupHelper; @Inject ServiceOfferingDao serviceOfferingDao; + @Inject + PublicIpQuarantineDao publicIpQuarantineDao; @Autowired @Qualifier("networkHelper") @@ -2151,12 +2156,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C throw new InvalidParameterValueException("Unable to find specified NetworkACL"); } - if (aclId != NetworkACL.DEFAULT_DENY && aclId != NetworkACL.DEFAULT_ALLOW) { - // ACL is not default DENY/ALLOW - // ACL should be associated with a VPC - if (!vpcId.equals(acl.getVpcId())) { - throw new InvalidParameterValueException("ACL: " + aclId + " do not belong to the VPC"); - } + Long aclVpcId = acl.getVpcId(); + if (!isDefaultAcl(aclId) && isAclAttachedToVpc(aclVpcId, vpcId)) { + throw new InvalidParameterValueException(String.format("ACL [%s] does not belong to the VPC [%s].", aclId, aclVpcId)); } } network = _vpcMgr.createVpcGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, networkDomain, owner, sharedDomainId, pNtwk, zoneId, aclType, @@ -2849,6 +2851,11 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C return _ipAddressDao.findById(ipAddressId); } + @Override + public IpAddress getIp(String ipAddress) { + return _ipAddressDao.findByIp(ipAddress); + } + protected boolean providersConfiguredForExternalNetworking(Collection providers) { for (String providerStr : providers) { Provider provider = Network.Provider.getProvider(providerStr); @@ -6012,4 +6019,83 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C public ConfigKey[] getConfigKeys() { return new ConfigKey[] {AllowDuplicateNetworkName, AllowEmptyStartEndIpAddress, VRPrivateInterfaceMtu, VRPublicInterfaceMtu, AllowUsersToSpecifyVRMtu}; } + + public boolean isDefaultAcl(Long aclId) { + return aclId == NetworkACL.DEFAULT_DENY || aclId == NetworkACL.DEFAULT_ALLOW; + } + + public boolean isAclAttachedToVpc(Long aclVpcId, Long vpcId) { + return aclVpcId != 0 && !vpcId.equals(aclVpcId); + } + + @Override + public PublicIpQuarantine updatePublicIpAddressInQuarantine(UpdateQuarantinedIpCmd cmd) throws CloudRuntimeException { + Long ipId = cmd.getId(); + String ipAddress = cmd.getIpAddress(); + Date newEndDate = cmd.getEndDate(); + + if (new Date().after(newEndDate)) { + throw new InvalidParameterValueException(String.format("The given end date [%s] is invalid as it is before the current date.", newEndDate)); + } + + PublicIpQuarantine publicIpQuarantine = retrievePublicIpQuarantine(ipId, ipAddress); + checkCallerForPublicIpQuarantineAccess(publicIpQuarantine); + + String publicIpQuarantineAddress = _ipAddressDao.findById(publicIpQuarantine.getPublicIpAddressId()).getAddress().toString(); + Date currentEndDate = publicIpQuarantine.getEndDate(); + + if (new Date().after(currentEndDate)) { + throw new CloudRuntimeException(String.format("The quarantine for the public IP address [%s] is no longer active; thus, it cannot be updated.", publicIpQuarantineAddress)); + } + + return _ipAddrMgr.updatePublicIpAddressInQuarantine(publicIpQuarantine.getId(), newEndDate); + } + + @Override + public void removePublicIpAddressFromQuarantine(RemoveQuarantinedIpCmd cmd) throws CloudRuntimeException { + Long ipId = cmd.getId(); + String ipAddress = cmd.getIpAddress(); + PublicIpQuarantine publicIpQuarantine = retrievePublicIpQuarantine(ipId, ipAddress); + + String removalReason = cmd.getRemovalReason(); + if (StringUtils.isBlank(removalReason)) { + s_logger.error("The removalReason parameter cannot be blank."); + ipAddress = ObjectUtils.defaultIfNull(ipAddress, _ipAddressDao.findById(publicIpQuarantine.getPublicIpAddressId()).getAddress().toString()); + throw new CloudRuntimeException(String.format("The given reason for removing the public IP address [%s] from quarantine is blank.", ipAddress)); + } + + checkCallerForPublicIpQuarantineAccess(publicIpQuarantine); + + _ipAddrMgr.removePublicIpAddressFromQuarantine(publicIpQuarantine.getId(), removalReason); + } + + /** + * Retrieves the active quarantine for the given public IP address. It can find by the ID of the quarantine or the address of the public IP. + * @throws CloudRuntimeException if it does not find an active quarantine for the given public IP. + */ + protected PublicIpQuarantine retrievePublicIpQuarantine(Long ipId, String ipAddress) throws CloudRuntimeException { + PublicIpQuarantine publicIpQuarantine; + if (ipId != null) { + s_logger.debug("The ID of the IP in quarantine was informed; therefore, the `ipAddress` parameter will be ignored."); + publicIpQuarantine = publicIpQuarantineDao.findById(ipId); + } else if (ipAddress != null) { + s_logger.debug("The address of the IP in quarantine was informed, it will be used to fetch its metadata."); + publicIpQuarantine = publicIpQuarantineDao.findByIpAddress(ipAddress); + } else { + throw new CloudRuntimeException("Either the ID or the address of the IP in quarantine must be informed."); + } + + if (publicIpQuarantine == null) { + throw new CloudRuntimeException("There is no active quarantine for the specified IP address."); + } + + return publicIpQuarantine; + } + + protected void checkCallerForPublicIpQuarantineAccess(PublicIpQuarantine publicIpQuarantine) { + Account callingAccount = CallContext.current().getCallingAccount(); + DomainVO domainOfThePreviousOwner = _domainDao.findById(_accountDao.findById(publicIpQuarantine.getPreviousOwnerId()).getDomainId()); + + _accountMgr.checkAccess(callingAccount, domainOfThePreviousOwner); + } } diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelper.java b/server/src/main/java/com/cloud/network/router/NetworkHelper.java index ea008e4c4ca..c9daa5eedb4 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelper.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelper.java @@ -20,6 +20,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.network.router.deployment.RouterDeploymentDefinition; import com.cloud.agent.api.to.NicTO; @@ -93,4 +94,6 @@ public interface NetworkHelper { throws ConcurrentOperationException, InsufficientAddressCapacityException; public boolean validateHAProxyLBRule(final LoadBalancingRule rule); + + public Map> getHypervisorRouterTemplateConfigMap(); } diff --git a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java index a03566b0e10..8acab701e42 100644 --- a/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java +++ b/server/src/main/java/com/cloud/network/router/NetworkHelperImpl.java @@ -972,4 +972,9 @@ public class NetworkHelperImpl implements NetworkHelper { public String acquireGuestIpAddressForVrouterRedundant(Network network) { return _ipAddrMgr.acquireGuestIpAddressByPlacement(network, null); } + + @Override + public Map> getHypervisorRouterTemplateConfigMap() { + return hypervisorsMap; + } } diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java index ed37eb5b375..8139ac1c49e 100644 --- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.exception.PermissionDeniedException; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; @@ -103,12 +104,15 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ @Override public NetworkACL createNetworkACL(final String name, final String description, final long vpcId, final Boolean forDisplay) { - final Account caller = CallContext.current().getCallingAccount(); - final Vpc vpc = _entityMgr.findById(Vpc.class, vpcId); - if (vpc == null) { - throw new InvalidParameterValueException("Unable to find VPC"); + if (vpcId != 0) { + final Account caller = CallContext.current().getCallingAccount(); + final Vpc vpc = _vpcDao.findById(vpcId); + if (vpc == null) { + throw new InvalidParameterValueException(String.format("Unable to find VPC with ID [%s].", vpcId)); + } + _accountMgr.checkAccess(caller, null, true, vpc); + } - _accountMgr.checkAccess(caller, null, true, vpc); return _networkAclMgr.createNetworkACL(name, description, vpcId, forDisplay); } @@ -212,22 +216,17 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ @Override @ActionEvent(eventType = EventTypes.EVENT_NETWORK_ACL_DELETE, eventDescription = "Deleting Network ACL List", async = true) public boolean deleteNetworkACL(final long id) { - final Account caller = CallContext.current().getCallingAccount(); final NetworkACL acl = _networkACLDao.findById(id); + Account account = CallContext.current().getCallingAccount(); if (acl == null) { throw new InvalidParameterValueException("Unable to find specified ACL"); } - //Do not allow deletion of default ACLs - if (acl.getId() == NetworkACL.DEFAULT_ALLOW || acl.getId() == NetworkACL.DEFAULT_DENY) { + if (isDefaultAcl(acl.getId())) { throw new InvalidParameterValueException("Default ACL cannot be removed"); } - final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Unable to find specified VPC associated with the ACL"); - } - _accountMgr.checkAccess(caller, null, true, vpc); + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Only Root Admin can delete global ACLs."); return _networkAclMgr.deleteNetworkACL(acl); } @@ -253,7 +252,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ throw new InvalidParameterValueException("Unable to find specified vpc id"); } - if (aclId != NetworkACL.DEFAULT_DENY && aclId != NetworkACL.DEFAULT_ALLOW) { + if (!isDefaultAcl(aclId)) { final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); if (vpc == null) { throw new InvalidParameterValueException("Unable to find Vpc associated with the NetworkACL"); @@ -293,15 +292,9 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ throw new InvalidParameterValueException("Network ACL can be created just for networks of type " + Networks.TrafficType.Guest); } - if (aclId != NetworkACL.DEFAULT_DENY && aclId != NetworkACL.DEFAULT_ALLOW) { - //ACL is not default DENY/ALLOW - // ACL should be associated with a VPC - final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Unable to find Vpc associated with the NetworkACL"); - } + if (!isDefaultAcl(aclId) && !isGlobalAcl(acl.getVpcId())) { + validateAclAssociatedToVpc(acl.getVpcId(), caller, acl.getUuid()); - _accountMgr.checkAccess(caller, null, true, vpc); if (!network.getVpcId().equals(acl.getVpcId())) { throw new InvalidParameterValueException("Network: " + networkId + " and ACL: " + aclId + " do not belong to the same VPC"); } @@ -340,6 +333,11 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ NetworkACL acl = _networkAclMgr.getNetworkACL(aclId); validateNetworkAcl(acl); + Account caller = CallContext.current().getCallingAccount(); + + if (isGlobalAcl(acl.getVpcId()) && !Account.Type.ADMIN.equals(caller.getType())) { + throw new PermissionDeniedException("Only Root Admins can create rules for a global ACL."); + } validateAclRuleNumber(createNetworkACLCmd, acl); NetworkACLItem.Action ruleAction = validateAndCreateNetworkAclRuleAction(action); @@ -409,7 +407,8 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ *
    *
  • If the parameter is null, we return an {@link InvalidParameterValueException}; *
  • Default ACLs {@link NetworkACL#DEFAULT_ALLOW} and {@link NetworkACL#DEFAULT_DENY} cannot be modified. Therefore, if any of them is provided we throw a {@link InvalidParameterValueException}; - *
  • If the network does not have a VPC, we will throw an {@link InvalidParameterValueException}. + *
  • If no VPC is given, then it is a global ACL and there is no need to check any VPC ID. However, if a VPC is given and it does not exist, throws an + * {@link InvalidParameterValueException}. *
* * After all validations, we check if the user has access to the given network ACL using {@link AccountManager#checkAccess(Account, org.apache.cloudstack.acl.SecurityChecker.AccessType, boolean, org.apache.cloudstack.acl.ControlledEntity...)}. @@ -419,16 +418,14 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ throw new InvalidParameterValueException("Unable to find specified ACL."); } - if (acl.getId() == NetworkACL.DEFAULT_DENY || acl.getId() == NetworkACL.DEFAULT_ALLOW) { + if (isDefaultAcl(acl.getId())) { throw new InvalidParameterValueException("Default ACL cannot be modified"); } - Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException(String.format("Unable to find Vpc associated with the NetworkACL [%s]", acl.getUuid())); + Long aclVpcId = acl.getVpcId(); + if (!isGlobalAcl(aclVpcId)) { + validateAclAssociatedToVpc(aclVpcId, CallContext.current().getCallingAccount(), acl.getUuid()); } - Account caller = CallContext.current().getCallingAccount(); - _accountMgr.checkAccess(caller, null, true, vpc); } /** @@ -792,17 +789,12 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ final NetworkACLItemVO aclItem = _networkACLItemDao.findById(ruleId); if (aclItem != null) { final NetworkACL acl = _networkAclMgr.getNetworkACL(aclItem.getAclId()); + final Account account = CallContext.current().getCallingAccount(); - final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - - if (aclItem.getAclId() == NetworkACL.DEFAULT_ALLOW || aclItem.getAclId() == NetworkACL.DEFAULT_DENY) { + if (isDefaultAcl(aclItem.getAclId())) { throw new InvalidParameterValueException("ACL Items in default ACL cannot be deleted"); } - - final Account caller = CallContext.current().getCallingAccount(); - - _accountMgr.checkAccess(caller, null, true, vpc); - + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Only Root Admin can delete global ACL rules."); } return _networkAclMgr.revokeNetworkACLItem(ruleId); } @@ -826,6 +818,9 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ NetworkACL acl = _networkAclMgr.getNetworkACL(networkACLItemVo.getAclId()); validateNetworkAcl(acl); + Account account = CallContext.current().getCallingAccount(); + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Only Root Admins can update global ACLs."); + transferDataToNetworkAclRulePojo(updateNetworkACLItemCmd, networkACLItemVo, acl); validateNetworkACLItem(networkACLItemVo); return _networkAclMgr.updateNetworkACLItem(networkACLItemVo); @@ -912,14 +907,13 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ } @Override - @ActionEvent(eventType = EventTypes.EVENT_NETWORK_ACL_UPDATE, eventDescription = "updating network acl", async = true) + @ActionEvent(eventType = EventTypes.EVENT_NETWORK_ACL_UPDATE, eventDescription = "Updating Network ACL", async = true) public NetworkACL updateNetworkACL(UpdateNetworkACLListCmd updateNetworkACLListCmd) { Long id = updateNetworkACLListCmd.getId(); NetworkACLVO acl = _networkACLDao.findById(id); - Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); + Account account = CallContext.current().getCallingAccount(); - Account caller = CallContext.current().getCallingAccount(); - _accountMgr.checkAccess(caller, null, true, vpc); + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Must be Root Admin to update a global ACL."); String name = updateNetworkACLListCmd.getName(); if (StringUtils.isNotBlank(name)) { @@ -1149,14 +1143,59 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ long aclId = ruleBeingMoved.getAclId(); if ((nextRule != null && nextRule.getAclId() != aclId) || (previousRule != null && previousRule.getAclId() != aclId)) { - throw new InvalidParameterValueException("Cannot use ACL rules from differenting ACLs. Rule being moved."); + throw new InvalidParameterValueException("Cannot use ACL rules from differentiating ACLs. Rule being moved."); } NetworkACLVO acl = _networkACLDao.findById(aclId); - Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Re-ordering rules for a default ACL is prohibited"); + Account account = CallContext.current().getCallingAccount(); + + if (isDefaultAcl(aclId)) { + throw new InvalidParameterValueException("Default ACL rules cannot be moved."); } - Account caller = CallContext.current().getCallingAccount(); - _accountMgr.checkAccess(caller, null, true, vpc); + + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account,"Must be Root Admin to move global ACL rules."); + } + + /** + * Checks if the given ACL is a global ACL. If it is, then checks if the account is Root Admin, and throws an exception according to {@code exceptionMessage} param if it + * does not have permission. + */ + protected void checkGlobalAclPermission(Long aclVpcId, Account account, String exceptionMessage) { + if (isGlobalAcl(aclVpcId) && !Account.Type.ADMIN.equals(account.getType())) { + throw new PermissionDeniedException(exceptionMessage); + } + } + + protected void validateAclAssociatedToVpc(Long aclVpcId, Account account, String aclUuid) { + final Vpc vpc = _vpcDao.findById(aclVpcId); + if (vpc == null) { + throw new InvalidParameterValueException(String.format("Unable to find specified VPC [%s] associated with the ACL [%s].", aclVpcId, aclUuid)); + } + _accountMgr.checkAccess(account, null, true, vpc); + } + + /** + * It performs two different verifications depending on if the ACL is global or not. + *
    + *
  • If the given ACL is a Global ACL, i.e. has VPC ID = 0, then checks if the account is Root Admin, and throws an exception if it isn't. + *
  • If the given ACL is associated to a VPC, i.e. VPC ID != 0, then calls {@link #validateAclAssociatedToVpc(Long, Account, String)} and checks if the given {@code + * aclVpcId} is from a valid VPC. + *
+ */ + protected void validateGlobalAclPermissionAndAclAssociatedToVpc(NetworkACL acl, Account account, String exception){ + if (isGlobalAcl(acl.getVpcId())) { + s_logger.info(String.format("Checking if account [%s] has permission to manipulate global ACL [%s].", account, acl)); + checkGlobalAclPermission(acl.getVpcId(), account, exception); + } else { + s_logger.info(String.format("Validating ACL [%s] associated to VPC [%s] with account [%s].", acl, acl.getVpcId(), account)); + validateAclAssociatedToVpc(acl.getVpcId(), account, acl.getUuid()); + } + } + + protected boolean isGlobalAcl(Long aclVpcId) { + return aclVpcId != null && aclVpcId == 0; + } + + protected boolean isDefaultAcl(long aclId) { + return aclId == NetworkACL.DEFAULT_ALLOW || aclId == NetworkACL.DEFAULT_DENY; } } diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java index 9501c6dc06b..5321f56b66b 100644 --- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.network.vpc; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1170,8 +1171,12 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis Vpc vpc = createVpc(cmd.getZoneId(), cmd.getVpcOffering(), cmd.getEntityOwnerId(), cmd.getVpcName(), cmd.getDisplayText(), cmd.getCidr(), cmd.getNetworkDomain(), cmd.getIp4Dns1(), cmd.getIp4Dns2(), cmd.getIp6Dns1(), cmd.getIp6Dns2(), cmd.isDisplay(), cmd.getPublicMtu()); - // associate cmd.getSourceNatIP() with this vpc - allocateSourceNatIp(vpc, cmd.getSourceNatIP()); + + String sourceNatIP = cmd.getSourceNatIP(); + if (sourceNatIP != null) { + s_logger.info(String.format("Trying to allocate the specified IP [%s] as the source NAT of VPC [%s].", sourceNatIP, vpc)); + allocateSourceNatIp(vpc, sourceNatIP); + } return vpc; } @@ -2067,16 +2072,14 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis searchBuilder.and("vpcId", searchBuilder.entity().getVpcId(), Op.IN); final SearchCriteria searchCriteria = searchBuilder.create(); - searchCriteria.setParameters("vpcId", vpcId, 0); + searchCriteria.setParameters("vpcId", vpcId); final Filter filter = new Filter(NetworkACLVO.class, "id", false, null, null); final Pair, Integer> aclsCountPair = _networkAclDao.searchAndCount(searchCriteria, filter); final List acls = aclsCountPair.first(); for (final NetworkACLVO networkAcl : acls) { - if (networkAcl.getId() != NetworkACL.DEFAULT_ALLOW && networkAcl.getId() != NetworkACL.DEFAULT_DENY) { - _networkAclMgr.deleteNetworkACL(networkAcl); - } + _networkAclMgr.deleteNetworkACL(networkAcl); } VpcVO vpc = vpcDao.findById(vpcId); @@ -2252,6 +2255,8 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis final PhysicalNetwork physNetFinal = physNet; VpcGatewayVO gatewayVO = null; try { + validateVpcPrivateGatewayAclId(vpcId, aclId); + s_logger.debug("Creating Private gateway for VPC " + vpc); // 1) create private network unless it is existing and // lswitch'd @@ -2291,18 +2296,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis _dcDao.update(dc.getId(), dc); } - long networkAclId = NetworkACL.DEFAULT_DENY; - if (aclId != null) { - final NetworkACLVO aclVO = _networkAclDao.findById(aclId); - if (aclVO == null) { - throw new InvalidParameterValueException("Invalid network acl id passed "); - } - if (aclVO.getVpcId() != vpcId && !(aclId == NetworkACL.DEFAULT_DENY || aclId == NetworkACL.DEFAULT_ALLOW)) { - throw new InvalidParameterValueException("Private gateway and network acl are not in the same vpc"); - } - - networkAclId = aclId; - } + Long networkAclId = ObjectUtils.defaultIfNull(aclId, NetworkACL.DEFAULT_DENY); { // experimental block, this is a hack // set vpc id in network to null @@ -2335,6 +2329,29 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis return getVpcPrivateGateway(gatewayVO.getId()); } + /** + * This method checks if the ACL that is being used to create the private gateway is valid. First, the aclId is used to search for a {@link NetworkACLVO} object + * by calling the {@link NetworkACLDao#findById(Serializable)} method. If the object is null, an {@link InvalidParameterValueException} exception is thrown. + * Secondly, we check if the ACL and the private gateway are in the same VPC and an {@link InvalidParameterValueException} is thrown if they are not. + * + * @param vpcId Private gateway VPC ID. + * @param aclId Private gateway ACL ID. + * @throws InvalidParameterValueException + */ + protected void validateVpcPrivateGatewayAclId(long vpcId, Long aclId) { + if (aclId == null) { + return; + } + + final NetworkACLVO aclVO = _networkAclDao.findById(aclId); + if (aclVO == null) { + throw new InvalidParameterValueException("Invalid network acl id passed."); + } + if (aclVO.getVpcId() != vpcId && !(aclId == NetworkACL.DEFAULT_DENY || aclId == NetworkACL.DEFAULT_ALLOW)) { + throw new InvalidParameterValueException("Private gateway and network acl are not in the same vpc."); + } + } + private void validateVpcPrivateGatewayAssociateNetworkId(NetworkOfferingVO ntwkOff, String broadcastUri, Long associatedNetworkId, Boolean bypassVlanOverlapCheck) { // Validate vlanId and associatedNetworkId if (broadcastUri == null && associatedNetworkId == null) { @@ -3252,4 +3269,12 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis } return filteredDomainIds; } + + protected boolean isGlobalAcl(Long aclVpcId) { + return aclVpcId != null && aclVpcId == 0; + } + + protected boolean isDefaultAcl(long aclId) { + return aclId == NetworkACL.DEFAULT_ALLOW || aclId == NetworkACL.DEFAULT_DENY; + } } diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index d1697646c39..922df25a726 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -32,11 +32,13 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.alert.AlertManager; +import com.cloud.host.HostTagVO; import com.cloud.exception.StorageConflictException; import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.Volume; @@ -856,7 +858,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (s_logger.isTraceEnabled()) { s_logger.trace("Adding Host Tags for KVM host, tags: :" + hostTags); } - _hostTagsDao.persist(host.getId(), hostTags); + _hostTagsDao.persist(host.getId(), hostTags, false); } hosts.add(host); @@ -1909,7 +1911,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } - private void updateHostTags(HostVO host, Long hostId, List hostTags) { + private void updateHostTags(HostVO host, Long hostId, List hostTags, Boolean isTagARule) { List activeVMs = _vmDao.listByHostId(hostId); s_logger.warn(String.format("The following active VMs [%s] are using the host [%s]. " + "Updating the host tags will not affect them.", activeVMs, host)); @@ -1917,17 +1919,17 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (s_logger.isDebugEnabled()) { s_logger.debug("Updating Host Tags to :" + hostTags); } - _hostTagsDao.persist(hostId, new ArrayList<>(new HashSet<>(hostTags))); + _hostTagsDao.persist(hostId, new ArrayList<>(new HashSet<>(hostTags)), isTagARule); } @Override public Host updateHost(final UpdateHostCmd cmd) throws NoTransitionException { return updateHost(cmd.getId(), cmd.getName(), cmd.getOsCategoryId(), - cmd.getAllocationState(), cmd.getUrl(), cmd.getHostTags(), cmd.getAnnotation(), false); + cmd.getAllocationState(), cmd.getUrl(), cmd.getHostTags(), cmd.getIsTagARule(), cmd.getAnnotation(), false); } private Host updateHost(Long hostId, String name, Long guestOSCategoryId, String allocationState, - String url, List hostTags, String annotation, boolean isUpdateFromHostHealthCheck) throws NoTransitionException { + String url, List hostTags, Boolean isTagARule, String annotation, boolean isUpdateFromHostHealthCheck) throws NoTransitionException { // Verify that the host exists final HostVO host = _hostDao.findById(hostId); if (host == null) { @@ -1948,7 +1950,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } if (hostTags != null) { - updateHostTags(host, hostId, hostTags); + updateHostTags(host, hostId, hostTags, isTagARule); } if (url != null) { @@ -2007,7 +2009,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public Host autoUpdateHostAllocationState(Long hostId, ResourceState.Event resourceEvent) throws NoTransitionException { - return updateHost(hostId, null, null, resourceEvent.toString(), null, null, null, true); + return updateHost(hostId, null, null, resourceEvent.toString(), null, null, null, null, true); } @Override @@ -2339,7 +2341,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, final List implicitHostTags = ssCmd.getHostTags(); if (!implicitHostTags.isEmpty()) { if (hostTags == null) { - hostTags = _hostTagsDao.getHostTags(host.getId()); + hostTags = _hostTagsDao.getHostTags(host.getId()).parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); } if (hostTags != null) { implicitHostTags.removeAll(hostTags); @@ -2367,7 +2369,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, host.setManagementServerId(_nodeId); host.setStorageUrl(startup.getIqn()); host.setLastPinged(System.currentTimeMillis() >> 10); - host.setHostTags(hostTags); + host.setHostTags(hostTags, false); host.setDetails(details); if (startup.getStorageIpAddressDeux() != null) { host.setStorageIpAddressDeux(startup.getStorageIpAddressDeux()); @@ -3351,7 +3353,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public String getHostTags(final long hostId) { - final List hostTags = _hostTagsDao.getHostTags(hostId); + final List hostTags = _hostTagsDao.getHostTags(hostId).parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); if (hostTags == null) { return null; } else { diff --git a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java index d881dee137a..25b2ad53bf2 100644 --- a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java @@ -54,6 +54,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; +import com.cloud.host.HostTagVO; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -615,7 +616,7 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin if (CollectionUtils.isEmpty(vmsRunning)) { return new Pair<>(true, "OK"); } - List hostTags = hostTagsDao.getHostTags(host.getId()); + List hostTags = hostTagsDao.getHostTags(host.getId()); int successfullyCheckedVmMigrations = 0; for (VMInstanceVO runningVM : vmsRunning) { @@ -668,14 +669,14 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin /** * Check hosts tags */ - private boolean checkHostTags(List hostTags, List hostInClusterTags, String offeringTag) { + private boolean checkHostTags(List hostTags, List hostInClusterTags, String offeringTag) { if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) { return true; } else if ((CollectionUtils.isNotEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) || (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isNotEmpty(hostInClusterTags))) { return false; } else { - return hostInClusterTags.contains(offeringTag); + return hostInClusterTags.parallelStream().anyMatch(hostTagVO -> offeringTag.equals(hostTagVO.getTag())); } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 02f62033b11..bf9c6178e5d 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -43,6 +43,7 @@ import javax.crypto.spec.SecretKeySpec; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.network.dao.PublicIpQuarantineDao; import com.cloud.hypervisor.HypervisorGuru; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; @@ -208,14 +209,17 @@ import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; import org.apache.cloudstack.api.command.admin.snapshot.ListSnapshotsCmdByAdmin; import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.AddImageStoreS3CMD; +import org.apache.cloudstack.api.command.admin.storage.AddObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.FindStoragePoolsForMigrationCmd; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd; @@ -226,8 +230,13 @@ import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForM import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateCloudToUseObjectStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStorageCapabilitiesCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd; import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd; import org.apache.cloudstack.api.command.admin.systemvm.DestroySystemVmCmd; @@ -329,9 +338,12 @@ import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd; import org.apache.cloudstack.api.command.user.address.DisassociateIPAddrCmd; import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd; +import org.apache.cloudstack.api.command.user.address.ListQuarantinedIpsCmd; import org.apache.cloudstack.api.command.user.address.ReleaseIPAddrCmd; +import org.apache.cloudstack.api.command.user.address.RemoveQuarantinedIpCmd; import org.apache.cloudstack.api.command.user.address.ReserveIPAddrCmd; import org.apache.cloudstack.api.command.user.address.UpdateIPAddrCmd; +import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd; import org.apache.cloudstack.api.command.user.affinitygroup.CreateAffinityGroupCmd; import org.apache.cloudstack.api.command.user.affinitygroup.DeleteAffinityGroupCmd; import org.apache.cloudstack.api.command.user.affinitygroup.ListAffinityGroupTypesCmd; @@ -356,6 +368,10 @@ import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScalePolicyCmd import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScaleVmGroupCmd; import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScaleVmProfileCmd; import org.apache.cloudstack.api.command.user.autoscale.UpdateConditionCmd; +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.DeleteBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; import org.apache.cloudstack.api.command.user.config.ListCapabilitiesCmd; import org.apache.cloudstack.api.command.user.consoleproxy.CreateConsoleEndpointCmd; import org.apache.cloudstack.api.command.user.event.ArchiveEventsCmd; @@ -979,6 +995,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject UserDataManager userDataManager; + @Inject + private PublicIpQuarantineDao publicIpQuarantineDao; + private LockControllerListener _lockControllerListener; private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker")); private final ScheduledExecutorService _alertExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AlertChecker")); @@ -2508,10 +2527,12 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final SearchBuilder sb2 = _publicIpAddressDao.createSearchBuilder(); buildParameters(sb2, cmd, false); sb2.and("ids", sb2.entity().getId(), SearchCriteria.Op.IN); + sb2.and("quarantinedPublicIpsIdsNIN", sb2.entity().getId(), SearchCriteria.Op.NIN); SearchCriteria sc2 = sb2.create(); setParameters(sc2, cmd, vlanType, isAllocated); sc2.setParameters("ids", freeAddrIds.toArray()); + _publicIpAddressDao.buildQuarantineSearchCriteria(sc2); addrs.addAll(_publicIpAddressDao.search(sc2, searchFilter)); // Allocated + Free } Collections.sort(addrs, Comparator.comparing(IPAddressVO::getAddress)); @@ -3801,6 +3822,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(UpdateRemoteAccessVpnCmd.class); cmdList.add(UpdateVpnConnectionCmd.class); cmdList.add(UpdateVpnGatewayCmd.class); + cmdList.add(ListQuarantinedIpsCmd.class); + cmdList.add(UpdateQuarantinedIpCmd.class); + cmdList.add(RemoveQuarantinedIpCmd.class); // separated admin commands cmdList.add(ListAccountsCmdByAdmin.class); cmdList.add(ListZonesCmdByAdmin.class); @@ -3880,6 +3904,11 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(PatchSystemVMCmd.class); cmdList.add(ListGuestVlansCmd.class); cmdList.add(AssignVolumeCmd.class); + cmdList.add(ListSecondaryStorageSelectorsCmd.class); + cmdList.add(CreateSecondaryStorageSelectorCmd.class); + cmdList.add(UpdateSecondaryStorageSelectorCmd.class); + cmdList.add(RemoveSecondaryStorageSelectorCmd.class); + // Out-of-band management APIs for admins cmdList.add(EnableOutOfBandManagementForHostCmd.class); @@ -3899,6 +3928,16 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(DeleteUserDataCmd.class); cmdList.add(ListUserDataCmd.class); cmdList.add(LinkUserDataToTemplateCmd.class); + + //object store APIs + cmdList.add(AddObjectStoragePoolCmd.class); + cmdList.add(ListObjectStoragePoolsCmd.class); + cmdList.add(UpdateObjectStoragePoolCmd.class); + cmdList.add(DeleteObjectStoragePoolCmd.class); + cmdList.add(CreateBucketCmd.class); + cmdList.add(UpdateBucketCmd.class); + cmdList.add(DeleteBucketCmd.class); + cmdList.add(ListBucketsCmd.class); return cmdList; } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 82a6c159565..ec432c96782 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -54,10 +55,15 @@ import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaint import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -95,6 +101,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.SyncVolumePathAnswer; @@ -104,6 +114,9 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; @@ -115,8 +128,11 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -192,6 +208,7 @@ import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume.Type; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.StoragePoolHostDao; @@ -237,6 +254,9 @@ import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.collect.Sets; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; + @Component public class StorageManagerImpl extends ManagerBase implements StorageManager, ClusterManagerListener, Configurable { @@ -348,8 +368,19 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject private AnnotationDao annotationDao; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + @Inject protected UserVmManager userVmManager; + @Inject + protected ObjectStoreDao _objectStoreDao; + + @Inject + protected ObjectStoreDetailsDao _objectStoreDetailsDao; + + @Inject + protected BucketDao _bucketDao; protected List _discoverers; public List getDiscoverers() { @@ -653,6 +684,41 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return true; } + private DataStore createLocalStorage(Map poolInfos) throws ConnectionException{ + Object existingUuid = poolInfos.get("uuid"); + if( existingUuid == null ){ + poolInfos.put("uuid", UUID.randomUUID().toString()); + } + String hostAddress = poolInfos.get("host").toString(); + Host host = _hostDao.findByName(hostAddress); + + if( host == null ) { + host = _hostDao.findByIp(hostAddress); + + if( host == null ) { + host = _hostDao.findByPublicIp(hostAddress); + + if( host == null ) { + throw new InvalidParameterValueException(String.format("host %s not found",hostAddress)); + } + } + } + + long capacityBytes = poolInfos.get("capacityBytes") != null ? Long.parseLong(poolInfos.get("capacityBytes").toString()) : 0; + + StoragePoolInfo pInfo = new StoragePoolInfo(poolInfos.get("uuid").toString(), + host.getPrivateIpAddress(), + poolInfos.get("hostPath").toString(), + poolInfos.get("hostPath").toString(), + StoragePoolType.Filesystem, + capacityBytes, + 0, + (Map)poolInfos.get("details"), + poolInfos.get("name").toString()); + + return createLocalStorage(host, pInfo); + } + @DB @Override public DataStore createLocalStorage(Host host, StoragePoolInfo pInfo) throws ConnectionException { @@ -698,17 +764,19 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle(); if (pool == null) { Map params = new HashMap(); - String name = createLocalStoragePoolName(host, pInfo); + String name = pInfo.getName() != null ? pInfo.getName() : createLocalStoragePoolName(host, pInfo); params.put("zoneId", host.getDataCenterId()); params.put("clusterId", host.getClusterId()); params.put("podId", host.getPodId()); params.put("hypervisorType", host.getHypervisorType()); - params.put("url", pInfo.getPoolType().toString() + "://" + pInfo.getHost() + "/" + pInfo.getHostPath()); params.put("name", name); params.put("localStorage", true); params.put("details", pInfo.getDetails()); params.put("uuid", pInfo.getUuid()); params.put("providerName", provider.getName()); + params.put("scheme", pInfo.getPoolType().toString()); + params.put("host", pInfo.getHost()); + params.put("hostPath", pInfo.getHostPath()); store = lifeCycle.initialize(params); } else { @@ -740,6 +808,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Override public PrimaryDataStoreInfo createPool(CreateStoragePoolCmd cmd) throws ResourceInUseException, IllegalArgumentException, UnknownHostException, ResourceUnavailableException { String providerName = cmd.getStorageProviderName(); + Map uriParams = extractUriParamsAsMap(cmd.getUrl()); DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName); if (storeProvider == null) { @@ -753,7 +822,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C Long podId = cmd.getPodId(); Long zoneId = cmd.getZoneId(); - ScopeType scopeType = ScopeType.CLUSTER; + ScopeType scopeType = uriParams.get("scheme").toString().equals("file") ? ScopeType.HOST : ScopeType.CLUSTER; String scope = cmd.getScope(); if (scope != null) { try { @@ -813,17 +882,23 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C params.put("hypervisorType", hypervisorType); params.put("url", cmd.getUrl()); params.put("tags", cmd.getTags()); + params.put("isTagARule", cmd.isTagARule()); params.put("name", cmd.getStoragePoolName()); params.put("details", details); params.put("providerName", storeProvider.getName()); params.put("managed", cmd.isManaged()); params.put("capacityBytes", cmd.getCapacityBytes()); params.put("capacityIops", cmd.getCapacityIops()); + params.putAll(uriParams); DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle(); DataStore store = null; try { - store = lifeCycle.initialize(params); + if (params.get("scheme").toString().equals("file")) { + store = createLocalStorage(params); + } else { + store = lifeCycle.initialize(params); + } if (scopeType == ScopeType.CLUSTER) { ClusterScope clusterScope = new ClusterScope(clusterId, podId, zoneId); lifeCycle.attachCluster(store, clusterScope); @@ -848,6 +923,62 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Primary); } + private Map extractUriParamsAsMap(String url){ + Map uriParams = new HashMap<>(); + UriUtils.UriInfo uriInfo = UriUtils.getUriInfo(url); + + String scheme = uriInfo.getScheme(); + String storageHost = uriInfo.getStorageHost(); + String storagePath = uriInfo.getStoragePath(); + try { + if (scheme == null) { + throw new InvalidParameterValueException("scheme is null " + url + ", add nfs:// (or cifs://) as a prefix"); + } else if (scheme.equalsIgnoreCase("nfs")) { + if (storageHost == null || storagePath == null || storageHost.trim().isEmpty() || storagePath.trim().isEmpty()) { + throw new InvalidParameterValueException("host or path is null, should be nfs://hostname/path"); + } + } else if (scheme.equalsIgnoreCase("cifs")) { + // Don't validate against a URI encoded URI. + URI cifsUri = new URI(url); + String warnMsg = UriUtils.getCifsUriParametersProblems(cifsUri); + if (warnMsg != null) { + throw new InvalidParameterValueException(warnMsg); + } + } else if (scheme.equalsIgnoreCase("sharedMountPoint")) { + if (storagePath == null) { + throw new InvalidParameterValueException("host or path is null, should be sharedmountpoint://localhost/path"); + } + } else if (scheme.equalsIgnoreCase("rbd")) { + if (storagePath == null) { + throw new InvalidParameterValueException("host or path is null, should be rbd://hostname/pool"); + } + } else if (scheme.equalsIgnoreCase("gluster")) { + if (storageHost == null || storagePath == null || storageHost.trim().isEmpty() || storagePath.trim().isEmpty()) { + throw new InvalidParameterValueException("host or path is null, should be gluster://hostname/volume"); + } + } + } catch (URISyntaxException e) { + throw new InvalidParameterValueException(url + " is not a valid uri"); + } + + String hostPath = null; + try { + hostPath = URLDecoder.decode(storagePath, "UTF-8"); + } catch (UnsupportedEncodingException e) { + s_logger.error("[ignored] we are on a platform not supporting \"UTF-8\"!?!", e); + } + if (hostPath == null) { // if decoding fails, use getPath() anyway + hostPath = storagePath; + } + + uriParams.put("scheme", scheme); + uriParams.put("host", storageHost); + uriParams.put("hostPath", hostPath); + uriParams.put("userInfo", uriInfo.getUserInfo()); + uriParams.put("port", uriInfo.getPort() + ""); + return uriParams; + } + private Map extractApiParamAsMap(Map ds) { Map details = new HashMap(); if (ds != null) { @@ -915,10 +1046,10 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (pool.getPoolType() == StoragePoolType.DatastoreCluster) { List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); for (StoragePoolVO childPool : childStoragePools) { - _storagePoolTagsDao.persist(childPool.getId(), storagePoolTags); + _storagePoolTagsDao.persist(childPool.getId(), storagePoolTags, cmd.isTagARule()); } } - _storagePoolTagsDao.persist(pool.getId(), storagePoolTags); + _storagePoolTagsDao.persist(pool.getId(), storagePoolTags, cmd.isTagARule()); } Long updatedCapacityBytes = null; @@ -1887,9 +2018,68 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return (PrimaryDataStoreInfo) _dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); } + + @Override + public Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd) { + String name = cmd.getName(); + String description = cmd.getDescription(); + long zoneId = cmd.getZoneId(); + String heuristicRule = cmd.getHeuristicRule(); + String type = cmd.getType(); + HeuristicType formattedType = EnumUtils.getEnumIgnoreCase(HeuristicType.class, type); + + if (formattedType == null) { + throw new IllegalArgumentException(String.format("The given heuristic type [%s] is not valid for creating a new secondary storage selector." + + " The valid options are %s.", type, Arrays.asList(HeuristicType.values()))); + } + + HeuristicVO heuristic = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, formattedType); + + if (heuristic != null) { + DataCenterVO dataCenter = _dcDao.findById(zoneId); + throw new CloudRuntimeException(String.format("There is already a heuristic rule in the specified %s with the type [%s].", + dataCenter, type)); + } + + validateHeuristicRule(heuristicRule); + + HeuristicVO heuristicVO = new HeuristicVO(name, description, zoneId, formattedType.toString(), heuristicRule); + return secondaryStorageHeuristicDao.persist(heuristicVO); + } + + @Override + public Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd) { + long heuristicId = cmd.getId(); + String heuristicRule = cmd.getHeuristicRule(); + + HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId); + validateHeuristicRule(heuristicRule); + heuristicVO.setHeuristicRule(heuristicRule); + + return secondaryStorageHeuristicDao.persist(heuristicVO); + } + + @Override + public void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd) { + Long heuristicId = cmd.getId(); + HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId); + + if (heuristicVO != null) { + secondaryStorageHeuristicDao.remove(heuristicId); + } else { + throw new CloudRuntimeException("Unable to find an active heuristic with the specified UUID."); + } + } + + protected void validateHeuristicRule(String heuristicRule) { + if (StringUtils.isBlank(heuristicRule)) { + throw new IllegalArgumentException("Unable to create a new secondary storage selector as the given heuristic rule is blank."); + } + } + public void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List childDatastoreAnswerList, long hostId) { StoragePoolVO datastoreClusterPool = _storagePoolDao.findById(datastoreClusterPoolId); - List storageTags = _storagePoolTagsDao.getStoragePoolTags(datastoreClusterPoolId); + List storageTags = _storagePoolTagsDao.findStoragePoolTags(datastoreClusterPoolId); List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(datastoreClusterPoolId); Set childDatastoreUUIDs = new HashSet<>(); for (StoragePoolVO childDatastore : childDatastores) { @@ -1917,18 +2107,18 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C dataStoreVO.setParent(datastoreClusterPoolId); _storagePoolDao.update(dataStoreVO.getId(), dataStoreVO); if (CollectionUtils.isNotEmpty(storageTags)) { - storageTags.addAll(_storagePoolTagsDao.getStoragePoolTags(dataStoreVO.getId())); + storageTags.addAll(_storagePoolTagsDao.findStoragePoolTags(dataStoreVO.getId())); } else { - storageTags = _storagePoolTagsDao.getStoragePoolTags(dataStoreVO.getId()); + storageTags = _storagePoolTagsDao.findStoragePoolTags(dataStoreVO.getId()); } if (CollectionUtils.isNotEmpty(storageTags)) { - Set set = new LinkedHashSet<>(storageTags); + Set set = new LinkedHashSet<>(storageTags); storageTags.clear(); storageTags.addAll(set); if (s_logger.isDebugEnabled()) { s_logger.debug("Updating Storage Pool Tags to :" + storageTags); } - _storagePoolTagsDao.persist(dataStoreVO.getId(), storageTags); + _storagePoolTagsDao.persist(storageTags); } } else { // This is to find datastores which are removed from datastore cluster. @@ -1936,7 +2126,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C childDatastoreUUIDs.remove(dataStoreVO.getUuid()); } } else { - dataStoreVO = createChildDatastoreVO(datastoreClusterPool, childDataStoreAnswer); + dataStoreVO = createChildDatastoreVO(datastoreClusterPool, childDataStoreAnswer, storageTags); } updateStoragePoolHostVOAndBytes(dataStoreVO, hostId, childDataStoreAnswer); } @@ -1977,9 +2167,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } - private StoragePoolVO createChildDatastoreVO(StoragePoolVO datastoreClusterPool, ModifyStoragePoolAnswer childDataStoreAnswer) { + private StoragePoolVO createChildDatastoreVO(StoragePoolVO datastoreClusterPool, ModifyStoragePoolAnswer childDataStoreAnswer, List storagePoolTagVOList) { StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); - List storageTags = _storagePoolTagsDao.getStoragePoolTags(datastoreClusterPool.getId()); StoragePoolVO dataStoreVO = new StoragePoolVO(); dataStoreVO.setStorageProviderName(datastoreClusterPool.getStorageProviderName()); @@ -2006,7 +2195,15 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { details.put("pool_type", childDataStoreAnswer.getPoolType()); } - _storagePoolDao.persist(dataStoreVO, details, storageTags); + + List storagePoolTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(storagePoolTagVOList)) { + storagePoolTags = storagePoolTagVOList.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()); + isTagARule = storagePoolTagVOList.get(0).isTagARule(); + } + + _storagePoolDao.persist(dataStoreVO, details, storagePoolTags, isTagARule); return dataStoreVO; } @@ -3523,4 +3720,124 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C volumeTO.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering)); } + @Override + @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_CREATE, eventDescription = "creating object storage") + public ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) + throws IllegalArgumentException, InvalidParameterValueException { + DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName); + + if (storeProvider == null) { + throw new InvalidParameterValueException("can't find object store provider: " + providerName); + } + + // Check Unique object store name + ObjectStoreVO objectStore = _objectStoreDao.findByName(name); + if (objectStore != null) { + throw new InvalidParameterValueException("The object store with name " + name + " already exists, try creating with another name"); + } + + try { + // Check URL + UriUtils.validateUrl(url); + } catch (final Exception e) { + throw new InvalidParameterValueException(url + " is not a valid URL"); + } + + // Check Unique object store url + ObjectStoreVO objectStoreUrl = _objectStoreDao.findByUrl(url); + if (objectStoreUrl != null) { + throw new InvalidParameterValueException("The object store with url " + url + " already exists"); + } + + + Map params = new HashMap<>(); + params.put("url", url); + params.put("name", name); + params.put("providerName", storeProvider.getName()); + params.put("role", DataStoreRole.Object); + params.put("details", details); + + DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle(); + + DataStore store; + try { + store = lifeCycle.initialize(params); + } catch (Exception e) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Failed to add object store: " + e.getMessage(), e); + } + throw new CloudRuntimeException("Failed to add object store: " + e.getMessage(), e); + } + + return (ObjectStore)_dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Object); + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_DELETE, eventDescription = "deleting object storage") + public boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd) { + final long storeId = cmd.getId(); + // Verify that object store exists + ObjectStoreVO store = _objectStoreDao.findById(storeId); + if (store == null) { + throw new InvalidParameterValueException("Object store with id " + storeId + " doesn't exist"); + } + + // Verify that there are no buckets in the store + List buckets = _bucketDao.listByObjectStoreId(storeId); + if(buckets != null && buckets.size() > 0) { + throw new InvalidParameterValueException("Cannot delete object store with buckets"); + } + + // ready to delete + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + _objectStoreDetailsDao.deleteDetails(storeId); + _objectStoreDao.remove(storeId); + } + }); + s_logger.debug("Successfully deleted object store with Id: "+storeId); + return true; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_UPDATE, eventDescription = "update object storage") + public ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd) { + + // Input validation + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(id); + if (objectStoreVO == null) { + throw new IllegalArgumentException("Unable to find object store with ID: " + id); + } + + if(cmd.getUrl() != null ) { + String url = cmd.getUrl(); + try { + // Check URL + UriUtils.validateUrl(url); + } catch (final Exception e) { + throw new InvalidParameterValueException(url + " is not a valid URL"); + } + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + String oldUrl = objectStoreVO.getUrl(); + objectStoreVO.setUrl(url); + _objectStoreDao.update(id, objectStoreVO); + //Update URL and check access + try { + objectStore.listBuckets(); + } catch (Exception e) { + //Revert to old URL on failure + objectStoreVO.setUrl(oldUrl); + _objectStoreDao.update(id, objectStoreVO); + throw new IllegalArgumentException("Unable to access Object Storage with URL: " + cmd.getUrl()); + } + } + + if(cmd.getName() != null ) { + objectStoreVO.setName(cmd.getName()); + } + _objectStoreDao.update(id, objectStoreVO); + s_logger.debug("Successfully updated object store with Id: "+id); + return objectStoreVO; + } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 7bfe8a0818b..69b5f984081 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -106,6 +106,7 @@ import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.commons.collections.CollectionUtils; @@ -346,6 +347,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject protected StoragePoolDetailsDao storagePoolDetailsDao; + protected Gson _gson; private static final List SupportedHypervisorsForVolResize = Arrays.asList(HypervisorType.KVM, HypervisorType.XenServer, @@ -375,6 +377,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic "Time (in milliseconds) to wait before assuming the VM was unable to detach a volume after the hypervisor sends the detach command.", true); + public static ConfigKey storageTagRuleExecutionTimeout = new ConfigKey<>("Advanced", Long.class, "storage.tag.rule.execution.timeout", "2000", "The maximum runtime," + + " in milliseconds, to execute a storage tag rule; if it is reached, a timeout will happen.", true); + private final StateMachine2 _volStateMachine; private static final Set STATES_VOLUME_CANNOT_BE_DESTROYED = new HashSet<>(Arrays.asList(Volume.State.Destroy, Volume.State.Expunging, Volume.State.Expunged, Volume.State.Allocated)); @@ -404,7 +409,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic String format = sanitizeFormat(cmd.getFormat()); Long diskOfferingId = cmd.getDiskOfferingId(); String imageStoreUuid = cmd.getImageStoreUuid(); - DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId); validateVolume(caller, ownerId, zoneId, volumeName, url, format, diskOfferingId); @@ -414,6 +418,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic RegisterVolumePayload payload = new RegisterVolumePayload(cmd.getUrl(), cmd.getChecksum(), format); vol.addPayload(payload); + DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume); volService.registerVolume(vol, store); return volume; @@ -446,7 +451,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic String format = sanitizeFormat(cmd.getFormat()); final Long diskOfferingId = cmd.getDiskOfferingId(); String imageStoreUuid = cmd.getImageStoreUuid(); - final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId); validateVolume(caller, ownerId, zoneId, volumeName, null, format, diskOfferingId); @@ -456,6 +460,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic VolumeVO volume = persistVolume(owner, zoneId, volumeName, null, format, diskOfferingId, Volume.State.NotUploaded); + final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume); + VolumeInfo vol = volFactory.getVolume(volume.getId()); RegisterVolumePayload payload = new RegisterVolumePayload(null, cmd.getChecksum(), format); @@ -650,6 +656,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic //url can be null incase of postupload if (url != null) { _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + volume.setSize(UriUtils.getRemoteSize(url)); } return volume; @@ -3266,7 +3273,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } if (!doesTargetStorageSupportDiskOffering(destPool, newDiskOffering)) { throw new InvalidParameterValueException(String.format("Migration failed: target pool [%s, tags:%s] has no matching tags for volume [%s, uuid:%s, tags:%s]", destPool.getName(), - getStoragePoolTags(destPool), volume.getName(), volume.getUuid(), newDiskOffering.getTags())); + storagePoolTagsDao.getStoragePoolTags(destPool.getId()), volume.getName(), volume.getUuid(), newDiskOffering.getTags())); } if (volume.getVolumeType().equals(Volume.Type.ROOT)) { VMInstanceVO vm = null; @@ -3329,17 +3336,31 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Override public boolean doesTargetStorageSupportDiskOffering(StoragePool destPool, String diskOfferingTags) { - if (StringUtils.isBlank(diskOfferingTags)) { + Pair, Boolean> storagePoolTags = getStoragePoolTags(destPool); + if ((storagePoolTags == null || !storagePoolTags.second()) && org.apache.commons.lang.StringUtils.isBlank(diskOfferingTags)) { + if (storagePoolTags == null) { + s_logger.debug(String.format("Destination storage pool [%s] does not have any tags, and so does the disk offering. Therefore, they are compatible", destPool.getUuid())); + } else { + s_logger.debug("Destination storage pool has tags [%s], and the disk offering has no tags. Therefore, they are compatible."); + } return true; } - String storagePoolTags = getStoragePoolTags(destPool); - if (StringUtils.isBlank(storagePoolTags)) { + if (storagePoolTags == null || CollectionUtils.isEmpty(storagePoolTags.first())) { + s_logger.debug(String.format("Destination storage pool [%s] has no tags, while disk offering has tags [%s]. Therefore, they are not compatible", destPool.getUuid(), + diskOfferingTags)); return false; } - String[] storageTagsAsStringArray = StringUtils.split(storagePoolTags, ","); - String[] newDiskOfferingTagsAsStringArray = StringUtils.split(diskOfferingTags, ","); + List storageTagsList = storagePoolTags.first(); + String[] newDiskOfferingTagsAsStringArray = org.apache.commons.lang.StringUtils.split(diskOfferingTags, ","); - return CollectionUtils.isSubCollection(Arrays.asList(newDiskOfferingTagsAsStringArray), Arrays.asList(storageTagsAsStringArray)); + boolean result; + if (storagePoolTags.second()) { + result = TagAsRuleHelper.interpretTagAsRule(storageTagsList.get(0), diskOfferingTags, storageTagRuleExecutionTimeout.value()); + } else { + result = CollectionUtils.isSubCollection(Arrays.asList(newDiskOfferingTagsAsStringArray), storageTagsList); + } + s_logger.debug(String.format("Destination storage pool [%s] accepts tags [%s]? %s", destPool.getUuid(), diskOfferingTags, result)); + return result; } public static boolean doesNewDiskOfferingHasTagsAsOldDiskOffering(DiskOfferingVO oldDO, DiskOfferingVO newDO) { @@ -3355,14 +3376,17 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } /** - * Retrieves the storage pool tags as a {@link String}. If the storage pool does not have tags we return a null value. + * Returns a {@link Pair}, where the first value is the list of the StoragePool tags, and the second value is whether the returned tags are to be interpreted as a rule, + * or a normal list of tags. + *

+ * If the storage pool does not have tags we return a null value. */ - protected String getStoragePoolTags(StoragePool destPool) { - List destPoolTags = storagePoolTagsDao.getStoragePoolTags(destPool.getId()); + protected Pair, Boolean> getStoragePoolTags(StoragePool destPool) { + List destPoolTags = storagePoolTagsDao.findStoragePoolTags(destPool.getId()); if (CollectionUtils.isEmpty(destPoolTags)) { return null; } - return StringUtils.join(destPoolTags, ","); + return new Pair<>(destPoolTags.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()), destPoolTags.get(0).isTagARule()); } private Volume orchestrateMigrateVolume(VolumeVO volume, StoragePool destPool, boolean liveMigrateVolume, DiskOfferingVO newDiskOffering) { diff --git a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java index 63ae60411ab..d6101046383 100644 --- a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java +++ b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java @@ -101,7 +101,7 @@ public class StoragePoolMonitor implements Listener { scCmd.getHypervisorType() == HypervisorType.Ovm || scCmd.getHypervisorType() == HypervisorType.Hyperv || scCmd.getHypervisorType() == HypervisorType.LXC || scCmd.getHypervisorType() == HypervisorType.Ovm3) { List pools = _poolDao.listBy(host.getDataCenterId(), host.getPodId(), host.getClusterId(), ScopeType.CLUSTER); - List zoneStoragePoolsByTags = _poolDao.findZoneWideStoragePoolsByTags(host.getDataCenterId(), null); + List zoneStoragePoolsByTags = _poolDao.findZoneWideStoragePoolsByTags(host.getDataCenterId(), null, false); List zoneStoragePoolsByHypervisor = _poolDao.findZoneWideStoragePoolsByHypervisor(host.getDataCenterId(), scCmd.getHypervisorType()); zoneStoragePoolsByTags.retainAll(zoneStoragePoolsByHypervisor); pools.addAll(zoneStoragePoolsByTags); diff --git a/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java b/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java index 928e54227a9..c02f4136863 100644 --- a/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java +++ b/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java @@ -81,6 +81,7 @@ public class ResourceManagerUtilImpl implements ResourceManagerUtil { s_typeMap.put(ResourceTag.ResourceObjectType.UserVm, UserVmVO.class); s_typeMap.put(ResourceTag.ResourceObjectType.Volume, VolumeVO.class); s_typeMap.put(ResourceTag.ResourceObjectType.Template, VMTemplateVO.class); + s_typeMap.put(ResourceTag.ResourceObjectType.VnfTemplate, VMTemplateVO.class); s_typeMap.put(ResourceTag.ResourceObjectType.ISO, VMTemplateVO.class); s_typeMap.put(ResourceTag.ResourceObjectType.Snapshot, SnapshotVO.class); s_typeMap.put(ResourceTag.ResourceObjectType.Network, NetworkVO.class); diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index aed6a83f0a3..b886f0868f6 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -29,6 +29,8 @@ import java.util.stream.Collectors; import javax.inject.Inject; import com.cloud.domain.Domain; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; import org.apache.cloudstack.annotation.AnnotationService; @@ -57,9 +59,11 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.collections.CollectionUtils; @@ -105,7 +109,7 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; public class HypervisorTemplateAdapter extends TemplateAdapterBase { - private final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); + protected final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); @Inject DownloadMonitor _downloadMonitor; @Inject @@ -142,6 +146,10 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject private AnnotationDao annotationDao; + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + @Inject + VMInstanceDao _vmInstanceDao; @Override public String getName() { @@ -267,17 +275,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } if (!profile.isDirectDownload()) { - List zones = profile.getZoneIdList(); - - //zones is null when this template is to be registered to all zones - if (zones == null){ - createTemplateWithinZone(null, profile, template); - } - else { - for (Long zId : zones) { - createTemplateWithinZone(zId, profile, template); - } - } + createTemplateWithinZones(profile, template); } else { //KVM direct download templates bypassing Secondary Storage persistDirectDownloadTemplate(template.getId(), profile.getSize()); @@ -287,46 +285,67 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { return template; } - private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTemplateVO template) { - // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zId)); - if (imageStores == null || imageStores.size() == 0) { - throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); + /** + * For each zone ID in {@link TemplateProfile#zoneIdList}, verifies if there is active heuristic rules for allocating template and returns the + * {@link DataStore} returned by the heuristic rule. If there is not an active heuristic rule, then allocate it to a random {@link DataStore}, if the ISO/template is private + * or allocate it to all {@link DataStore} in the zone, if it is public. + * @param profile + * @param template + */ + protected void createTemplateWithinZones(TemplateProfile profile, VMTemplateVO template) { + List zonesIds = profile.getZoneIdList(); + + if (zonesIds == null) { + zonesIds = _dcDao.listAllZones().stream().map(DataCenterVO::getId).collect(Collectors.toList()); } + List imageStores = getImageStoresThrowsExceptionIfNotFound(zonesIds, profile); + + for (long zoneId : zonesIds) { + DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); + + if (imageStore == null) { + standardImageStoreAllocation(imageStores, template); + } else { + validateSecondaryStorageAndCreateTemplate(List.of(imageStore), template, null); + } + } + } + + protected List getImageStoresThrowsExceptionIfNotFound(List zonesIds, TemplateProfile profile) { + List imageStores = storeMgr.getImageStoresByZoneIds(zonesIds.toArray(new Long[0])); + if (imageStores == null || imageStores.size() == 0) { + throw new CloudRuntimeException(String.format("Unable to find image store to download the template [%s].", profile.getTemplate())); + } + return imageStores; + } + + protected DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) { + HeuristicType heuristicType; + if (ImageFormat.ISO.equals(template.getFormat())) { + heuristicType = HeuristicType.ISO; + } else { + heuristicType = HeuristicType.TEMPLATE; + } + return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template); + } + + protected void standardImageStoreAllocation(List imageStores, VMTemplateVO template) { Set zoneSet = new HashSet(); Collections.shuffle(imageStores); - // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. + validateSecondaryStorageAndCreateTemplate(imageStores, template, zoneSet); + } + + protected void validateSecondaryStorageAndCreateTemplate(List imageStores, VMTemplateVO template, Set zoneSet) { for (DataStore imageStore : imageStores) { - // skip data stores for a disabled zone Long zoneId = imageStore.getScope().getScopeId(); - if (zoneId != null) { - DataCenterVO zone = _dcDao.findById(zoneId); - if (zone == null) { - s_logger.warn("Unable to find zone by id " + zoneId + ", so skip downloading template to its image store " + imageStore.getId()); - continue; - } - // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - s_logger.info("Zone " + zoneId + " is disabled. Skip downloading template to its image store " + imageStore.getId()); - continue; - } - - // Check if image store has enough capacity for template - if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { - s_logger.info("Image store doesn't have enough capacity. Skip downloading template to this image store " + imageStore.getId()); - continue; - } - // We want to download private template to one of the image store in a zone - if (isPrivateTemplate(template) && zoneSet.contains(zoneId)) { - continue; - } else { - zoneSet.add(zoneId); - } + if (!isZoneAndImageStoreAvailable(imageStore, zoneId, zoneSet, isPrivateTemplate(template))) { + continue; } + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); - CreateTemplateContext context = new CreateTemplateContext(null, tmpl); + CreateTemplateContext context = new CreateTemplateContext<>(null, tmpl); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null)); caller.setContext(context); @@ -334,6 +353,44 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } } + protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId, Set zoneSet, boolean isTemplatePrivate) { + if (zoneId == null) { + s_logger.warn(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", imageStore)); + return false; + } + + DataCenterVO zone = _dcDao.findById(zoneId); + if (zone == null) { + s_logger.warn(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].", zoneId, imageStore.getId())); + return false; + } + + if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + s_logger.info(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, imageStore.getId())); + return false; + } + + if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + s_logger.info(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].", imageStore.getId())); + return false; + } + + if (zoneSet == null) { + s_logger.info(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage of zone [%s].", zone)); + return true; + } + + if (isTemplatePrivate && zoneSet.contains(zoneId)) { + s_logger.info(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; therefore, image store [%s] will be skipped.", + zone, imageStore)); + return false; + } + + s_logger.info(String.format("Private template will be allocated in image store [%s] in zone [%s].", imageStore, zone)); + zoneSet.add(zoneId); + return true; + } + @Override public List createTemplateForPostUpload(final TemplateProfile profile) { // persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync. @@ -348,80 +405,27 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate()); } - if (profile.getZoneIdList() != null && profile.getZoneIdList().size() > 1) - throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time"); + List zoneIdList = profile.getZoneIdList(); - Long zoneId = null; - if (profile.getZoneIdList() != null) - zoneId = profile.getZoneIdList().get(0); - - // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); - if (imageStores == null || imageStores.size() == 0) { - throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); + if (zoneIdList == null) { + throw new CloudRuntimeException("Zone ID is null, cannot upload ISO/template."); } + if (zoneIdList.size() > 1) + throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time."); + + Long zoneId = zoneIdList.get(0); + DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); List payloads = new LinkedList<>(); - Set zoneSet = new HashSet(); - Collections.shuffle(imageStores); // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. - for (DataStore imageStore : imageStores) { - // skip data stores for a disabled zone - Long zoneId_is = imageStore.getScope().getScopeId(); - if (zoneId != null) { - DataCenterVO zone = _dcDao.findById(zoneId_is); - if (zone == null) { - s_logger.warn("Unable to find zone by id " + zoneId_is + - ", so skip downloading template to its image store " + imageStore.getId()); - continue; - } - // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - s_logger.info("Zone " + zoneId_is + - " is disabled, so skip downloading template to its image store " + imageStore.getId()); - continue; - } + if (imageStore == null) { + List imageStores = getImageStoresThrowsExceptionIfNotFound(List.of(zoneId), profile); + postUploadAllocation(imageStores, template, payloads); + } else { + postUploadAllocation(List.of(imageStore), template, payloads); - // We want to download private template to one of the image store in a zone - if (isPrivateTemplate(template) && zoneSet.contains(zoneId_is)) { - continue; - } else { - zoneSet.add(zoneId_is); - } - - } - - TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); - //imageService.createTemplateAsync(tmpl, imageStore, caller); - - // persist template_store_ref entry - DataObject templateOnStore = imageStore.create(tmpl); - // update template_store_ref and template state - - EndPoint ep = _epSelector.select(templateOnStore); - if (ep == null) { - String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName(); - s_logger.warn(errMsg); - throw new CloudRuntimeException(errMsg); - } - - TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl - .getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), - templateOnStore.getDataStore().getRole().toString()); - //using the existing max template size configuration - payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); - - Long accountId = template.getAccountId(); - Account account = _accountDao.findById(accountId); - Domain domain = _domainDao.findById(account.getDomainId()); - - payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); - payload.setAccountId(accountId); - payload.setRemoteEndPoint(ep.getPublicAddr()); - payload.setRequiresHvm(template.requiresHvm()); - payload.setDescription(template.getDisplayText()); - payloads.add(payload); } + if(payloads.isEmpty()) { throw new CloudRuntimeException("unable to find zone or an image store with enough capacity"); } @@ -431,6 +435,52 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { }); } + /** + * If the template/ISO is marked as private, then it is allocated to a random secondary storage; otherwise, allocates to every storage pool in every zone given by the + * {@link TemplateProfile#zoneIdList}. + */ + private void postUploadAllocation(List imageStores, VMTemplateVO template, List payloads) { + Set zoneSet = new HashSet(); + Collections.shuffle(imageStores); + for (DataStore imageStore : imageStores) { + Long zoneId_is = imageStore.getScope().getScopeId(); + + if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is, zoneSet, isPrivateTemplate(template))) { + continue; + } + + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); + + // persist template_store_ref entry + DataObject templateOnStore = imageStore.create(tmpl); + + // update template_store_ref and template state + EndPoint ep = _epSelector.select(templateOnStore); + if (ep == null) { + String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName(); + s_logger.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } + + TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl + .getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), + templateOnStore.getDataStore().getRole().toString()); + //using the existing max template size configuration + payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); + + Long accountId = template.getAccountId(); + Account account = _accountDao.findById(accountId); + Domain domain = _domainDao.findById(account.getDomainId()); + + payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); + payload.setAccountId(accountId); + payload.setRemoteEndPoint(ep.getPublicAddr()); + payload.setRequiresHvm(template.requiresHvm()); + payload.setDescription(template.getDisplayText()); + payloads.add(payload); + } + } + private boolean isPrivateTemplate(VMTemplateVO template){ // if public OR featured OR system template @@ -666,11 +716,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { Pair, Long> tmplt = new Pair, Long>(VirtualMachineTemplate.class, template.getId()); _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, tmplt); - // Remove template details - templateDetailsDao.removeDetails(template.getId()); - - // Remove deploy-as-is details (if any) - templateDeployAsIsDetailsDao.removeDetails(template.getId()); + checkAndRemoveTemplateDetails(template); // Remove comments (if any) AnnotationService.EntityType entityType = template.getFormat().equals(ImageFormat.ISO) ? @@ -681,6 +727,23 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { return success; } + /** + * removes details of the template and + * if the template is registered as deploy as is, + * then it also deletes the details related to deploy as is only if there are no VMs using the template + * @param template + */ + void checkAndRemoveTemplateDetails(VMTemplateVO template) { + templateDetailsDao.removeDetails(template.getId()); + + if (template.isDeployAsIs()) { + List vmInstanceVOList = _vmInstanceDao.listNonExpungedByTemplate(template.getId()); + if (CollectionUtils.isEmpty(vmInstanceVOList)) { + templateDeployAsIsDetailsDao.removeDetails(template.getId()); + } + } + } + @Override public TemplateProfile prepareDelete(DeleteTemplateCmd cmd) { TemplateProfile profile = super.prepareDelete(cmd); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index f5d385b4e10..2ed42087020 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -86,6 +86,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CommandResult; @@ -98,6 +100,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.storage.template.VnfTemplateUtils; @@ -311,6 +314,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject VnfTemplateManager vnfTemplateManager; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; if (type == HypervisorType.BareMetal) { @@ -451,17 +460,21 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } @Override - public DataStore getImageStore(String storeUuid, Long zoneId) { + public DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume) { DataStore imageStore = null; if (storeUuid != null) { imageStore = _dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image); } else { - imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.VOLUME, volume); if (imageStore == null) { - throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); + imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); } } + if (imageStore == null) { + throw new CloudRuntimeException(String.format("Cannot find an image store for zone [%s].", zoneId)); + } + return imageStore; } diff --git a/server/src/main/java/com/cloud/test/TestAppender.java b/server/src/main/java/com/cloud/test/TestAppender.java index d8e97faef7c..9a6ec62efc6 100644 --- a/server/src/main/java/com/cloud/test/TestAppender.java +++ b/server/src/main/java/com/cloud/test/TestAppender.java @@ -46,6 +46,7 @@ import static org.apache.log4j.Level.ERROR; import static org.apache.log4j.Level.FATAL; import static org.apache.log4j.Level.INFO; import static org.apache.log4j.Level.OFF; +import static org.apache.log4j.Level.WARN; /** * @@ -152,6 +153,7 @@ public final class TestAppender extends AppenderSkeleton { expectedPatterns.put(FATAL, new HashSet()); expectedPatterns.put(INFO, new HashSet()); expectedPatterns.put(OFF, new HashSet()); + expectedPatterns.put(WARN, new HashSet()); } public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) { checkArgument(level != null, "addExpectedPattern requires a non-null level"); diff --git a/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java b/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java index 59879791ba3..6a9d40cad18 100644 --- a/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java @@ -197,6 +197,7 @@ public final class AnnotationManagerImpl extends ManagerBase implements Annotati s_typeMap.put(EntityType.SYSTEM_VM, ApiCommandResourceType.SystemVm); s_typeMap.put(EntityType.AUTOSCALE_VM_GROUP, ApiCommandResourceType.AutoScaleVmGroup); s_typeMap.put(EntityType.MANAGEMENT_SERVER, ApiCommandResourceType.Host); + s_typeMap.put(EntityType.OBJECT_STORAGE, ApiCommandResourceType.ObjectStore); } public List getKubernetesClusterHelpers() { diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java new file mode 100644 index 00000000000..9dfc75e06f2 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics; + +import com.cloud.api.ApiDBUtils; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StorageStats; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.heuristics.presetvariables.Account; +import org.apache.cloudstack.storage.heuristics.presetvariables.Domain; +import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables; +import org.apache.cloudstack.storage.heuristics.presetvariables.SecondaryStorage; +import org.apache.cloudstack.storage.heuristics.presetvariables.Snapshot; +import org.apache.cloudstack.storage.heuristics.presetvariables.Template; +import org.apache.cloudstack.storage.heuristics.presetvariables.Volume; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Helper class for building and injecting the heuristics preset variables into the JS script. + */ +public class HeuristicRuleHelper { + + protected static final Logger LOGGER = Logger.getLogger(HeuristicRuleHelper.class); + + private static final Long HEURISTICS_SCRIPT_TIMEOUT = StorageManager.HEURISTICS_SCRIPT_TIMEOUT.value(); + + @Inject + private DataStoreManager dataStoreManager; + + @Inject + private ImageStoreDao imageStoreDao; + + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + + @Inject + private DomainDao domainDao; + + @Inject + private AccountDao accountDao; + + /** + * Returns the {@link DataStore} object if the zone, specified by the ID, has an active heuristic rule for the given {@link HeuristicType}. + * It returns null otherwise. + * @param zoneId used to search for the heuristic rules. + * @param heuristicType used for checking if there is a heuristic rule for the given {@link HeuristicType}. + * @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}. + * They are used to retrieve attributes for injecting in the JS rule. + * @return the corresponding {@link DataStore} if there is a heuristic rule, returns null otherwise. + */ + public DataStore getImageStoreIfThereIsHeuristicRule(Long zoneId, HeuristicType heuristicType, Object obj) { + HeuristicVO heuristicsVO = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, heuristicType); + + if (heuristicsVO == null) { + LOGGER.debug(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.", zoneId, heuristicType)); + return null; + } else { + LOGGER.debug(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicsVO, zoneId)); + return interpretHeuristicRule(heuristicsVO.getHeuristicRule(), heuristicType, obj, zoneId); + } + } + + /** + * Build the preset variables ({@link Template}, {@link Snapshot} and {@link Volume}) for the JS script. + */ + protected void buildPresetVariables(JsInterpreter jsInterpreter, HeuristicType heuristicType, long zoneId, Object obj) { + PresetVariables presetVariables = new PresetVariables(); + Long accountId = null; + + switch (heuristicType) { + case TEMPLATE: + case ISO: + presetVariables.setTemplate(setTemplatePresetVariable((VMTemplateVO) obj)); + accountId = ((VMTemplateVO) obj).getAccountId(); + break; + case SNAPSHOT: + presetVariables.setSnapshot(setSnapshotPresetVariable((SnapshotInfo) obj)); + accountId = ((SnapshotInfo) obj).getAccountId(); + break; + case VOLUME: + presetVariables.setVolume(setVolumePresetVariable((VolumeVO) obj)); + accountId = ((VolumeVO) obj).getAccountId(); + break; + } + presetVariables.setAccount(setAccountPresetVariable(accountId)); + presetVariables.setSecondaryStorages(setSecondaryStoragesVariable(zoneId)); + + injectPresetVariables(jsInterpreter, presetVariables); + } + + /** + * Inject the {@link PresetVariables} ({@link Template}, {@link Snapshot} and {@link Volume}) into the JS {@link JsInterpreter}. + * For each type, they can be accessed using the following variables in the script: + *
    + *
  • ISO/Template: using the variable iso or template, e.g. iso.format
    • + *
  • Snapshot: using the variable snapshot, e.g. snapshot.size
    • + *
  • Volume: using the variable volume, e.g. volume.format
    • + *
+ * @param jsInterpreter the JS script + * @param presetVariables used for injecting in the JS interpreter. + */ + protected void injectPresetVariables(JsInterpreter jsInterpreter, PresetVariables presetVariables) { + jsInterpreter.injectVariable("secondaryStorages", presetVariables.getSecondaryStorages().toString()); + + if (presetVariables.getTemplate() != null) { + jsInterpreter.injectVariable("template", presetVariables.getTemplate().toString()); + jsInterpreter.injectVariable("iso", presetVariables.getTemplate().toString()); + } + + if (presetVariables.getSnapshot() != null) { + jsInterpreter.injectVariable("snapshot", presetVariables.getSnapshot().toString()); + } + + if (presetVariables.getVolume() != null) { + jsInterpreter.injectVariable("volume", presetVariables.getVolume().toString()); + } + + if (presetVariables.getAccount() != null) { + jsInterpreter.injectVariable("account", presetVariables.getAccount().toString()); + } + } + + protected List setSecondaryStoragesVariable(long zoneId) { + List secondaryStorageList = new ArrayList<>(); + List imageStoreVOS = imageStoreDao.listStoresByZoneId(zoneId); + + for (ImageStoreVO imageStore : imageStoreVOS) { + SecondaryStorage secondaryStorage = new SecondaryStorage(); + + secondaryStorage.setName(imageStore.getName()); + secondaryStorage.setId(imageStore.getUuid()); + secondaryStorage.setProtocol(imageStore.getProtocol()); + StorageStats storageStats = ApiDBUtils.getSecondaryStorageStatistics(imageStore.getId()); + + if (storageStats != null) { + secondaryStorage.setUsedDiskSize(storageStats.getByteUsed()); + secondaryStorage.setTotalDiskSize(storageStats.getCapacityBytes()); + } + + secondaryStorageList.add(secondaryStorage); + } + return secondaryStorageList; + } + + protected Template setTemplatePresetVariable(VMTemplateVO templateVO) { + Template template = new Template(); + + template.setName(templateVO.getName()); + template.setFormat(templateVO.getFormat()); + template.setHypervisorType(templateVO.getHypervisorType()); + + return template; + } + + protected Volume setVolumePresetVariable(VolumeVO volumeVO) { + Volume volume = new Volume(); + + volume.setName(volumeVO.getName()); + volume.setFormat(volumeVO.getFormat()); + volume.setSize(volumeVO.getSize()); + + return volume; + } + + protected Snapshot setSnapshotPresetVariable(SnapshotInfo snapshotInfo) { + Snapshot snapshot = new Snapshot(); + + snapshot.setName(snapshotInfo.getName()); + snapshot.setSize(snapshotInfo.getSize()); + snapshot.setHypervisorType(snapshotInfo.getHypervisorType()); + + return snapshot; + } + + protected Account setAccountPresetVariable(Long accountId) { + if (accountId == null) { + return null; + } + + AccountVO account = accountDao.findById(accountId); + if (account == null) { + return null; + } + + Account accountVariable = new Account(); + accountVariable.setName(account.getName()); + accountVariable.setId(account.getUuid()); + + accountVariable.setDomain(setDomainPresetVariable(account.getDomainId())); + + return accountVariable; + } + + protected Domain setDomainPresetVariable(long domainId) { + DomainVO domain = domainDao.findById(domainId); + if (domain == null) { + return null; + } + Domain domainVariable = new Domain(); + domainVariable.setName(domain.getName()); + domainVariable.setId(domain.getUuid()); + + return domainVariable; + } + + /** + * This method calls {@link HeuristicRuleHelper#buildPresetVariables(JsInterpreter, HeuristicType, long, Object)} for building the preset variables and + * execute the JS script specified in the rule ({@link String}) parameter. The script is pre-injected with the preset variables, to allow the JS script to reference them + * in the code scope. + *
+ *
+ * The JS script needs to return a valid UUID ({@link String}) of a secondary storage, otherwise a {@link CloudRuntimeException} is thrown. + * @param rule the {@link String} representing the JS script. + * @param heuristicType used for building the preset variables accordingly to the {@link HeuristicType} specified. + * @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}. + * They are used to retrieve attributes for injecting in the JS rule. + * @param zoneId used for injecting the {@link SecondaryStorage} preset variables. + * @return the {@link DataStore} returned by the script. + */ + public DataStore interpretHeuristicRule(String rule, HeuristicType heuristicType, Object obj, long zoneId) { + try (JsInterpreter jsInterpreter = new JsInterpreter(HEURISTICS_SCRIPT_TIMEOUT)) { + buildPresetVariables(jsInterpreter, heuristicType, zoneId, obj); + Object scriptReturn = jsInterpreter.executeScript(rule); + + if (!(scriptReturn instanceof String)) { + throw new CloudRuntimeException(String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", rule)); + } + + DataStore dataStore = dataStoreManager.getImageStoreByUuid((String) scriptReturn); + + if (dataStore == null) { + throw new CloudRuntimeException(String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " + + "returning a valid UUID.", scriptReturn, rule)); + } + + return dataStore; + } catch (IOException ex) { + String message = String.format("Error while executing script [%s].", rule); + LOGGER.error(message, ex); + throw new CloudRuntimeException(message, ex); + } + } + +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java new file mode 100644 index 00000000000..67750e8ec5c --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class Account extends GenericHeuristicPresetVariable { + private String id; + + private Domain domain; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } + + public Domain getDomain() { + return domain; + } + + public void setDomain(Domain domain) { + this.domain = domain; + fieldNamesToIncludeInToString.add("domain"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java new file mode 100644 index 00000000000..6565c06bfbb --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class Domain extends GenericHeuristicPresetVariable{ + private String id; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java new file mode 100644 index 00000000000..f8ded3a864a --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import java.util.HashSet; +import java.util.Set; + +public class GenericHeuristicPresetVariable { + + protected transient Set fieldNamesToIncludeInToString = new HashSet<>(); + + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + fieldNamesToIncludeInToString.add("name"); + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, fieldNamesToIncludeInToString.toArray(new String[0])); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java new file mode 100644 index 00000000000..d0487495327 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import java.util.List; + +public class PresetVariables { + + private Account account; + + private List secondaryStorages; + + private Template template; + + private Snapshot snapshot; + + private Volume volume; + + public List getSecondaryStorages() { + return secondaryStorages; + } + + public void setSecondaryStorages(List secondaryStorages) { + this.secondaryStorages = secondaryStorages; + } + + public Template getTemplate() { + return template; + } + + public void setTemplate(Template template) { + this.template = template; + } + + public Snapshot getSnapshot() { + return snapshot; + } + + public void setSnapshot(Snapshot snapshot) { + this.snapshot = snapshot; + } + + public Volume getVolume() { + return volume; + } + + public void setVolume(Volume volume) { + this.volume = volume; + } + + public Account getAccount() { + return account; + } + + public void setAccount(Account account) { + this.account = account; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java new file mode 100644 index 00000000000..ad7058d8336 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class SecondaryStorage extends GenericHeuristicPresetVariable { + + private String id; + + private Long usedDiskSize; + + private Long totalDiskSize; + + private String protocol; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } + + public Long getUsedDiskSize() { + return usedDiskSize; + } + + public void setUsedDiskSize(Long usedDiskSize) { + this.usedDiskSize = usedDiskSize; + fieldNamesToIncludeInToString.add("usedDiskSize"); + } + + public Long getTotalDiskSize() { + return totalDiskSize; + } + + public void setTotalDiskSize(Long totalDiskSize) { + this.totalDiskSize = totalDiskSize; + fieldNamesToIncludeInToString.add("totalDiskSize"); + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + fieldNamesToIncludeInToString.add("protocol"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java new file mode 100644 index 00000000000..34acd394dbe --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; + +public class Snapshot extends GenericHeuristicPresetVariable { + + private Long size; + + private Hypervisor.HypervisorType hypervisorType; + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + fieldNamesToIncludeInToString.add("size"); + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + fieldNamesToIncludeInToString.add("hypervisorType"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java new file mode 100644 index 00000000000..297c95fad9a --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; + +public class Template extends GenericHeuristicPresetVariable { + + private Hypervisor.HypervisorType hypervisorType; + + private Storage.ImageFormat format; + + private Storage.TemplateType templateType; + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + fieldNamesToIncludeInToString.add("hypervisorType"); + } + + public Storage.ImageFormat getFormat() { + return format; + } + + public void setFormat(Storage.ImageFormat format) { + this.format = format; + fieldNamesToIncludeInToString.add("format"); + } + + public Storage.TemplateType getTemplateType() { + return templateType; + } + + public void setTemplateType(Storage.TemplateType templateType) { + this.templateType = templateType; + fieldNamesToIncludeInToString.add("templateType"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java new file mode 100644 index 00000000000..4e5e81b117f --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.storage.Storage; + +public class Volume extends GenericHeuristicPresetVariable { + + private Long size; + + private Storage.ImageFormat format; + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + fieldNamesToIncludeInToString.add("size"); + } + + public Storage.ImageFormat getFormat() { + return format; + } + + public void setFormat(Storage.ImageFormat format) { + this.format = format; + fieldNamesToIncludeInToString.add("format"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java new file mode 100644 index 00000000000..bfd29cc0442 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import com.amazonaws.services.s3.internal.BucketNameUtils; +import com.amazonaws.services.s3.model.IllegalBucketNameException; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.storage.BucketVO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.dao.BucketDao; +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.usage.dao.BucketStatisticsDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class BucketApiServiceImpl extends ManagerBase implements BucketApiService, Configurable { + private final static Logger s_logger = Logger.getLogger(BucketApiServiceImpl.class); + + @Inject + private ObjectStoreDao _objectStoreDao; + @Inject + DataStoreManager _dataStoreMgr; + @Inject + private BucketDao _bucketDao; + @Inject + private AccountManager _accountMgr; + + @Inject + private BucketStatisticsDao _bucketStatisticsDao; + + private ScheduledExecutorService _executor = null; + + private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; + + protected BucketApiServiceImpl() { + + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Bucket-Usage")); + return true; + } + + @Override + public boolean start() { + _executor.scheduleWithFixedDelay(new BucketUsageTask(), 60L, 3600L, TimeUnit.SECONDS); + return true; + } + + @Override + public boolean stop() { + _executor.shutdown(); + return true; + } + + @Override + public String getConfigComponentName() { + return BucketApiService.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + }; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_CREATE, eventDescription = "creating bucket", create = true) + public Bucket allocBucket(CreateBucketCmd cmd) { + try { + BucketNameUtils.validateBucketName(cmd.getBucketName()); + } catch (IllegalBucketNameException e) { + s_logger.error("Invalid Bucket Name: " +cmd.getBucketName(), e); + throw new InvalidParameterValueException("Invalid Bucket Name: "+e.getMessage()); + } + //ToDo check bucket exists + long ownerId = cmd.getEntityOwnerId(); + Account owner = _accountMgr.getActiveAccountById(ownerId); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(cmd.getObjectStoragePoolId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + try { + if(!objectStore.createUser(ownerId)) { + s_logger.error("Failed to create user in objectstore "+ objectStore.getName()); + return null; + } + } catch (CloudRuntimeException e) { + s_logger.error("Error while checking object store user.", e); + return null; + } + + BucketVO bucket = new BucketVO(ownerId, owner.getDomainId(), cmd.getObjectStoragePoolId(), cmd.getBucketName(), cmd.getQuota(), + cmd.isVersioning(), cmd.isEncryption(), cmd.isObjectLocking(), cmd.getPolicy()); + _bucketDao.persist(bucket); + return bucket; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_CREATE, eventDescription = "creating bucket", async = true) + public Bucket createBucket(CreateBucketCmd cmd) { + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(cmd.getObjectStoragePoolId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + BucketVO bucket = _bucketDao.findById(cmd.getEntityId()); + boolean objectLock = false; + boolean bucketCreated = false; + if(cmd.isObjectLocking()) { + objectLock = true; + } + try { + objectStore.createBucket(bucket, objectLock); + bucketCreated = true; + + if (cmd.isVersioning()) { + objectStore.setBucketVersioning(bucket.getName()); + } + + if (cmd.isEncryption()) { + objectStore.setBucketEncryption(bucket.getName()); + } + + if (cmd.getQuota() != null) { + objectStore.setQuota(bucket.getName(), cmd.getQuota()); + } + + if (cmd.getPolicy() != null) { + objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + } + + bucket.setState(Bucket.State.Created); + _bucketDao.update(bucket.getId(), bucket); + } catch (Exception e) { + s_logger.debug("Failed to create bucket with name: "+bucket.getName(), e); + if(bucketCreated) { + objectStore.deleteBucket(bucket.getName()); + } + _bucketDao.remove(bucket.getId()); + throw new CloudRuntimeException("Failed to create bucket with name: "+bucket.getName()+". "+e.getMessage()); + } + return bucket; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_DELETE, eventDescription = "deleting bucket") + public boolean deleteBucket(long bucketId, Account caller) { + Bucket bucket = _bucketDao.findById(bucketId); + if (bucket == null) { + throw new InvalidParameterValueException("Unable to find bucket with ID: " + bucketId); + } + _accountMgr.checkAccess(caller, null, true, bucket); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + if (objectStore.deleteBucket(bucket.getName())) { + return _bucketDao.remove(bucketId); + } + return false; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_UPDATE, eventDescription = "updating bucket") + public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { + BucketVO bucket = _bucketDao.findById(cmd.getId()); + if (bucket == null) { + throw new InvalidParameterValueException("Unable to find bucket with ID: " + cmd.getId()); + } + _accountMgr.checkAccess(caller, null, true, bucket); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + try { + if (cmd.getEncryption() != null) { + if (cmd.getEncryption()) { + objectStore.setBucketEncryption(bucket.getName()); + } else { + objectStore.deleteBucketEncryption(bucket.getName()); + } + bucket.setEncryption(cmd.getEncryption()); + } + + if (cmd.getVersioning() != null) { + if (cmd.getVersioning()) { + objectStore.setBucketVersioning(bucket.getName()); + } else { + objectStore.deleteBucketVersioning(bucket.getName()); + } + bucket.setVersioning(cmd.getVersioning()); + } + + if (cmd.getPolicy() != null) { + objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + bucket.setPolicy(cmd.getPolicy()); + } + + if (cmd.getQuota() != null) { + objectStore.setQuota(bucket.getName(), cmd.getQuota()); + bucket.setQuota(cmd.getQuota()); + } + _bucketDao.update(bucket.getId(), bucket); + } catch (Exception e) { + throw new CloudRuntimeException("Error while updating bucket: " +bucket.getName() +". "+e.getMessage()); + } + + return true; + } + + public void getBucketUsage() { + //ToDo track usage one last time when object store or bucket is removed + List objectStores = _objectStoreDao.listObjectStores(); + for(ObjectStoreVO objectStoreVO: objectStores) { + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + Map bucketSizes = objectStore.getAllBucketsUsage(); + List buckets = _bucketDao.listByObjectStoreId(objectStoreVO.getId()); + for(BucketVO bucket : buckets) { + Long size = bucketSizes.get(bucket.getName()); + if( size != null){ + bucket.setSize(size); + _bucketDao.update(bucket.getId(), bucket); + } + } + } + } + + private class BucketUsageTask extends ManagedContextRunnable { + public BucketUsageTask() { + } + + @Override + protected void runInContext() { + GlobalLock scanLock = GlobalLock.getInternLock("BucketUsage"); + try { + if (scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { + try { + List objectStores = _objectStoreDao.listObjectStores(); + for(ObjectStoreVO objectStoreVO: objectStores) { + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + Map bucketSizes = objectStore.getAllBucketsUsage(); + List buckets = _bucketDao.listByObjectStoreId(objectStoreVO.getId()); + for(BucketVO bucket : buckets) { + Long size = bucketSizes.get(bucket.getName()); + if( size != null){ + bucket.setSize(size); + _bucketDao.update(bucket.getId(), bucket); + + //Update Bucket Usage stats + BucketStatisticsVO bucketStatisticsVO = _bucketStatisticsDao.findBy(bucket.getAccountId(), bucket.getId()); + if(bucketStatisticsVO != null) { + bucketStatisticsVO.setSize(size); + _bucketStatisticsDao.update(bucketStatisticsVO.getId(), bucketStatisticsVO); + } else { + bucketStatisticsVO = new BucketStatisticsVO(bucket.getAccountId(), bucket.getId()); + bucketStatisticsVO.setSize(size); + _bucketStatisticsDao.persist(bucketStatisticsVO); + } + } + } + } + s_logger.debug("Completed updating bucket usage for all object stores"); + } catch (Exception e) { + s_logger.error("Error while fetching bucket usage", e); + } finally { + scanLock.unlock(); + } + } + } finally { + scanLock.releaseRef(); + } + } + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index ca3a77b8e89..7227264e229 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -352,6 +352,7 @@ class="com.cloud.tags.ResourceManagerUtilImpl"/> + diff --git a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java index 764d9437fa9..4af84cfa814 100644 --- a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java +++ b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java @@ -27,6 +27,8 @@ import com.cloud.network.Network; import com.cloud.network.VNF; import com.cloud.network.dao.NetworkVO; import com.cloud.server.ResourceTag; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; @@ -40,12 +42,17 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.EventResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -66,6 +73,8 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -95,6 +104,12 @@ public class QueryManagerImplTest { @Mock SearchCriteria searchCriteriaMock; + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + BucketDao bucketDao; + private AccountVO account; private UserVO user; @@ -288,4 +303,41 @@ public class QueryManagerImplTest { Assert.assertTrue(set.contains(domainId)); } + + @Test + public void testSearchForObjectStores() { + ListObjectStoragePoolsCmd cmd = new ListObjectStoragePoolsCmd(); + List objectStores = new ArrayList<>(); + ObjectStoreVO os1 = new ObjectStoreVO(); + os1.setName("MinIOStore"); + ObjectStoreVO os2 = new ObjectStoreVO(); + os1.setName("Simulator"); + objectStores.add(os1); + objectStores.add(os2); + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + when(sb.entity()).thenReturn(objectStoreVO); + when(objectStoreDao.createSearchBuilder()).thenReturn(sb); + when(objectStoreDao.searchAndCount(any(), any())).thenReturn(new Pair<>(objectStores, 2)); + ListResponse result = queryManagerImplSpy.searchForObjectStores(cmd); + assertEquals(2, result.getCount().intValue()); + } + + @Test + public void testSearchForBuckets() { + ListBucketsCmd listBucketsCmd = new ListBucketsCmd(); + List buckets = new ArrayList<>(); + BucketVO b1 = new BucketVO(); + b1.setName("test-bucket-1"); + BucketVO b2 = new BucketVO(); + b2.setName("test-bucket-1"); + buckets.add(b1); + buckets.add(b2); + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + BucketVO bucketVO = Mockito.mock(BucketVO.class); + when(sb.entity()).thenReturn(bucketVO); + when(bucketDao.createSearchBuilder()).thenReturn(sb); + when(bucketDao.searchAndCount(any(), any())).thenReturn(new Pair<>(buckets, 2)); + queryManagerImplSpy.searchForBuckets(listBucketsCmd); + } } diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java index 8493535cf85..c2d748ee587 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java @@ -58,6 +58,7 @@ import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.projects.ProjectManager; import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.StoragePoolTagVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolTagsDao; @@ -1163,12 +1164,17 @@ public class ConfigurationManagerTest { @Test public void updateDiskOfferingTagsWithPrimaryStorageWithCorrectTagsTestSuccess(){ String tags = "tag1,tag2"; - List storageTagsWithCorrectTags = new ArrayList<>(Arrays.asList("tag1","tag2")); List pools = new ArrayList<>(Arrays.asList(storagePoolVO)); List volumes = new ArrayList<>(Arrays.asList(volumeVO)); + StoragePoolTagVO poolTagMock1 = Mockito.mock(StoragePoolTagVO.class); + StoragePoolTagVO poolTagMock2 = Mockito.mock(StoragePoolTagVO.class); + List poolTags = List.of(poolTagMock1, poolTagMock2); + Mockito.doReturn("tag1").when(poolTagMock1).getTag(); + Mockito.doReturn("tag2").when(poolTagMock2).getTag(); + Mockito.when(primaryDataStoreDao.listStoragePoolsWithActiveVolumesByOfferingId(anyLong())).thenReturn(pools); - Mockito.when(storagePoolTagsDao.getStoragePoolTags(anyLong())).thenReturn(storageTagsWithCorrectTags); + Mockito.when(storagePoolTagsDao.findStoragePoolTags(anyLong())).thenReturn(poolTags); Mockito.when(diskOfferingDao.findById(anyLong())).thenReturn(diskOfferingVOMock); Mockito.when(_volumeDao.findByDiskOfferingId(anyLong())).thenReturn(volumes); diff --git a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java index 2dfd2829ed9..6bfc8fb629d 100644 --- a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.deploy; + import com.cloud.agent.AgentManager; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.dao.CapacityDao; @@ -25,6 +26,7 @@ import com.cloud.dc.ClusterDetailsVO; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.DedicatedResourceDao; @@ -52,6 +54,7 @@ import com.cloud.storage.ScopeType; import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -61,11 +64,14 @@ import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; +import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Type; @@ -80,7 +86,10 @@ import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.affinity.dao.AffinityGroupDomainMapDao; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMReservationDao; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.messagebus.MessageBus; @@ -127,6 +136,8 @@ import java.util.Set; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) @@ -202,6 +213,18 @@ public class DeploymentPlanningManagerImplTest { @Mock ConfigurationDao configDao; + @Mock + AccountManager _accountMgr; + + @Inject + DiskOfferingDao _diskOfferingDao; + + @Mock + DataStoreManager _dataStoreManager; + + @Inject + HostPodDao _podDao; + private static final long dataCenterId = 1L; private static final long instanceId = 123L; private static final long hostId = 0L; @@ -247,6 +270,8 @@ public class DeploymentPlanningManagerImplTest { List planners = new ArrayList(); planners.add(_planner); _dpm.setPlanners(planners); + StoragePoolAllocator allocator = Mockito.mock(StoragePoolAllocator.class); + _dpm.setStoragePoolAllocators(Arrays.asList(allocator)); Mockito.when(host.getId()).thenReturn(hostId); Mockito.doNothing().when(_dpm).avoidDisabledResources(vmProfile, dc, avoids); @@ -255,13 +280,13 @@ public class DeploymentPlanningManagerImplTest { @Test public void dataCenterAvoidTest() throws InsufficientServerCapacityException, AffinityConflictException { ServiceOfferingVO svcOffering = - new ServiceOfferingVO("testOffering", 1, 512, 500, 1, 1, false, false, false, "test dpm", - false, VirtualMachine.Type.User, null, "FirstFitPlanner", true, false); + new ServiceOfferingVO("testOffering", 1, 512, 500, 1, 1, false, false, false, "test dpm", + false, VirtualMachine.Type.User, null, "FirstFitPlanner", true, false); Mockito.when(vmProfile.getServiceOffering()).thenReturn(svcOffering); DataCenterDeployment plan = new DataCenterDeployment(dataCenterId); - Mockito.when(avoids.shouldAvoid((DataCenterVO)Matchers.anyObject())).thenReturn(true); + Mockito.when(avoids.shouldAvoid((DataCenterVO) Matchers.anyObject())).thenReturn(true); DeployDestination dest = _dpm.planDeployment(vmProfile, plan, avoids, null); assertNull("DataCenter is in avoid set, destination should be null! ", dest); } @@ -269,12 +294,12 @@ public class DeploymentPlanningManagerImplTest { @Test public void plannerCannotHandleTest() throws InsufficientServerCapacityException, AffinityConflictException { ServiceOfferingVO svcOffering = - new ServiceOfferingVO("testOffering", 1, 512, 500, 1, 1, false, false, false, "test dpm", - false, VirtualMachine.Type.User, null, "UserDispersingPlanner", true, false); + new ServiceOfferingVO("testOffering", 1, 512, 500, 1, 1, false, false, false, "test dpm", + false, VirtualMachine.Type.User, null, "UserDispersingPlanner", true, false); Mockito.when(vmProfile.getServiceOffering()).thenReturn(svcOffering); DataCenterDeployment plan = new DataCenterDeployment(dataCenterId); - Mockito.when(avoids.shouldAvoid((DataCenterVO)Matchers.anyObject())).thenReturn(false); + Mockito.when(avoids.shouldAvoid((DataCenterVO) Matchers.anyObject())).thenReturn(false); Mockito.when(_planner.canHandle(vmProfile, plan, avoids)).thenReturn(false); DeployDestination dest = _dpm.planDeployment(vmProfile, plan, avoids, null); @@ -284,15 +309,15 @@ public class DeploymentPlanningManagerImplTest { @Test public void emptyClusterListTest() throws InsufficientServerCapacityException, AffinityConflictException { ServiceOfferingVO svcOffering = - new ServiceOfferingVO("testOffering", 1, 512, 500, 1, 1, false, false, false, "test dpm", - false, VirtualMachine.Type.User, null, "FirstFitPlanner", true, false); + new ServiceOfferingVO("testOffering", 1, 512, 500, 1, 1, false, false, false, "test dpm", + false, VirtualMachine.Type.User, null, "FirstFitPlanner", true, false); Mockito.when(vmProfile.getServiceOffering()).thenReturn(svcOffering); DataCenterDeployment plan = new DataCenterDeployment(dataCenterId); - Mockito.when(avoids.shouldAvoid((DataCenterVO)Matchers.anyObject())).thenReturn(false); + Mockito.when(avoids.shouldAvoid((DataCenterVO) Matchers.anyObject())).thenReturn(false); Mockito.when(_planner.canHandle(vmProfile, plan, avoids)).thenReturn(true); - Mockito.when(((DeploymentClusterPlanner)_planner).orderClusters(vmProfile, plan, avoids)).thenReturn(null); + Mockito.when(((DeploymentClusterPlanner) _planner).orderClusters(vmProfile, plan, avoids)).thenReturn(null); DeployDestination dest = _dpm.planDeployment(vmProfile, plan, avoids, null); assertNull("Planner cannot handle, destination should be null! ", dest); } @@ -368,7 +393,8 @@ public class DeploymentPlanningManagerImplTest { } } - private void prepareAndVerifyAvoidDisabledResourcesTest(int timesRouter, int timesAdminVm, int timesDisabledResource, long roleId, Type vmType, boolean isSystemDepolyable, + private void prepareAndVerifyAvoidDisabledResourcesTest(int timesRouter, int timesAdminVm, + int timesDisabledResource, long roleId, Type vmType, boolean isSystemDepolyable, boolean isAdminVmDeployable) { Mockito.doReturn(isSystemDepolyable).when(_dpm).isRouterDeployableInDisabledResources(); Mockito.doReturn(isAdminVmDeployable).when(_dpm).isAdminVmDeployableInDisabledResources(); @@ -481,9 +507,9 @@ public class DeploymentPlanningManagerImplTest { @Test public void volumesRequireEncryptionTest() { - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path", Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); - VolumeVO vol2 = new VolumeVO("vol2", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.DATADISK); - VolumeVO vol3 = new VolumeVO("vol3", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.DATADISK); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); + VolumeVO vol2 = new VolumeVO("vol2", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.DATADISK); + VolumeVO vol3 = new VolumeVO("vol3", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.DATADISK); vol2.setPassphraseId(1L); List volumes = List.of(vol1, vol2, vol3); @@ -492,9 +518,9 @@ public class DeploymentPlanningManagerImplTest { @Test public void volumesDoNotRequireEncryptionTest() { - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); - VolumeVO vol2 = new VolumeVO("vol2", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.DATADISK); - VolumeVO vol3 = new VolumeVO("vol3", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.DATADISK); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); + VolumeVO vol2 = new VolumeVO("vol2", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.DATADISK); + VolumeVO vol3 = new VolumeVO("vol3", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.DATADISK); List volumes = List.of(vol1, vol2, vol3); Assert.assertFalse("Volumes do not require encryption, but reporting they do", _dpm.anyVolumeRequiresEncryption(volumes)); @@ -511,7 +537,7 @@ public class DeploymentPlanningManagerImplTest { }}; host.setDetails(hostDetails); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); vol1.setPassphraseId(1L); setupMocksForPlanDeploymentHostTests(host, vol1); @@ -536,7 +562,7 @@ public class DeploymentPlanningManagerImplTest { }}; host.setDetails(hostDetails); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); vol1.setPassphraseId(1L); setupMocksForPlanDeploymentHostTests(host, vol1); @@ -561,7 +587,7 @@ public class DeploymentPlanningManagerImplTest { }}; host.setDetails(hostDetails); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); setupMocksForPlanDeploymentHostTests(host, vol1); @@ -585,7 +611,7 @@ public class DeploymentPlanningManagerImplTest { }}; host.setDetails(hostDetails); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); setupMocksForPlanDeploymentHostTests(host, vol1); @@ -610,7 +636,7 @@ public class DeploymentPlanningManagerImplTest { host.setDetails(hostDetails); Mockito.when(host.getStatus()).thenReturn(Status.Up); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); vol1.setPassphraseId(1L); setupMocksForPlanDeploymentHostTests(host, vol1); @@ -640,7 +666,7 @@ public class DeploymentPlanningManagerImplTest { host.setDetails(hostDetails); Mockito.when(host.getStatus()).thenReturn(Status.Up); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); vol1.setPassphraseId(1L); setupMocksForPlanDeploymentHostTests(host, vol1); @@ -666,7 +692,7 @@ public class DeploymentPlanningManagerImplTest { host.setDetails(hostDetails); Mockito.when(host.getStatus()).thenReturn(Status.Up); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); vol1.setPassphraseId(1L); DeploymentClusterPlanner planner = setupMocksForPlanDeploymentHostTests(host, vol1); @@ -691,7 +717,7 @@ public class DeploymentPlanningManagerImplTest { host.setDetails(hostDetails); Mockito.when(host.getStatus()).thenReturn(Status.Up); - VolumeVO vol1 = new VolumeVO("vol1", dataCenterId,podId,1L,1L, instanceId,"folder","path",Storage.ProvisioningType.THIN, (long)10<<30, Volume.Type.ROOT); + VolumeVO vol1 = new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT); vol1.setPassphraseId(1L); DeploymentClusterPlanner planner = setupMocksForPlanDeploymentHostTests(host, vol1); @@ -707,6 +733,84 @@ public class DeploymentPlanningManagerImplTest { } } + @Test + public void findSuitablePoolsForVolumesTest() throws Exception { + Long diskOfferingId = 1L; + HostVO host = Mockito.spy(new HostVO("host")); + Map hostDetails = new HashMap<>() { + { + put(Host.HOST_VOLUME_ENCRYPTION, "true"); + } + }; + host.setDetails(hostDetails); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + + VolumeVO vol1 = Mockito.spy(new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", + Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT)); + Mockito.when(vol1.getId()).thenReturn(1L); + vol1.setState(Volume.State.Allocated); + vol1.setPassphraseId(1L); + vol1.setPoolId(1L); + vol1.setDiskOfferingId(diskOfferingId); + + StoragePoolVO storagePool = new StoragePoolVO(); + storagePool.setStatus(StoragePoolStatus.Maintenance); + storagePool.setId(vol1.getPoolId()); + storagePool.setDataCenterId(dataCenterId); + storagePool.setPodId(podId); + storagePool.setClusterId(clusterId); + + DiskProfile diskProfile = Mockito.mock(DiskProfile.class); + + StoragePoolAllocator allocator = Mockito.mock(StoragePoolAllocator.class); + + DataCenterDeployment plan = new DataCenterDeployment(dataCenterId, podId, clusterId, null, null, null); + + Account account = Mockito.mock(Account.class); + Mockito.when(account.getId()).thenReturn(1L); + Mockito.when(vmProfile.getOwner()).thenReturn(account); + Mockito.when(_accountMgr.isRootAdmin(account.getId())).thenReturn(Boolean.FALSE); + + Mockito.when(_dcDao.findById(dataCenterId)).thenReturn(dc); + Mockito.when(dc.getAllocationState()).thenReturn(AllocationState.Enabled); + + HostPodVO podVo = Mockito.mock(HostPodVO.class); + Mockito.when(podVo.getAllocationState()).thenReturn(AllocationState.Enabled); + Mockito.doReturn(podVo).when(_podDao).findById(podId); + + ClusterVO cluster = Mockito.mock(ClusterVO.class); + Mockito.when(cluster.getAllocationState()).thenReturn(AllocationState.Enabled); + Mockito.when(_clusterDao.findById(clusterId)).thenReturn(cluster); + + DiskOfferingVO diskOffering = Mockito.mock(DiskOfferingVO.class); + + Mockito.when(_diskOfferingDao.findById(vol1.getDiskOfferingId())).thenReturn(diskOffering); + VirtualMachineTemplate vmt = Mockito.mock(VirtualMachineTemplate.class); + + ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(vmProfile.getServiceOffering()).thenReturn(serviceOffering); + + PrimaryDataStore primaryDataStore = Mockito.mock(PrimaryDataStore.class); + + Mockito.when(vmt.getFormat()).thenReturn(Storage.ImageFormat.ISO); + Mockito.when(vmProfile.getTemplate()).thenReturn(vmt); + + Mockito.when(vmProfile.getId()).thenReturn(1L); + Mockito.when(vmProfile.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(volDao.findUsableVolumesForInstance(1L)).thenReturn(Arrays.asList(vol1)); + Mockito.when(volDao.findByInstanceAndType(1L, Volume.Type.ROOT)).thenReturn(Arrays.asList(vol1)); + Mockito.when(_dataStoreManager.getPrimaryDataStore(vol1.getPoolId())).thenReturn((DataStore) primaryDataStore); + Mockito.when(avoids.shouldAvoid(storagePool)).thenReturn(Boolean.FALSE); + + Mockito.doReturn(Arrays.asList(storagePool)).when(allocator).allocateToPool(diskProfile, vmProfile, plan, + avoids, 10); + Mockito.when(volDao.update(vol1.getId(), vol1)).thenReturn(true); + _dpm.findSuitablePoolsForVolumes(vmProfile, plan, avoids, 10); + verify(vol1, times(1)).setPoolId(null); + assertTrue(vol1.getPoolId() == null); + + } + // This is so ugly but everything is so intertwined... private DeploymentClusterPlanner setupMocksForPlanDeploymentHostTests(HostVO host, VolumeVO vol1) { long diskOfferingId = 345L; @@ -739,10 +843,10 @@ public class DeploymentPlanningManagerImplTest { Mockito.doNothing().when(hostDao).loadDetails(host); Mockito.doReturn(volumeVOs).when(volDao).findByInstance(ArgumentMatchers.anyLong()); Mockito.doReturn(suitable).when(_dpm).findSuitablePoolsForVolumes( - ArgumentMatchers.any(VirtualMachineProfile.class), - ArgumentMatchers.any(DataCenterDeployment.class), - ArgumentMatchers.any(ExcludeList.class), - ArgumentMatchers.anyInt() + ArgumentMatchers.any(VirtualMachineProfile.class), + ArgumentMatchers.any(DataCenterDeployment.class), + ArgumentMatchers.any(ExcludeList.class), + ArgumentMatchers.anyInt() ); ClusterVO clusterVO = new ClusterVO(); @@ -750,10 +854,10 @@ public class DeploymentPlanningManagerImplTest { Mockito.when(_clusterDao.findById(ArgumentMatchers.anyLong())).thenReturn(clusterVO); Mockito.doReturn(List.of(host)).when(_dpm).findSuitableHosts( - ArgumentMatchers.any(VirtualMachineProfile.class), - ArgumentMatchers.any(DeploymentPlan.class), - ArgumentMatchers.any(ExcludeList.class), - ArgumentMatchers.anyInt() + ArgumentMatchers.any(VirtualMachineProfile.class), + ArgumentMatchers.any(DeploymentPlan.class), + ArgumentMatchers.any(ExcludeList.class), + ArgumentMatchers.anyInt() ); Map suitableVolumeStoragePoolMap = new HashMap<>() {{ @@ -766,13 +870,13 @@ public class DeploymentPlanningManagerImplTest { Mockito.when(capacityMgr.checkIfHostReachMaxGuestLimit(host)).thenReturn(false); Mockito.when(capacityMgr.checkIfHostHasCpuCapability(ArgumentMatchers.anyLong(), ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt())).thenReturn(true); Mockito.when(capacityMgr.checkIfHostHasCapacity( - ArgumentMatchers.anyLong(), - ArgumentMatchers.anyInt(), - ArgumentMatchers.anyLong(), - ArgumentMatchers.anyBoolean(), - ArgumentMatchers.anyFloat(), - ArgumentMatchers.anyFloat(), - ArgumentMatchers.anyBoolean() + ArgumentMatchers.anyLong(), + ArgumentMatchers.anyInt(), + ArgumentMatchers.anyLong(), + ArgumentMatchers.anyBoolean(), + ArgumentMatchers.anyFloat(), + ArgumentMatchers.anyFloat(), + ArgumentMatchers.anyBoolean() )).thenReturn(true); Mockito.when(serviceOfferingDetailsDao.findDetail(vmProfile.getServiceOfferingId(), GPU.Keys.vgpuType.toString())).thenReturn(null); @@ -783,9 +887,9 @@ public class DeploymentPlanningManagerImplTest { DeploymentClusterPlanner planner = Mockito.spy(new FirstFitPlanner()); try { Mockito.doReturn(List.of(clusterId), List.of()).when(planner).orderClusters( - ArgumentMatchers.any(VirtualMachineProfile.class), - ArgumentMatchers.any(DeploymentPlan.class), - ArgumentMatchers.any(ExcludeList.class) + ArgumentMatchers.any(VirtualMachineProfile.class), + ArgumentMatchers.any(DeploymentPlan.class), + ArgumentMatchers.any(ExcludeList.class) ); } catch (Exception ex) { ex.printStackTrace(); @@ -803,7 +907,8 @@ public class DeploymentPlanningManagerImplTest { return dc; } - private void assertAvoidIsEmpty(ExcludeList avoids, boolean isDcEmpty, boolean isPodsEmpty, boolean isClustersEmpty, boolean isHostsEmpty) { + private void assertAvoidIsEmpty(ExcludeList avoids, boolean isDcEmpty, boolean isPodsEmpty, boolean isClustersEmpty, + boolean isHostsEmpty) { Assert.assertEquals(isDcEmpty, CollectionUtils.isEmpty(avoids.getDataCentersToAvoid())); Assert.assertEquals(isPodsEmpty, CollectionUtils.isEmpty(avoids.getPodsToAvoid())); Assert.assertEquals(isClustersEmpty, CollectionUtils.isEmpty(avoids.getClustersToAvoid())); @@ -811,8 +916,9 @@ public class DeploymentPlanningManagerImplTest { } @Configuration - @ComponentScan(basePackageClasses = {DeploymentPlanningManagerImpl.class}, includeFilters = {@Filter(value = TestConfiguration.Library.class, - type = FilterType.CUSTOM)}, useDefaultFilters = false) + @ComponentScan(basePackageClasses = {DeploymentPlanningManagerImpl.class}, + includeFilters = {@Filter(value = TestConfiguration.Library.class, + type = FilterType.CUSTOM)}, useDefaultFilters = false) public static class TestConfiguration extends SpringUtils.CloudStackTestConfiguration { @Bean @@ -1081,7 +1187,8 @@ public class DeploymentPlanningManagerImplTest { Assert.assertEquals(2, hosts.get(7).getId()); } - private List prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(boolean configValue, boolean mockVolumes, boolean mockClusterStoreVolume) { + private List prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(boolean configValue, + boolean mockVolumes, boolean mockClusterStoreVolume) { try { Field f = ConfigKey.class.getDeclaredField("_defaultValue"); f.setAccessible(true); @@ -1136,28 +1243,28 @@ public class DeploymentPlanningManagerImplTest { @Test public void avoidOtherClustersForDeploymentIfMigrationDisabledConfigAllows() { - prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(true,false, false); + prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(true, false, false); Assert.assertTrue(CollectionUtils.isEmpty(runAvoidOtherClustersForDeploymentIfMigrationDisabledTest())); } @Test public void avoidOtherClustersForDeploymentIfMigrationDisabledNoVmVolumes() { - prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(false,false, false); + prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(false, false, false); Assert.assertTrue(CollectionUtils.isEmpty(runAvoidOtherClustersForDeploymentIfMigrationDisabledTest())); } @Test public void avoidOtherClustersForDeploymentIfMigrationDisabledVmVolumesNonValidScope() { - prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(false,true, false); + prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(false, true, false); Assert.assertTrue(CollectionUtils.isEmpty(runAvoidOtherClustersForDeploymentIfMigrationDisabledTest())); } @Test public void avoidOtherClustersForDeploymentIfMigrationDisabledValid() { - List allClusters = prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(false,true, true); + List allClusters = prepareMockForAvoidOtherClustersForDeploymentIfMigrationDisabled(false, true, true); Set avoidedClusters = runAvoidOtherClustersForDeploymentIfMigrationDisabledTest(); Assert.assertTrue(CollectionUtils.isNotEmpty(avoidedClusters)); - Assert.assertEquals(allClusters.size()-1, avoidedClusters.size()); + Assert.assertEquals(allClusters.size() - 1, avoidedClusters.size()); Assert.assertFalse(avoidedClusters.contains(allClusters.get(0))); } } diff --git a/server/src/test/java/com/cloud/network/IpAddressManagerTest.java b/server/src/test/java/com/cloud/network/IpAddressManagerTest.java index d7863e438e6..935fb4e8c3b 100644 --- a/server/src/test/java/com/cloud/network/IpAddressManagerTest.java +++ b/server/src/test/java/com/cloud/network/IpAddressManagerTest.java @@ -17,19 +17,25 @@ package com.cloud.network; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.network.Network.Service; -import com.cloud.network.dao.IPAddressDao; -import com.cloud.network.dao.IPAddressVO; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; -import com.cloud.network.rules.StaticNat; -import com.cloud.network.rules.StaticNatImpl; -import com.cloud.offerings.NetworkOfferingVO; -import com.cloud.offerings.dao.NetworkOfferingDao; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Vector; + +import com.cloud.network.dao.PublicIpQuarantineDao; +import com.cloud.network.vo.PublicIpQuarantineVO; import com.cloud.user.Account; -import com.cloud.user.AccountVO; -import com.cloud.utils.net.Ip; +import com.cloud.user.AccountManager; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -40,19 +46,18 @@ import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.runners.MockitoJUnitRunner; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Vector; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.Network.Service; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.rules.StaticNat; +import com.cloud.network.rules.StaticNatImpl; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.user.AccountVO; +import com.cloud.utils.net.Ip; @RunWith(MockitoJUnitRunner.class) public class IpAddressManagerTest { @@ -80,6 +85,34 @@ public class IpAddressManagerTest { AccountVO account; + @Mock + PublicIpQuarantineVO publicIpQuarantineVOMock; + + @Mock + PublicIpQuarantineDao publicIpQuarantineDaoMock; + + @Mock + IpAddress ipAddressMock; + + @Mock + AccountVO newOwnerMock; + + @Mock + AccountVO previousOwnerMock; + + @Mock + AccountManager accountManagerMock; + + final long dummyID = 1L; + + final String UUID = "uuid"; + + private static final Date currentDate = new Date(100L); + + private static final Date beforeCurrentDate = new Date(99L); + + private static final Date afterCurrentDate = new Date(101L); + @Before public void setup() throws ResourceUnavailableException { @@ -234,6 +267,136 @@ public class IpAddressManagerTest { return network; } + @Test + public void isPublicIpAddressStillInQuarantineTestRemovedDateIsNullAndCurrentDateIsEqualToEndDateShouldReturnFalse() { + Date endDate = currentDate; + + Mockito.when(publicIpQuarantineVOMock.getRemoved()).thenReturn(null); + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(endDate); + + boolean result = ipAddressManager.isPublicIpAddressStillInQuarantine(publicIpQuarantineVOMock, currentDate); + + Assert.assertFalse(result); + } + + @Test + public void isPublicIpAddressStillInQuarantineTestRemovedDateIsNullAndEndDateIsBeforeCurrentDateShouldReturnFalse() { + Date endDate = beforeCurrentDate; + + Mockito.when(publicIpQuarantineVOMock.getRemoved()).thenReturn(null); + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(endDate); + + boolean result = ipAddressManager.isPublicIpAddressStillInQuarantine(publicIpQuarantineVOMock, currentDate); + + Assert.assertFalse(result); + } + + @Test + public void isPublicIpAddressStillInQuarantineTestRemovedDateIsNullAndEndDateIsAfterCurrentDateShouldReturnTrue() { + Date endDate = afterCurrentDate; + + Mockito.when(publicIpQuarantineVOMock.getRemoved()).thenReturn(null); + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(endDate); + + boolean result = ipAddressManager.isPublicIpAddressStillInQuarantine(publicIpQuarantineVOMock, currentDate); + + Assert.assertTrue(result); + } + + @Test + public void isPublicIpAddressStillInQuarantineTestRemovedDateIsEqualCurrentDateShouldReturnFalse() { + Date removedDate = currentDate; + + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(currentDate); + Mockito.when(publicIpQuarantineVOMock.getRemoved()).thenReturn(removedDate); + + boolean result = ipAddressManager.isPublicIpAddressStillInQuarantine(publicIpQuarantineVOMock, currentDate); + + Assert.assertFalse(result); + } + + @Test + public void isPublicIpAddressStillInQuarantineTestRemovedDateIsBeforeCurrentDateShouldReturnFalse() { + Date removedDate = beforeCurrentDate; + + Mockito.when(publicIpQuarantineVOMock.getRemoved()).thenReturn(removedDate); + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(null); + + boolean result = ipAddressManager.isPublicIpAddressStillInQuarantine(publicIpQuarantineVOMock, currentDate); + + Assert.assertFalse(result); + } + + @Test + public void isPublicIpAddressStillInQuarantineTestRemovedDateIsAfterCurrentDateShouldReturnTrue() { + Date removedDate = afterCurrentDate; + + Mockito.when(publicIpQuarantineVOMock.getRemoved()).thenReturn(removedDate); + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(null); + + boolean result = ipAddressManager.isPublicIpAddressStillInQuarantine(publicIpQuarantineVOMock, currentDate); + + Assert.assertTrue(result); + } + + @Test + public void checkIfPublicIpAddressIsNotInQuarantineAndCanBeAllocatedTestIpIsNotInQuarantineShouldReturnTrue() { + Mockito.when(ipAddressMock.getId()).thenReturn(dummyID); + Mockito.when(publicIpQuarantineDaoMock.findByPublicIpAddressId(Mockito.anyLong())).thenReturn(null); + + boolean result = ipAddressManager.canPublicIpAddressBeAllocated(ipAddressMock, account); + + Assert.assertTrue(result); + } + + @Test + public void checkIfPublicIpAddressIsNotInQuarantineAndCanBeAllocatedTestIpIsNoLongerInQuarantineShouldReturnTrue() { + Mockito.when(ipAddressMock.getId()).thenReturn(dummyID); + Mockito.when(publicIpQuarantineDaoMock.findByPublicIpAddressId(Mockito.anyLong())).thenReturn(publicIpQuarantineVOMock); + Mockito.doReturn(false).when(ipAddressManager).isPublicIpAddressStillInQuarantine(Mockito.any(PublicIpQuarantineVO.class), Mockito.any(Date.class)); + + boolean result = ipAddressManager.canPublicIpAddressBeAllocated(ipAddressMock, newOwnerMock); + + Assert.assertTrue(result); + } + + @Test + public void checkIfPublicIpAddressIsNotInQuarantineAndCanBeAllocatedTestIpIsInQuarantineAndThePreviousOwnerIsTheSameAsTheNewOwnerShouldReturnTrue() { + Mockito.when(ipAddressMock.getId()).thenReturn(dummyID); + Mockito.when(publicIpQuarantineDaoMock.findByPublicIpAddressId(Mockito.anyLong())).thenReturn(publicIpQuarantineVOMock); + + Mockito.doReturn(true).when(ipAddressManager).isPublicIpAddressStillInQuarantine(Mockito.any(PublicIpQuarantineVO.class), Mockito.any(Date.class)); + Mockito.doNothing().when(ipAddressManager).removePublicIpAddressFromQuarantine(Mockito.anyLong(), Mockito.anyString()); + + Mockito.when(publicIpQuarantineVOMock.getPreviousOwnerId()).thenReturn(dummyID); + Mockito.when(accountManagerMock.getAccount(Mockito.anyLong())).thenReturn(previousOwnerMock); + Mockito.when(previousOwnerMock.getUuid()).thenReturn(UUID); + Mockito.when(newOwnerMock.getUuid()).thenReturn(UUID); + + boolean result = ipAddressManager.canPublicIpAddressBeAllocated(ipAddressMock, newOwnerMock); + + Assert.assertTrue(result); + } + + @Test + public void checkIfPublicIpAddressIsNotInQuarantineAndCanBeAllocatedTestIpIsInQuarantineAndThePreviousOwnerIsDifferentFromTheNewOwnerShouldReturnFalse() { + final String UUID_2 = "uuid_2"; + + Mockito.when(ipAddressMock.getId()).thenReturn(dummyID); + Mockito.when(publicIpQuarantineDaoMock.findByPublicIpAddressId(Mockito.anyLong())).thenReturn(publicIpQuarantineVOMock); + + Mockito.doReturn(true).when(ipAddressManager).isPublicIpAddressStillInQuarantine(Mockito.any(PublicIpQuarantineVO.class), Mockito.any(Date.class)); + + Mockito.when(publicIpQuarantineVOMock.getPreviousOwnerId()).thenReturn(dummyID); + Mockito.when(accountManagerMock.getAccount(Mockito.anyLong())).thenReturn(previousOwnerMock); + Mockito.when(previousOwnerMock.getUuid()).thenReturn(UUID); + Mockito.when(newOwnerMock.getUuid()).thenReturn(UUID_2); + + boolean result = ipAddressManager.canPublicIpAddressBeAllocated(ipAddressMock, newOwnerMock); + + Assert.assertFalse(result); + } + @Test public void updateSourceNatIpAddress() throws Exception { IPAddressVO requestedIp = Mockito.mock(IPAddressVO.class); diff --git a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java index d6f5dbd9a7b..c993f7b7095 100644 --- a/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/NetworkServiceImplTest.java @@ -16,13 +16,60 @@ // under the License. package com.cloud.network; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.doReturn; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import com.cloud.domain.Domain; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.network.dao.PublicIpQuarantineDao; +import com.cloud.network.vo.PublicIpQuarantineVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.net.Ip; +import com.cloud.exception.InsufficientAddressCapacityException; +import org.apache.cloudstack.alert.AlertService; +import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd; +import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd; +import org.apache.cloudstack.api.command.user.network.UpdateNetworkCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + import com.cloud.agent.api.to.IpAddressTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; @@ -60,44 +107,9 @@ import com.cloud.vm.NicVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; -import org.apache.cloudstack.alert.AlertService; -import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd; -import org.apache.cloudstack.api.command.user.network.UpdateNetworkCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.framework.config.ConfigKey; import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; import org.mockito.junit.MockitoJUnitRunner; -import org.springframework.test.util.ReflectionTestUtils; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyBoolean; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class NetworkServiceImplTest { @@ -157,7 +169,7 @@ public class NetworkServiceImplTest { ServiceOfferingVO serviceOfferingVoMock; @Mock - IpAddressManager ipAddressManager; + ConfigKey privateMtuKey; @Mock private CallContext callContextMock; @InjectMocks @@ -169,9 +181,46 @@ public class NetworkServiceImplTest { CommandSetupHelper commandSetupHelper; @Mock private Account accountMock; + + @Mock + private AccountVO accountVOMock; + @Mock + private DomainVO domainVOMock; @InjectMocks NetworkServiceImpl service = new NetworkServiceImpl(); + @Mock + DomainDao domainDaoMock; + + @Mock + AccountDao accountDaoMock; + + @Mock + UpdateQuarantinedIpCmd updateQuarantinedIpCmdMock; + + @Mock + PublicIpQuarantineDao publicIpQuarantineDaoMock; + + @Mock + private PublicIpQuarantineVO publicIpQuarantineVOMock; + + @Mock + private IPAddressVO ipAddressVOMock; + + @Mock + private IpAddressManager ipAddressManagerMock; + + @Mock + private Ip ipMock; + + private static Date beforeDate; + + private static Date afterDate; + + private final Long publicIpId = 1L; + + private final String dummyIpAddress = "192.168.0.1"; + private static final String VLAN_ID_900 = "900"; private static final String VLAN_ID_901 = "901"; private static final String VLAN_ID_902 = "902"; @@ -196,6 +245,20 @@ public class NetworkServiceImplTest { private AutoCloseable closeable; + @BeforeClass + public static void setUpBeforeClass() { + Date date = new Date(); + Calendar calendar = Calendar.getInstance(); + + calendar.setTime(date); + calendar.add(Calendar.DATE, -1); + beforeDate = calendar.getTime(); + + calendar.setTime(date); + calendar.add(Calendar.DATE, 1); + afterDate = calendar.getTime(); + } + private void registerCallContext() { account = new AccountVO("testaccount", 1L, "networkdomain", Account.Type.NORMAL, "uuid"); account.setId(ACCOUNT_ID); @@ -231,7 +294,7 @@ public class NetworkServiceImplTest { service.routerDao = routerDao; service.commandSetupHelper = commandSetupHelper; service.networkHelper = networkHelper; - service._ipAddrMgr = ipAddressManager; + service._ipAddrMgr = ipAddressManagerMock; callContextMocked = Mockito.mockStatic(CallContext.class); CallContext callContextMock = Mockito.mock(CallContext.class); callContextMocked.when(CallContext::current).thenReturn(callContextMock); @@ -744,6 +807,113 @@ public class NetworkServiceImplTest { networkServiceImplMock.validateIfServiceOfferingIsActiveAndSystemVmTypeIsDomainRouter(1l); } + @Test + public void updatePublicIpAddressInQuarantineTestQuarantineIsAlreadyExpiredShouldThrowCloudRuntimeException() { + Mockito.when(updateQuarantinedIpCmdMock.getId()).thenReturn(publicIpId); + Mockito.when(updateQuarantinedIpCmdMock.getEndDate()).thenReturn(afterDate); + Mockito.when(publicIpQuarantineDaoMock.findById(Mockito.anyLong())).thenReturn(publicIpQuarantineVOMock); + Mockito.when(accountDaoMock.findById(Mockito.anyLong())).thenReturn(accountVOMock); + Mockito.when(domainDaoMock.findById(Mockito.anyLong())).thenReturn(domainVOMock); + Mockito.doNothing().when(accountManager).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + Mockito.when(ipAddressDao.findById(Mockito.anyLong())).thenReturn(ipAddressVOMock); + Mockito.when(ipAddressVOMock.getAddress()).thenReturn(ipMock); + Mockito.when(ipMock.toString()).thenReturn(dummyIpAddress); + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(beforeDate); + String expectedMessage = String.format("The quarantine for the public IP address [%s] is no longer active; thus, it cannot be updated.", dummyIpAddress); + CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class, + () -> service.updatePublicIpAddressInQuarantine(updateQuarantinedIpCmdMock)); + + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void updatePublicIpAddressInQuarantineTestGivenEndDateIsBeforeCurrentDateShouldThrowInvalidParameterValueException() { + Mockito.when(updateQuarantinedIpCmdMock.getId()).thenReturn(publicIpId); + Mockito.when(updateQuarantinedIpCmdMock.getEndDate()).thenReturn(beforeDate); + + String expectedMessage = String.format("The given end date [%s] is invalid as it is before the current date.", beforeDate); + InvalidParameterValueException assertThrows = Assert.assertThrows(InvalidParameterValueException.class, + () -> service.updatePublicIpAddressInQuarantine(updateQuarantinedIpCmdMock)); + + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void updatePublicIpAddressInQuarantineTestQuarantineIsStillValidAndGivenEndDateIsAfterCurrentDateShouldWork() { + Calendar calendar = Calendar.getInstance(); + calendar.setTime(afterDate); + calendar.add(Calendar.DATE, 5); + Date expectedNewEndDate = calendar.getTime(); + + Mockito.when(updateQuarantinedIpCmdMock.getId()).thenReturn(publicIpId); + Mockito.when(updateQuarantinedIpCmdMock.getEndDate()).thenReturn(expectedNewEndDate); + Mockito.when(publicIpQuarantineDaoMock.findById(Mockito.anyLong())).thenReturn(publicIpQuarantineVOMock); + Mockito.when(accountDaoMock.findById(Mockito.anyLong())).thenReturn(accountVOMock); + Mockito.when(domainDaoMock.findById(Mockito.anyLong())).thenReturn(domainVOMock); + Mockito.doNothing().when(accountManager).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + Mockito.when(ipAddressDao.findById(Mockito.anyLong())).thenReturn(ipAddressVOMock); + Mockito.when(ipAddressDao.findById(Mockito.anyLong())).thenReturn(ipAddressVOMock); + Mockito.when(ipAddressVOMock.getAddress()).thenReturn(ipMock); + Mockito.when(ipMock.toString()).thenReturn(dummyIpAddress); + Mockito.when(publicIpQuarantineVOMock.getEndDate()).thenReturn(afterDate); + Mockito.when(ipAddressManagerMock.updatePublicIpAddressInQuarantine(anyLong(), Mockito.any(Date.class))).thenReturn(publicIpQuarantineVOMock); + + PublicIpQuarantine actualPublicIpQuarantine = service.updatePublicIpAddressInQuarantine(updateQuarantinedIpCmdMock); + Mockito.when(actualPublicIpQuarantine.getEndDate()).thenReturn(expectedNewEndDate); + + Assert.assertEquals(expectedNewEndDate , actualPublicIpQuarantine.getEndDate()); + } + + @Test(expected = CloudRuntimeException.class) + public void retrievePublicIpQuarantineTestIpIdNullAndIpAddressNullShouldThrowException() { + service.retrievePublicIpQuarantine(null, null); + } + + @Test + public void retrievePublicIpQuarantineTestValidIpIdShouldReturnPublicQuarantine() { + Mockito.when(publicIpQuarantineDaoMock.findById(Mockito.anyLong())).thenReturn(publicIpQuarantineVOMock); + + service.retrievePublicIpQuarantine(1L, null); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(1)).findById(Mockito.anyLong()); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(0)).findByIpAddress(Mockito.anyString()); + } + + @Test(expected = CloudRuntimeException.class) + public void retrievePublicIpQuarantineTestInvalidIpIdShouldThrowException() { + Mockito.when(publicIpQuarantineDaoMock.findById(Mockito.anyLong())).thenReturn(null); + + service.retrievePublicIpQuarantine(1L, null); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(1)).findById(Mockito.anyLong()); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(0)).findByIpAddress(Mockito.anyString()); + } + + @Test + public void retrievePublicIpQuarantineTestValidIpAddressShouldReturnPublicQuarantine() { + Mockito.when(publicIpQuarantineDaoMock.findByIpAddress(Mockito.anyString())).thenReturn(publicIpQuarantineVOMock); + + service.retrievePublicIpQuarantine(null, "10.1.1.1"); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(0)).findById(Mockito.anyLong()); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(1)).findByIpAddress(Mockito.anyString()); + } + + @Test(expected = CloudRuntimeException.class) + public void retrievePublicIpQuarantineTestInvalidIpAddressShouldThrowException() { + Mockito.when(publicIpQuarantineDaoMock.findByIpAddress(Mockito.anyString())).thenReturn(null); + + service.retrievePublicIpQuarantine(null, "10.1.1.1"); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(0)).findById(Mockito.anyLong()); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(1)).findByIpAddress(Mockito.anyString()); + } + + @Test + public void retrievePublicIpQuarantineTestIpIdAndAddressInformedShouldUseId() { + Mockito.when(publicIpQuarantineDaoMock.findById(Mockito.anyLong())).thenReturn(publicIpQuarantineVOMock); + + service.retrievePublicIpQuarantine(1L, "10.1.1.1"); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(1)).findById(Mockito.anyLong()); + Mockito.verify(publicIpQuarantineDaoMock, Mockito.times(0)).findByIpAddress(Mockito.anyString()); + } + @Test public void validateNotSharedNetworkRouterIPv4() { NetworkOffering ntwkOff = Mockito.mock(NetworkOffering.class); @@ -876,7 +1046,7 @@ public class NetworkServiceImplTest { when(networkVO.getId()).thenReturn(networkId); when(networkVO.getGuestType()).thenReturn(Network.GuestType.Isolated); try { - when(ipAddressManager.allocateIp(any(), anyBoolean(), any(), anyLong(), any(), any(), eq(srcNatIp))).thenReturn(ipAddress); + when(ipAddressManagerMock.allocateIp(any(), anyBoolean(), any(), anyLong(), any(), any(), eq(srcNatIp))).thenReturn(ipAddress); service.checkAndSetRouterSourceNatIp(account, createNetworkCmd, networkVO); } catch (InsufficientAddressCapacityException | ResourceAllocationException e) { Assert.fail(e.getMessage()); diff --git a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java index 8dd3f32d6c4..18a072172ad 100644 --- a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java @@ -17,6 +17,40 @@ package com.cloud.network.vpc; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.times; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.cloud.exception.PermissionDeniedException; +import com.cloud.network.vpc.dao.VpcDao; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; +import org.apache.cloudstack.api.command.user.network.MoveNetworkAclItemCmd; +import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLItemCmd; +import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLListCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.Network; @@ -30,40 +64,9 @@ import com.cloud.user.AccountManager; import com.cloud.user.User; import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; -import org.apache.cloudstack.api.command.user.network.MoveNetworkAclItemCmd; -import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLItemCmd; -import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLListCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.commons.lang3.StringUtils; import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InOrder; -import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.times; @RunWith(MockitoJUnitRunner.class) public class NetworkACLServiceImplTest { @@ -80,6 +83,8 @@ public class NetworkACLServiceImplTest { @Mock private EntityManager entityManagerMock; @Mock + private VpcDao vpcDaoMock; + @Mock private AccountManager accountManagerMock; @Mock private NetworkACLDao networkAclDaoMock; @@ -99,10 +104,11 @@ public class NetworkACLServiceImplTest { @Mock private UpdateNetworkACLListCmd updateNetworkACLListCmdMock; - private Long networkAclMockId = 1L; + private Long networkAclMockId = 5L; private Long networkOfferingMockId = 2L; private Long networkMockVpcMockId = 3L; private long networkAclListId = 1l; + private static final String SOME_UUID = "someUuid"; @Mock private MoveNetworkAclItemCmd moveNetworkAclItemCmdMock; @@ -121,6 +127,12 @@ public class NetworkACLServiceImplTest { @Mock private CallContext callContextMock; + @Mock + private VpcVO vpcVOMock; + + @Mock + private Account accountMock; + private MockedStatic callContextMocked; @Before @@ -178,9 +190,9 @@ public class NetworkACLServiceImplTest { } }); - NetworkACLItem netowkrAclRuleCreated = networkAclServiceImpl.createNetworkACLItem(createNetworkAclCmdMock); + NetworkACLItem networkAclRuleCreated = networkAclServiceImpl.createNetworkACLItem(createNetworkAclCmdMock); - Assert.assertEquals(number == null ? 6 : number, netowkrAclRuleCreated.getNumber()); + Assert.assertEquals(number == null ? 6 : number, networkAclRuleCreated.getNumber()); InOrder inOrder = Mockito.inOrder( networkAclServiceImpl, networkAclManagerMock, networkAclItemDaoMock); inOrder.verify(networkAclServiceImpl).createAclListIfNeeded(createNetworkAclCmdMock); @@ -392,13 +404,13 @@ public class NetworkACLServiceImplTest { @Test(expected = InvalidParameterValueException.class) public void validateNetworkAclTestAclNotDefaulWithoutVpc() { Mockito.when(networkAclMock.getId()).thenReturn(3L); - Mockito.doReturn(null).when(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); + Mockito.doReturn(null).when(vpcDaoMock).findById(networkMockVpcMockId); networkAclServiceImpl.validateNetworkAcl(networkAclMock); } @Test - public void validateNetworkAclTestAclNotDefaulWithVpc() { + public void validateNetworkAclTestAclNotDefaultWithVpc() { CallContext callContextMock = Mockito.mock(CallContext.class); Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount(); @@ -407,12 +419,11 @@ public class NetworkACLServiceImplTest { Mockito.when(networkAclMock.getId()).thenReturn(3L); Mockito.when(networkAclMock.getVpcId()).thenReturn(networkMockVpcMockId); - Mockito.doReturn(Mockito.mock(Vpc.class)).when(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(networkMockVpcMockId); Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); networkAclServiceImpl.validateNetworkAcl(networkAclMock); - Mockito.verify(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); Mockito.verify(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); callContextMocked.verify(() -> CallContext.current()); @@ -702,16 +713,22 @@ public class NetworkACLServiceImplTest { Mockito.doReturn(networkAclItemVoMock).when(networkAclServiceImpl).validateNetworkAclRuleIdAndRetrieveIt(updateNetworkACLItemCmdMock); Mockito.doReturn(networkAclMock).when(networkAclManagerMock).getNetworkACL(networkAclMockId); Mockito.doNothing().when(networkAclServiceImpl).validateNetworkAcl(Mockito.eq(networkAclMock)); + Mockito.doNothing().when(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(Mockito.any(NetworkACL.class), Mockito.any(Account.class), Mockito.anyString()); Mockito.doNothing().when(networkAclServiceImpl).transferDataToNetworkAclRulePojo(Mockito.eq(updateNetworkACLItemCmdMock), Mockito.eq(networkAclItemVoMock), Mockito.eq(networkAclMock)); Mockito.doNothing().when(networkAclServiceImpl).validateNetworkACLItem(networkAclItemVoMock); Mockito.doReturn(networkAclItemVoMock).when(networkAclManagerMock).updateNetworkACLItem(networkAclItemVoMock); + CallContext callContextMock = Mockito.mock(CallContext.class); + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.when(CallContext.current()).thenReturn(callContextMock); + networkAclServiceImpl.updateNetworkACLItem(updateNetworkACLItemCmdMock); InOrder inOrder = Mockito.inOrder(networkAclServiceImpl, networkAclManagerMock); inOrder.verify(networkAclServiceImpl).validateNetworkAclRuleIdAndRetrieveIt(updateNetworkACLItemCmdMock); inOrder.verify(networkAclManagerMock).getNetworkACL(networkAclMockId); inOrder.verify(networkAclServiceImpl).validateNetworkAcl(networkAclMock); + inOrder.verify(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(networkAclMock, accountMock, "Only Root Admins can update global ACLs."); inOrder.verify(networkAclServiceImpl).transferDataToNetworkAclRulePojo(Mockito.eq(updateNetworkACLItemCmdMock), Mockito.eq(networkAclItemVoMock), Mockito.eq(networkAclMock)); inOrder.verify(networkAclServiceImpl).validateNetworkACLItem(networkAclItemVoMock); inOrder.verify(networkAclManagerMock).updateNetworkACLItem(networkAclItemVoMock); @@ -865,13 +882,18 @@ public class NetworkACLServiceImplTest { Mockito.when(updateNetworkACLListCmdMock.getCustomId()).thenReturn(customId); Mockito.when(updateNetworkACLListCmdMock.getId()).thenReturn(networkAclListId); Mockito.when(updateNetworkACLListCmdMock.getDisplay()).thenReturn(false); + Mockito.when(networkACLVOMock.getVpcId()).thenReturn(networkMockVpcMockId); + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(networkMockVpcMockId); + Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), + Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); + networkAclServiceImpl.updateNetworkACL(updateNetworkACLListCmdMock); - InOrder inOrder = Mockito.inOrder(networkAclDaoMock, entityManagerMock, entityManagerMock, accountManagerMock, networkACLVOMock); + InOrder inOrder = Mockito.inOrder(networkAclDaoMock, vpcDaoMock, accountManagerMock, networkACLVOMock); inOrder.verify(networkAclDaoMock).findById(networkAclListId); - inOrder.verify(entityManagerMock).findById(Mockito.eq(Vpc.class), Mockito.anyLong()); + inOrder.verify(vpcDaoMock).findById(Mockito.anyLong()); inOrder.verify(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), nullable(Vpc.class)); @@ -1063,13 +1085,17 @@ public class NetworkACLServiceImplTest { Mockito.when(updateNetworkACLListCmdMock.getCustomId()).thenReturn(null); Mockito.when(updateNetworkACLListCmdMock.getId()).thenReturn(networkAclListId); Mockito.when(updateNetworkACLListCmdMock.getDisplay()).thenReturn(null); + Mockito.when(networkACLVOMock.getVpcId()).thenReturn(networkMockVpcMockId); + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(networkMockVpcMockId); + Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), + Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); networkAclServiceImpl.updateNetworkACL(updateNetworkACLListCmdMock); - InOrder inOrder = Mockito.inOrder(networkAclDaoMock, entityManagerMock, accountManagerMock, networkACLVOMock); + InOrder inOrder = Mockito.inOrder(networkAclDaoMock, vpcDaoMock, accountManagerMock, networkACLVOMock); inOrder.verify(networkAclDaoMock).findById(networkAclListId); - inOrder.verify(entityManagerMock).findById(eq(Vpc.class), Mockito.anyLong()); + inOrder.verify(vpcDaoMock).findById(Mockito.anyLong()); inOrder.verify(accountManagerMock).checkAccess(any(Account.class), isNull(), eq(true), nullable(Vpc.class)); Mockito.verify(networkACLVOMock, Mockito.times(0)).setName(null); @@ -1086,21 +1112,18 @@ public class NetworkACLServiceImplTest { Mockito.when(nextAclRuleMock.getAclId()).thenReturn(networkAclMockId); Mockito.when(previousAclRuleMock.getAclId()).thenReturn(networkAclMockId); - Mockito.doReturn(networkAclMock).when(networkAclDaoMock).findById(networkAclMockId); - Mockito.doReturn(Mockito.mock(Vpc.class)).when(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); - CallContext callContextMock = Mockito.mock(CallContext.class); Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount(); + Mockito.doReturn(networkAclMock).when(networkAclDaoMock).findById(networkAclMockId); Mockito.when(CallContext.current()).thenReturn(callContextMock); - Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); + Mockito.doNothing().when(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(Mockito.any(NetworkACL.class), Mockito.any(Account.class), Mockito.anyString()); networkAclServiceImpl.validateMoveAclRulesData(aclRuleBeingMovedMock, previousAclRuleMock, nextAclRuleMock); Mockito.verify(networkAclDaoMock).findById(networkAclMockId); - Mockito.verify(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); - Mockito.verify(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); + Mockito.verify(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(Mockito.any(NetworkACL.class), Mockito.any(Account.class), Mockito.anyString()); } @Test @@ -1359,9 +1382,41 @@ public class NetworkACLServiceImplTest { ArrayList allAclRules = new ArrayList<>(); allAclRules.add(networkAclItemVoMock); - Mockito.doReturn("someUuid").when(networkAclItemVoMock).getUuid(); + Mockito.doReturn(SOME_UUID).when(networkAclItemVoMock).getUuid(); networkAclServiceImpl.validateAclConsistency(moveNetworkAclItemCmdMock, networkACLVOMock, allAclRules); Mockito.verify(moveNetworkAclItemCmdMock, Mockito.times(1)).getAclConsistencyHash(); } + + @Test + public void checkGlobalAclPermissionTestGlobalAclWithRootAccountShouldWork() { + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(true).when(networkAclServiceImpl).isGlobalAcl(Mockito.anyLong()); + + networkAclServiceImpl.checkGlobalAclPermission(networkMockVpcMockId, accountMock, "exception"); + } + + @Test(expected = PermissionDeniedException.class) + public void checkGlobalAclPermissionTestGlobalAclWithNonRootAccountShouldThrow() { + Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + Mockito.doReturn(true).when(networkAclServiceImpl).isGlobalAcl(Mockito.anyLong()); + + networkAclServiceImpl.checkGlobalAclPermission(networkMockVpcMockId, accountMock, "exception"); + } + + @Test + public void validateAclAssociatedToVpcTestNonNullVpcShouldCheckAccess() { + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(Mockito.anyLong()); + + networkAclServiceImpl.validateAclAssociatedToVpc(networkMockVpcMockId, accountMock, SOME_UUID); + + Mockito.verify(accountManagerMock, Mockito.times(1)).checkAccess(Mockito.any(Account.class), isNull(), Mockito.anyBoolean(), Mockito.any(Vpc.class)); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateAclAssociatedToVpcTestNullVpcShouldThrowInvalidParameterValueException() { + Mockito.doReturn(null).when(vpcDaoMock).findById(Mockito.anyLong()); + + networkAclServiceImpl.validateAclAssociatedToVpc(networkMockVpcMockId, accountMock, SOME_UUID); + } } diff --git a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java index b802d1fb355..deffb165f29 100644 --- a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java @@ -47,6 +47,7 @@ import com.cloud.network.element.NetworkElement; import com.cloud.network.router.CommandSetupHelper; import com.cloud.network.router.NetworkHelper; import com.cloud.network.router.VirtualRouter; +import com.cloud.network.vpc.dao.NetworkACLDao; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao; @@ -127,6 +128,8 @@ public class VpcManagerImplTest { @Mock NetworkDao networkDao; @Mock + NetworkACLDao networkACLDaoMock; + @Mock NetworkModel networkModel; @Mock NetworkOfferingServiceMapDao networkOfferingServiceMapDao; @@ -150,6 +153,8 @@ public class VpcManagerImplTest { NetworkService networkServiceMock; @Mock FirewallRulesDao firewallDao; + @Mock + NetworkACLVO networkACLVOMock; public static final long ACCOUNT_ID = 1; private AccountVO account; @@ -168,6 +173,9 @@ public class VpcManagerImplTest { final Long vpcOwnerId = 1L; final String vpcName = "Test-VPC"; final String vpcDomain = "domain"; + final Long aclId = 1L; + final Long differentVpcAclId = 3L; + final Long vpcId = 1L; private AutoCloseable closeable; @@ -203,6 +211,7 @@ public class VpcManagerImplTest { manager._dcDao = dataCenterDao; manager._ntwkSvc = networkServiceMock; manager._firewallDao = firewallDao; + manager._networkAclDao = networkACLDaoMock; CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); registerCallContext(); overrideDefaultConfigValue(NetworkService.AllowUsersToSpecifyVRMtu, "_defaultValue", "false"); @@ -487,4 +496,18 @@ public class VpcManagerImplTest { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } } + + @Test + public void validateVpcPrivateGatewayAclIdTestNullAclVoThrowsInvalidParameterValueException() { + Mockito.doReturn(null).when(networkACLDaoMock).findById(aclId); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateVpcPrivateGatewayAclId(vpcId, aclId)); + } + + @Test + public void validateVpcPrivateGatewayTestAclFromDifferentVpcThrowsInvalidParameterValueException() { + Mockito.doReturn(2L).when(networkACLVOMock).getVpcId(); + Mockito.doReturn(networkACLVOMock).when(networkACLDaoMock).findById(differentVpcAclId); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateVpcPrivateGatewayAclId(vpcId, differentVpcAclId)); + } + } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 8df824502cd..478547bff1e 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -21,6 +21,8 @@ import com.cloud.host.Host; import com.cloud.storage.dao.VolumeDao; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -127,4 +129,32 @@ public class StorageManagerImplTest { Assert.assertTrue(storageManagerImpl.isVolumeSuspectedDestroyDuplicateOfVmVolume(volume)); } + @Test + public void storagePoolCompatibleWithVolumePoolTestVolumeWithPoolIdInAllocatedState() { + StoragePoolVO storagePool = new StoragePoolVO(); + storagePool.setPoolType(Storage.StoragePoolType.PowerFlex); + storagePool.setId(1L); + VolumeVO volume = new VolumeVO(); + volume.setState(Volume.State.Allocated); + volume.setPoolId(1L); + PrimaryDataStoreDao storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class); + storageManagerImpl._storagePoolDao = storagePoolDao; + Mockito.doReturn(storagePool).when(storagePoolDao).findById(volume.getPoolId()); + Assert.assertFalse(storageManagerImpl.storagePoolCompatibleWithVolumePool(storagePool, volume)); + + } + + @Test + public void storagePoolCompatibleWithVolumePoolTestVolumeWithoutPoolIdInAllocatedState() { + StoragePoolVO storagePool = new StoragePoolVO(); + storagePool.setPoolType(Storage.StoragePoolType.PowerFlex); + storagePool.setId(1L); + VolumeVO volume = new VolumeVO(); + volume.setState(Volume.State.Allocated); + PrimaryDataStoreDao storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class); + storageManagerImpl._storagePoolDao = storagePoolDao; + Assert.assertTrue(storageManagerImpl.storagePoolCompatibleWithVolumePool(storagePool, volume)); + + } + } diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 8ea14846bef..d56a223082a 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.when; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -117,6 +118,7 @@ import com.cloud.user.ResourceLimitService; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; +import com.cloud.utils.Pair; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; @@ -650,29 +652,35 @@ public class VolumeApiServiceImplTest { @Test public void getStoragePoolTagsTestStorageWithoutTags() { - Mockito.when(storagePoolTagsDao.getStoragePoolTags(storagePoolMockId)).thenReturn(new ArrayList<>()); - - String returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); + Pair, Boolean> returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); Assert.assertNull(returnedStoragePoolTags); } @Test public void getStoragePoolTagsTestStorageWithTags() { - ArrayList tags = new ArrayList<>(); - String tag1 = "tag1"; - String tag2 = "tag2"; - String tag3 = "tag3"; + StoragePoolTagVO tag1 = new StoragePoolTagVO(1,"tag1", false); + StoragePoolTagVO tag2 = new StoragePoolTagVO(1,"tag2", false); + StoragePoolTagVO tag3 = new StoragePoolTagVO(1,"tag3", false); + List tags = Arrays.asList(tag1, tag2, tag3); - tags.add(tag1); - tags.add(tag2); - tags.add(tag3); + Mockito.when(storagePoolTagsDao.findStoragePoolTags(storagePoolMockId)).thenReturn(tags); - Mockito.when(storagePoolTagsDao.getStoragePoolTags(storagePoolMockId)).thenReturn(tags); + Pair, Boolean> returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); - String returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); + Assert.assertEquals(new Pair<>(Arrays.asList("tag1","tag2","tag3"), false), returnedStoragePoolTags); + } - Assert.assertEquals("tag1,tag2,tag3", returnedStoragePoolTags); + @Test + public void getStoragePoolTagsTestStorageWithRuleTag() { + StoragePoolTagVO tag1 = new StoragePoolTagVO(1,"tag1", true); + List tags = List.of(tag1); + + Mockito.when(storagePoolTagsDao.findStoragePoolTags(storagePoolMockId)).thenReturn(tags); + + Pair, Boolean> returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); + + Assert.assertEquals(new Pair<>(List.of("tag1"), true), returnedStoragePoolTags); } @Test @@ -757,7 +765,7 @@ public class VolumeApiServiceImplTest { Mockito.when(newDiskOfferingMock.getTags()).thenReturn("tag1"); - Mockito.doReturn("tag1").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("tag1"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); volumeApiServiceImpl.validateConditionsToReplaceDiskOfferingOfVolume(volumeVoMock, newDiskOfferingMock, storagePoolMock); @@ -1138,7 +1146,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A,B,C").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("A").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("A"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1151,7 +1159,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A,B,C").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("A,B,C,D,X,Y").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("A","B","C","D","X","Y"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1164,7 +1172,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.lenient().doReturn("A,B,C,D,X,Y").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.lenient().doReturn(new Pair<>(List.of("A,B,C,D,X,Y"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1177,7 +1185,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of(""), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1190,7 +1198,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.lenient().doReturn("").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.lenient().doReturn(new Pair<>(List.of(""), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1203,7 +1211,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A,B").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("C,D").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("C,D"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1216,13 +1224,52 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("A").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("A"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); Assert.assertTrue(result); } + @Test + public void doesTargetStorageSupportDiskOfferingTestStorageRuleTagWithDiskOfferingTagThatMatches() { + DiskOfferingVO diskOfferingVoMock = Mockito.mock(DiskOfferingVO.class); + Mockito.doReturn("A").when(diskOfferingVoMock).getTags(); + + StoragePool storagePoolMock = Mockito.mock(StoragePool.class); + Mockito.doReturn(new Pair<>(List.of("tags[0] == 'A'"), true)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + + boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); + + Assert.assertTrue(result); + } + + @Test + public void doesTargetStorageSupportDiskOfferingTestStorageRuleTagWithDiskOfferingTagThatDoesNotMatch() { + DiskOfferingVO diskOfferingVoMock = Mockito.mock(DiskOfferingVO.class); + Mockito.doReturn("'").when(diskOfferingVoMock).getTags(); + + StoragePool storagePoolMock = Mockito.mock(StoragePool.class); + Mockito.doReturn(new Pair<>(List.of("tags[0] == 'A'"), true)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + + boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); + + Assert.assertFalse(result); + } + + @Test + public void doesTargetStorageSupportDiskOfferingTestStorageRuleTagWithNullDiskOfferingTag() { + DiskOfferingVO diskOfferingVoMock = Mockito.mock(DiskOfferingVO.class); + Mockito.doReturn(null).when(diskOfferingVoMock).getTags(); + + StoragePool storagePoolMock = Mockito.mock(StoragePool.class); + Mockito.doReturn(new Pair<>(List.of("tags[0] == 'A'"), true)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + + boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); + + Assert.assertFalse(result); + } + @Test public void validateIfVmHaveBackupsTestExceptionWhenTryToDetachVolumeFromVMWhichBackupOffering() { try { diff --git a/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java b/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java index b892b80c9d4..fa6b71d0cb2 100644 --- a/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java +++ b/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java @@ -60,7 +60,7 @@ public class StoragePoolMonitorTest { @Test public void testProcessConnectStoragePoolNormal() throws Exception { Mockito.when(poolDao.listBy(nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.any(ScopeType.class))).thenReturn(Collections.singletonList(pool)); - Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class))).thenReturn(Collections.emptyList()); + Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class), Mockito.anyBoolean())).thenReturn(Collections.emptyList()); Mockito.when(poolDao.findZoneWideStoragePoolsByHypervisor(Mockito.anyLong(), Mockito.any(Hypervisor.HypervisorType.class))).thenReturn(Collections.emptyList()); Mockito.doReturn(true).when(storageManager).connectHostToSharedPool(host.getId(), pool.getId()); @@ -73,7 +73,7 @@ public class StoragePoolMonitorTest { @Test public void testProcessConnectStoragePoolFailureOnHost() throws Exception { Mockito.when(poolDao.listBy(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(ScopeType.class))).thenReturn(Collections.singletonList(pool)); - Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class))).thenReturn(Collections.emptyList()); + Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class), Mockito.anyBoolean())).thenReturn(Collections.emptyList()); Mockito.when(poolDao.findZoneWideStoragePoolsByHypervisor(Mockito.anyLong(), Mockito.any(Hypervisor.HypervisorType.class))).thenReturn(Collections.emptyList()); Mockito.doThrow(new StorageUnavailableException("unable to mount storage", 123L)).when(storageManager).connectHostToSharedPool(Mockito.anyLong(), Mockito.anyLong()); diff --git a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java index 9d4d34f4ca6..8657c07b5ef 100644 --- a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java +++ b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java @@ -18,21 +18,26 @@ package com.cloud.template; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.event.UsageEventVO; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.org.Grouping; +import com.cloud.server.StatsCollector; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.test.TestAppender; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -45,9 +50,12 @@ import org.apache.cloudstack.framework.events.Event; import org.apache.cloudstack.framework.events.EventBus; import org.apache.cloudstack.framework.events.EventBusException; import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -58,6 +66,7 @@ import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; @@ -68,9 +77,12 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; @@ -119,10 +131,17 @@ public class HypervisorTemplateAdapterTest { ConfigurationDao _configDao; @Mock - DataStoreManager storeMgr; + DataStoreManager dataStoreManagerMock; + @Mock + HeuristicRuleHelper heuristicRuleHelperMock; + + @Mock + StatsCollector statsCollectorMock; + + @Spy @InjectMocks - HypervisorTemplateAdapter _adapter; + HypervisorTemplateAdapter _adapter = new HypervisorTemplateAdapter(); //UsageEventUtils reflection abuse helpers private Map oldFields = new HashMap<>(); @@ -298,6 +317,282 @@ public class HypervisorTemplateAdapterTest { cleanupUsageUtils(); } + @Test + public void createTemplateWithinZonesTestZoneIdsNullShouldCallListAllZones() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(null); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_dcDao, Mockito.times(1)).listAllZones(); + } + + @Test + public void createTemplateWithinZonesTestZoneIdsNotNullShouldNotCallListAllZones() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_dcDao, Mockito.times(0)).listAllZones(); + } + + @Test + public void createTemplateWithinZonesTestZoneDoesNotHaveActiveHeuristicRulesShouldCallStandardImageStoreAllocation() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_adapter, Mockito.times(1)).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + } + + @Test + public void createTemplateWithinZonesTestZoneWithHeuristicRuleShouldCallValidateSecondaryStorageAndCreateTemplate() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + DataStore dataStoreMock = Mockito.mock(DataStore.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(dataStoreMock).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_adapter, Mockito.times(1)).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoresThrowsExceptionIfNotFoundTestNullImageStoreShouldThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(null); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoresThrowsExceptionIfNotFoundTestEmptyImageStoreShouldThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + List imageStoresList = new ArrayList<>(); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test + public void getImageStoresThrowsExceptionIfNotFoundTestNonEmptyImageStoreShouldNotThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + DataStore dataStoreMock = Mockito.mock(DataStore.class); + List imageStoresList = List.of(dataStoreMock); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.ISO); + _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.QCOW2); + _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = null; + Set zoneSet = null; + boolean isTemplatePrivate = false; + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", dataStoreMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIsNullShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = null; + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].", + zoneId, dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIsDisabledShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Disabled); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestImageStoreDoesNotHaveEnoughCapacityShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(false); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].", + dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestImageStoreHasEnoughCapacityAndZoneSetIsNullShouldReturnTrue() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage " + + "of zone [%s].", dataCenterVOMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertTrue(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsAlreadyAllocatedToTheSameZoneShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = Set.of(1L); + boolean isTemplatePrivate = true; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; " + + "therefore, image store [%s] will be skipped.", dataCenterVOMock, dataStoreMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsNotAlreadyAllocatedToTheSameZoneShouldReturnTrue() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = new HashSet<>(); + boolean isTemplatePrivate = true; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Private template will be allocated in image store [%s] in zone [%s].", + dataStoreMock, dataCenterVOMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertTrue(result); + } + @Test public void testCheckZoneImageStoresDirectDownloadTemplate() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); @@ -309,7 +604,7 @@ public class HypervisorTemplateAdapterTest { public void testCheckZoneImageStoresRegularTemplateWithStore() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); Mockito.when(templateVO.isDirectDownload()).thenReturn(false); - Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class))); + Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class))); _adapter.checkZoneImageStores(templateVO, List.of(1L)); } @@ -317,7 +612,7 @@ public class HypervisorTemplateAdapterTest { public void testCheckZoneImageStoresRegularTemplateNoStore() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); Mockito.when(templateVO.isDirectDownload()).thenReturn(false); - Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>()); + Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>()); _adapter.checkZoneImageStores(templateVO, List.of(1L)); } } diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index cb4d701d8a5..a69795cf2c8 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -19,80 +19,6 @@ package com.cloud.template; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; -import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; -import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; -import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.snapshot.SnapshotHelper; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.storage.template.VnfTemplateManager; -import org.apache.cloudstack.test.utils.SpringUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.core.type.classreading.MetadataReader; -import org.springframework.core.type.classreading.MetadataReaderFactory; -import org.springframework.core.type.filter.TypeFilter; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - import com.cloud.agent.AgentManager; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; @@ -106,6 +32,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.projects.ProjectManager; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -117,6 +44,7 @@ import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.SnapshotDao; @@ -140,6 +68,82 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.snapshot.SnapshotHelper; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; +import org.apache.cloudstack.storage.template.VnfTemplateManager; +import org.apache.cloudstack.test.utils.SpringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) @@ -201,6 +205,9 @@ public class TemplateManagerImplTest { @Inject VnfTemplateManager vnfTemplateManager; + @Inject + HeuristicRuleHelper heuristicRuleHelperMock; + public class CustomThreadPoolExecutor extends ThreadPoolExecutor { AtomicInteger ai = new AtomicInteger(0); public CustomThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, @@ -596,6 +603,50 @@ public class TemplateManagerImplTest { Assert.assertEquals(template, resultTemplate); } + @Test + public void getImageStoreTestStoreUuidIsNotNullShouldReturnAValidImageStoreIfValidUuid() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(dataStore); + + templateManager.getImageStore("UUID", 1L, volumeVO); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoreTestStoreUuidIsNotNullShouldThrowCloudRuntimeExceptionIfInvalidUuid() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + + templateManager.getImageStore("UUID", 1L, volumeVO); + } + + @Test + public void getImageStoreTestStoreUuidIsNullAndThereIsNoActiveHeuristicRulesShouldCallGetImageStoreWithFreeCapacity() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(null); + Mockito.when(dataStoreManager.getImageStoreWithFreeCapacity(Mockito.anyLong())).thenReturn(dataStore); + + templateManager.getImageStore(null, 1L, volumeVO); + Mockito.verify(dataStoreManager, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + + @Test + public void getImageStoreTestStoreUuidIsNullAndThereIsActiveHeuristicRulesShouldNotCallGetImageStoreWithFreeCapacity() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(dataStore); + + templateManager.getImageStore(null, 1L, volumeVO); + Mockito.verify(dataStoreManager, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + @Test public void testRegisterTemplateWithTemplateType() { RegisterTemplateCmd cmd = Mockito.mock(RegisterTemplateCmd.class); @@ -915,6 +966,16 @@ public class TemplateManagerImplTest { return Mockito.mock(SnapshotService.class); } + @Bean + public SecondaryStorageHeuristicDao secondaryStorageHeuristicDao() { + return Mockito.mock(SecondaryStorageHeuristicDao.class); + } + + @Bean + public HeuristicRuleHelper heuristicRuleHelper() { + return Mockito.mock(HeuristicRuleHelper.class); + } + public static class Library implements TypeFilter { @Override public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException { diff --git a/server/src/test/java/com/cloud/user/MockUsageEventDao.java b/server/src/test/java/com/cloud/user/MockUsageEventDao.java index 97792871b4d..4639c509249 100644 --- a/server/src/test/java/com/cloud/user/MockUsageEventDao.java +++ b/server/src/test/java/com/cloud/user/MockUsageEventDao.java @@ -261,6 +261,11 @@ public class MockUsageEventDao implements UsageEventDao{ return null; } + @Override + public UsageEventVO findOneBy(SearchCriteria sc, Filter filter) { + return null; + } + @Override public Class getEntityBeanType() { return null; diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 8e8f3e36428..de79baf0660 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -40,6 +40,7 @@ import com.cloud.network.Networks.TrafficType; import com.cloud.network.PhysicalNetwork; import com.cloud.network.PhysicalNetworkServiceProvider; import com.cloud.network.PhysicalNetworkTrafficType; +import com.cloud.network.PublicIpQuarantine; import com.cloud.network.dao.NetworkServiceMapDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.element.DhcpServiceProvider; @@ -73,6 +74,8 @@ import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd; import org.apache.cloudstack.api.command.admin.network.ListGuestVlansCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd; +import org.apache.cloudstack.api.command.user.address.RemoveQuarantinedIpCmd; +import org.apache.cloudstack.api.command.user.address.UpdateQuarantinedIpCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkPermissionsCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkPermissionsCmd; @@ -276,6 +279,15 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return null; } + /* (non-Javadoc) + * @see com.cloud.network.NetworkService#getIp(String) + */ + @Override + public IpAddress getIp(String ipAddress) { + // TODO Auto-generated method stub + return null; + } + @Override public Network updateGuestNetwork(final UpdateNetworkCmd cmd) { // TODO Auto-generated method stub @@ -1058,4 +1070,14 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches @Override public void validateIfServiceOfferingIsActiveAndSystemVmTypeIsDomainRouter(final Long serviceOfferingId) { } + + @Override + public PublicIpQuarantine updatePublicIpAddressInQuarantine(UpdateQuarantinedIpCmd cmd) { + return null; + } + + @Override + public void removePublicIpAddressFromQuarantine(RemoveQuarantinedIpCmd cmd) { + + } } diff --git a/server/src/test/java/org/apache/cloudstack/networkoffering/CreateNetworkOfferingTest.java b/server/src/test/java/org/apache/cloudstack/networkoffering/CreateNetworkOfferingTest.java index f7a6f55b8fe..838bb3dadcb 100644 --- a/server/src/test/java/org/apache/cloudstack/networkoffering/CreateNetworkOfferingTest.java +++ b/server/src/test/java/org/apache/cloudstack/networkoffering/CreateNetworkOfferingTest.java @@ -17,6 +17,31 @@ package org.apache.cloudstack.networkoffering; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.inject.Inject; + +import com.cloud.network.dao.PublicIpQuarantineDao; +import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.event.dao.UsageEventDao; @@ -39,28 +64,6 @@ import com.cloud.user.UserVO; import com.cloud.utils.component.ComponentContext; import com.cloud.vm.dao.UserVmDetailsDao; import junit.framework.TestCase; -import org.apache.cloudstack.annotation.dao.AnnotationDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import org.apache.cloudstack.resourcedetail.dao.UserIpAddressDetailsDao; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import javax.inject.Inject; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.nullable; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(locations = "classpath:/createNetworkOffering.xml") @@ -104,6 +107,9 @@ public class CreateNetworkOfferingTest extends TestCase { @Inject VlanDetailsDao vlanDetailsDao; + @Inject + PublicIpQuarantineDao publicIpQuarantineDao; + @Override @Before public void setUp() { diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java new file mode 100644 index 00000000000..6bf7eef2844 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics; + +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.test.TestAppender; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; +import org.apache.log4j.Level; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.regex.Pattern; + +@RunWith(MockitoJUnitRunner.class) +public class HeuristicRuleHelperTest { + + @Mock + SecondaryStorageHeuristicDao secondaryStorageHeuristicDaoMock; + + @Mock + HeuristicVO heuristicVOMock; + + @Mock + VMTemplateVO vmTemplateVOMock; + + @Mock + SnapshotInfo snapshotInfoMock; + + @Mock + VolumeVO volumeVOMock; + + @Mock + DataStoreManager dataStoreManagerMock; + + @Mock + DataStore dataStoreMock; + + @Spy + @InjectMocks + HeuristicRuleHelper heuristicRuleHelperSpy = new HeuristicRuleHelper(); + + @Test + public void getImageStoreIfThereIsHeuristicRuleTestZoneDoesNotHaveHeuristicRuleShouldReturnNull() { + Long zoneId = 1L; + + Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(null); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.", + zoneId, HeuristicType.TEMPLATE))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender); + + DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null); + + testLogAppender.assertMessagesLogged(); + Assert.assertNull(result); + } + + @Test + public void getImageStoreIfThereIsHeuristicRuleTestZoneHasHeuristicRuleShouldCallInterpretHeuristicRule() { + Long zoneId = 1L; + + Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(heuristicVOMock); + Mockito.when(heuristicVOMock.getHeuristicRule()).thenReturn("rule"); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).interpretHeuristicRule(Mockito.anyString(), Mockito.any(HeuristicType.class), Mockito.isNull(), + Mockito.anyLong()); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicVOMock, zoneId))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender); + + DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null); + + testLogAppender.assertMessagesLogged(); + Assert.assertNull(result); + } + + @Test + public void buildPresetVariablesTestWithTemplateHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.TEMPLATE, 1L, vmTemplateVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithIsoHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.ISO, 1L, vmTemplateVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetSnapshotAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.SNAPSHOT, 1L, snapshotInfoMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSnapshotPresetVariable(Mockito.any(SnapshotInfo.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetVolumeAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.VOLUME, 1L, volumeVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setVolumePresetVariable(Mockito.any(VolumeVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleDoesNotReturnAStringShouldThrowCloudRuntimeException() { + String heuristicRule = "1"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + + String expectedMessage = String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", heuristicRule); + CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class, + () -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L)); + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithInvalidUuidShouldThrowCloudRuntimeException() { + String heuristicRule = "'uuid'"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + Mockito.doReturn(null).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString()); + + String expectedMessage = String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " + + "returning a valid UUID.", "uuid", heuristicRule); + CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class, + () -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L)); + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithAValidUuidShouldReturnAImageStore() { + String heuristicRule = "'uuid'"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + Mockito.doReturn(dataStoreMock).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString()); + + DataStore result = heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L); + + Assert.assertNotNull(result); + } +} diff --git a/server/src/test/resources/createNetworkOffering.xml b/server/src/test/resources/createNetworkOffering.xml index b801116cf48..787de99a754 100644 --- a/server/src/test/resources/createNetworkOffering.xml +++ b/server/src/test/resources/createNetworkOffering.xml @@ -74,4 +74,5 @@ + diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 5d76393d81e..cc17e48fe07 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -85,6 +85,7 @@ import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -201,6 +202,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private static final String POST_UPLOAD_KEY_LOCATION = "/etc/cloudstack/agent/ms-psk"; private static final String ORIGINAL_FILE_EXTENSION = ".orig"; + private static final String USE_HTTPS_TO_UPLOAD = "useHttpsToUpload"; + private static final Map updatableConfigData = Maps.newHashMap(); static { @@ -3537,7 +3540,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return null; } - private String getPostUploadPSK() { + protected String getPostUploadPSK() { if (_ssvmPSK == null) { try { _ssvmPSK = FileUtils.readFileToString(new File(POST_UPLOAD_KEY_LOCATION), "utf-8"); @@ -3573,14 +3576,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S throw new InvalidParameterValueException("content length is not set in the request or has invalid value."); } - //validate signature - String fullUrl = "https://" + hostname + "/upload/" + uuid; - String computedSignature = EncryptionUtil.generateSignature(metadata + fullUrl + timeout, getPostUploadPSK()); - boolean isSignatureValid = computedSignature.equals(signature); - if (!isSignatureValid) { - updateStateMapWithError(uuid, "signature validation failed."); - throw new InvalidParameterValueException("signature validation failed."); - } + validatePostUploadRequestSignature(signature, hostname, uuid, metadata, timeout); //validate timeout DateTime timeoutDateTime = DateTime.parse(timeout, ISODateTimeFormat.dateTime()); @@ -3590,6 +3586,48 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + /** + * Validates whether the provided signature matches the signature generated from the other parameters; + * throws an InvalidParameterValueException if it does not. + */ + protected void validatePostUploadRequestSignature(String signature, String hostname, String uuid, String metadata, String timeout) { + s_logger.trace(String.format("Validating signature [%s] for post upload request [%s].", signature, uuid)); + String protocol = getUploadProtocol(); + String fullUrl = String.format("%s://%s/upload/%s", protocol, hostname, uuid); + String data = String.format("%s%s%s", metadata, fullUrl, timeout); + + String computedSignature = EncryptionUtil.generateSignature(data, getPostUploadPSK()); + s_logger.debug(String.format("Computed signature for post upload request [%s] is [%s].", uuid, computedSignature)); + + boolean isSignatureValid = computedSignature.equals(signature); + if (!isSignatureValid) { + s_logger.debug(String.format("Signature for post upload request [%s] is invalid.", uuid)); + String errorMsg = "signature validation failed."; + updateStateMapWithError(uuid, errorMsg); + throw new InvalidParameterValueException(errorMsg); + } + s_logger.debug(String.format("Signature for post upload request [%s] is valid.", uuid)); + } + + /** + * Returns the protocol used for uploads as a string. + */ + protected String getUploadProtocol() { + if (useHttpsToUpload()) { + s_logger.debug(String.format("Param [%s] is set to true; therefore, HTTPS is being used.", USE_HTTPS_TO_UPLOAD)); + return NetUtils.HTTPS_PROTO; + } + s_logger.debug(String.format("Param [%s] is set to false; therefore, HTTP is being used.", USE_HTTPS_TO_UPLOAD)); + return NetUtils.HTTP_PROTO; + } + + /** + * Retrieves the value of "useHttpsToUpload" from the params as a boolean + */ + protected boolean useHttpsToUpload() { + return BooleanUtils.toBoolean((String) _params.get(USE_HTTPS_TO_UPLOAD)); + } + private TemplateOrVolumePostUploadCommand getTemplateOrVolumePostUploadCmd(String metadata) { TemplateOrVolumePostUploadCommand cmd = null; try { diff --git a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index cd6444a8b4b..72b9f5abe19 100644 --- a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -28,6 +28,9 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Stream; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.net.NetUtils; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; @@ -51,6 +54,22 @@ public class NfsSecondaryStorageResourceTest { @Spy private NfsSecondaryStorageResource resource; + private static final String HOSTNAME = "hostname"; + + private static final String UUID = "uuid"; + + private static final String METADATA = "metadata"; + + private static final String TIMEOUT = "timeout"; + + private static final String PSK = "6HyGMx9Vat7rZw1pMZrM4OlD4FFwLUPznTsFqVFSOIvk0mAWMRCVZ6UCq42gZvhp"; + + private static final String PROTOCOL = NetUtils.HTTP_PROTO; + + private static final String EXPECTED_SIGNATURE = "expectedSignature"; + + private static final String COMPUTED_SIGNATURE = "computedSignature"; + @Test public void testSwiftWriteMetadataFile() throws Exception { String metaFileName = "test_metadata_file"; @@ -141,4 +160,86 @@ public class NfsSecondaryStorageResourceTest { Assert.assertEquals(dir + File.separator + fileName + ".ovf", result.get(1)); } } + + private void prepareForValidatePostUploadRequestSignatureTests(MockedStatic encryptionUtilMock) { + Mockito.doReturn(PROTOCOL).when(resource).getUploadProtocol(); + Mockito.doReturn(PSK).when(resource).getPostUploadPSK(); + encryptionUtilMock.when(() -> EncryptionUtil.generateSignature(Mockito.anyString(), Mockito.anyString())).thenReturn(COMPUTED_SIGNATURE); + String fullUrl = String.format("%s://%s/upload/%s", PROTOCOL, HOSTNAME, UUID); + String data = String.format("%s%s%s", METADATA, fullUrl, TIMEOUT); + encryptionUtilMock.when(() -> EncryptionUtil.generateSignature(data, PSK)).thenReturn(EXPECTED_SIGNATURE); + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenProtocolDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + Mockito.doReturn(NetUtils.HTTPS_PROTO).when(resource).getUploadProtocol(); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, METADATA, TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenHostnameDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, "test", UUID, METADATA, TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenUuidDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, "test", METADATA, TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenMetadataDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, "test", TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenTimeoutDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, METADATA, "test"); + } + } + + @Test + public void validatePostUploadRequestSignatureTestSuccessWhenDataIsTheSame() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, METADATA, TIMEOUT); + } + } + + @Test + public void getUploadProtocolTestReturnHttpsWhenUseHttpsToUploadIsTrue() { + Mockito.doReturn(true).when(resource).useHttpsToUpload(); + + String result = resource.getUploadProtocol(); + + Assert.assertEquals(NetUtils.HTTPS_PROTO, result); + } + + @Test + public void getUploadProtocolTestReturnHttpWhenUseHttpsToUploadIsFalse() { + Mockito.doReturn(false).when(resource).useHttpsToUpload(); + + String result = resource.getUploadProtocol(); + + Assert.assertEquals(NetUtils.HTTP_PROTO, result); + } } diff --git a/systemvm/agent/scripts/ssvm-check.sh b/systemvm/agent/scripts/ssvm-check.sh index b2721a93b3f..f5d69cb4548 100644 --- a/systemvm/agent/scripts/ssvm-check.sh +++ b/systemvm/agent/scripts/ssvm-check.sh @@ -103,10 +103,10 @@ else echo "Verifying if we can at least ping the storage" STORAGE_ADDRESSES=`grep "secondaryStorageServerAddress" $CMDLINE | sed -E 's/.*secondaryStorageServerAddress=([^ ]*).*/\1/g'` - if [[ -z "$STORAGE_ADDRESS" ]] + if [[ -z "$STORAGE_ADDRESSES" ]] then STORAGE_NETWORK_GATEWAY=`grep "storagegateway" $CMDLINE | sed -E 's/.*storagegateway=([^ ]*).*/\1/g'` - echo "Storage address is empty, trying to ping storage network gateway instead ($STORAGE_NETWORK_GATEWAY)" + echo "Storage address list is empty, trying to ping storage network gateway instead ($STORAGE_NETWORK_GATEWAY)" ping -c 2 $STORAGE_NETWORK_GATEWAY if [ $? -eq 0 ] then @@ -118,7 +118,7 @@ else fi else echo "Storage address(s): $STORAGE_ADDRESSES, trying to ping" - STORAGE_ADDRESS_LIST=$(echo $STORAGE_ADDRESSES | tr ",") + STORAGE_ADDRESS_LIST=$(echo $STORAGE_ADDRESSES | tr "," "\n") for STORAGE_ADDRESS in $STORAGE_ADDRESS_LIST do echo "Pinging storage address: $STORAGE_ADDRESS" diff --git a/test/integration/smoke/test_bucket.py b/test/integration/smoke/test_bucket.py new file mode 100644 index 00000000000..7d92ea98b07 --- /dev/null +++ b/test/integration/smoke/test_bucket.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" BVT tests for Bucket Operations""" + +#Import Local Modules +from marvin.cloudstackTestCase import * +from nose.plugins.attrib import attr +from marvin.lib.base import (ObjectStoragePool, Bucket) +from marvin.lib.utils import (cleanup_resources) + +_multiprocess_shared_ = True + +class TestObjectStore(cloudstackTestCase): + + def setUp(self): + self.services = self.testClient.getParsedTestDataConfig() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + #Clean up, terminate the created resources + cleanup_resources(self.apiclient, self.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["smoke"], required_hardware="false") + def test_01_create_bucket(self): + """Test to create bucket in object store + + """ + + object_store = ObjectStoragePool.create( + self.apiclient, + "testOS-9", + "http://192.168.0.1", + "Simulator", + None + ) + + self.debug("Created Object Store with ID: %s" % object_store.id) + + bucket = Bucket.create( + self.apiclient, + "mybucket", + object_store.id + ) + + list_buckets_response = Bucket.list( + self.apiclient, + id=bucket.id + ) + + self.assertNotEqual( + len(list_buckets_response), + 0, + "Check List Bucket response" + ) + + bucket_response = list_buckets_response[0] + self.assertEqual( + object_store.id, + bucket_response.objectstorageid, + "Check object store id of the created Bucket" + ) + self.assertEqual( + "mybucket", + bucket_response.name, + "Check Name of the created Bucket" + ) + + bucket.update( + self.apiclient, + quota=100 + ) + + list_buckets_response_updated = Bucket.list( + self.apiclient, + id=bucket.id + ) + + bucket_response_updated = list_buckets_response_updated[0] + + self.assertEqual( + 100, + bucket_response_updated.quota, + "Check quota of the updated bucket" + ) + + self.cleanup.append(bucket) + self.cleanup.append(object_store) + + return diff --git a/test/integration/smoke/test_global_acls.py b/test/integration/smoke/test_global_acls.py new file mode 100644 index 00000000000..47db858c121 --- /dev/null +++ b/test/integration/smoke/test_global_acls.py @@ -0,0 +1,245 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (Network, NetworkACLList, NetworkOffering, VpcOffering, VPC, NetworkACL) +from marvin.lib.common import (get_domain, get_zone) +from nose.plugins.attrib import attr +from marvin.cloudstackException import CloudstackAPIException + + +class Services: + """Test Global ACLs + """ + + def __init__(self): + self.services = { + "root_domain": { + "name": "ROOT", + }, + "domain": { + "name": "Domain", + }, + "user": { + "username": "user", + "roletype": 0, + }, + "domain_admin": { + "username": "Domain admin", + "roletype": 2, + }, + "root_admin": { + "username": "Root admin", + "roletype": 1, + }, + "vpc": { + "name": "vpc-networkacl", + "displaytext": "vpc-networkacl", + "cidr": "10.1.1.0/24", + }, + "vpcnetwork": { + "name": "vpcnetwork", + "displaytext": "vpcnetwork", + }, + "rule": { + "protocol": "all", + "traffictype": "ingress", + } + } + + +class TestGlobalACLs(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super(TestGlobalACLs, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + + cls.services = Services().services + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + return + + def setUp(self): + self.user_apiclient = self.testClient.getUserApiClient(self.services["user"]["username"], + self.services["domain"]["name"], + self.services["user"]["roletype"]) + + self.domain_admin_apiclient = self.testClient.getUserApiClient(self.services["domain_admin"]["username"], + self.services["domain"]["name"], + self.services["domain_admin"]["roletype"]) + + self.admin_apiclient = self.testClient.getUserApiClient(self.services["root_admin"]["username"], + self.services["root_domain"]["name"], + self.services["root_admin"]["roletype"]) + + self.cleanup = [] + return + + def tearDown(self): + super(TestGlobalACLs, self).tearDown() + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_create_global_acl(self): + """ Test create global ACL as a normal user, domain admin and root admin users. + """ + + self.debug("Creating ACL list as a normal user, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admin can create global ACLs.", + NetworkACLList.create, apiclient=self.user_apiclient, services={}, + name="acl", description="acl") + + self.debug("Creating ACL list as a domain admin, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admin can create global ACLs.", + NetworkACLList.create, apiclient=self.domain_admin_apiclient, services={}, + name="acl", description="acl") + + self.debug("Creating ACL list as a root admin, should work.") + acl = NetworkACLList.create(apiclient=self.admin_apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + self.assertIsNotNone(acl, "A root admin user should be able to create a global ACL.") + + return + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_replace_acl_of_network(self): + """ Test to replace ACL of a VPC as a normal user, domain admin and root admin users. + """ + # Get network offering + networkOffering = NetworkOffering.list(self.apiclient, name="DefaultIsolatedNetworkOfferingForVpcNetworks") + self.assertTrue(networkOffering is not None and len(networkOffering) > 0, "No VPC network offering") + + # Getting VPC offering + vpcOffering = VpcOffering.list(self.apiclient, name="Default VPC offering") + self.assertTrue(vpcOffering is not None and len(vpcOffering) > 0, "No VPC offerings found") + + # Creating VPC + vpc = VPC.create( + apiclient=self.apiclient, + services=self.services["vpc"], + networkDomain="vpc.networkacl", + vpcofferingid=vpcOffering[0].id, + zoneid=self.zone.id, + domainid=self.domain.id + ) + self.cleanup.append(vpc) + self.assertTrue(vpc is not None, "VPC creation failed") + + # Creating ACL list + acl = NetworkACLList.create(apiclient=self.apiclient, services={}, name="acl", description="acl") + + # Creating tier on VPC with ACL list + network = Network.create( + apiclient=self.apiclient, + services=self.services["vpcnetwork"], + accountid="Admin", + domainid=self.domain.id, + networkofferingid=networkOffering[0].id, + zoneid=self.zone.id, + vpcid=vpc.id, + aclid=acl.id, + gateway="10.1.1.1", + netmask="255.255.255.192" + ) + self.cleanup.append(network) + + # User should be able to replace ACL + network.replaceACLList(apiclient=self.user_apiclient, aclid=acl.id) + # Domain Admin should be able to replace ACL + network.replaceACLList(apiclient=self.domain_admin_apiclient, aclid=acl.id) + # Admin should be able to replace ACL + network.replaceACLList(apiclient=self.admin_apiclient, aclid=acl.id) + + return + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_create_acl_rule(self): + """ Test to create ACL rule as a normal user, domain admin and root admin users. + """ + # Creating ACL list + acl = NetworkACLList.create(apiclient=self.admin_apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + + self.debug("Creating ACL rule as a user, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admins can create rules for a global ACL.", + NetworkACL.create, self.user_apiclient, services=self.services["rule"], aclid=acl.id) + self.debug("Creating ACL rule as a domain admin, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admins can create rules for a global ACL.", + NetworkACL.create, self.domain_admin_apiclient, services=self.services["rule"], aclid=acl.id) + self.debug("Creating ACL rule as a root admin, should work.") + acl_rule = NetworkACL.create(self.admin_apiclient, services=self.services["rule"], aclid=acl.id) + self.cleanup.append(acl_rule) + + return + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_delete_acl_rule(self): + """ Test to delete ACL rule as a normal user, domain admin and root admin users. + """ + # Creating ACL list + acl = NetworkACLList.create(apiclient=self.apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + + # Creating ACL rule + acl_rule = NetworkACL.create(self.apiclient, services=self.services["rule"], aclid=acl.id) + self.cleanup.append(acl_rule) + + self.debug("Deleting ACL rule as a user, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACL rules.", + NetworkACL.delete, acl_rule, self.user_apiclient) + self.debug("Deleting ACL rule as a domain admin, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACL rules.", + NetworkACL.delete, acl_rule, self.domain_admin_apiclient) + + self.debug("Deleting ACL rule as a root admin, should work.") + NetworkACL.delete(acl_rule, self.admin_apiclient) + self.cleanup.remove(acl_rule) + + # Verify if the number of ACL rules is equal to four, i.e. the number of rules + # for the default ACLs `default_allow` (2 rules) and `default_deny` (2 rules) ACLs + number_of_acl_rules = acl_rule.list(apiclient=self.admin_apiclient) + self.assertEqual(len(number_of_acl_rules), 4) + + return + + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_delete_global_acl(self): + """ Test delete global ACL as a normal user, domain admin and root admin users. + """ + + # Creating ACL list. Not adding to cleanup as it will be deleted in this method + acl = NetworkACLList.create(apiclient=self.apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + + self.debug("Deleting ACL list as a normal user, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACLs.", + NetworkACLList.delete, acl, apiclient=self.user_apiclient) + + self.debug("Deleting ACL list as a domain admin, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACLs.", + NetworkACLList.delete, acl, apiclient=self.domain_admin_apiclient) + + self.debug("Deleting ACL list as a root admin, should work.") + acl.delete(apiclient=self.admin_apiclient) + self.cleanup.remove(acl) + + # Verify if number of ACLs is equal to two, i.e. the number of default ACLs `default_allow` and `default_deny` + number_of_acls = NetworkACLList.list(apiclient=self.admin_apiclient) + self.assertEqual(len(number_of_acls), 2) + + return diff --git a/test/integration/smoke/test_network.py b/test/integration/smoke/test_network.py index 56b434f3241..8f3f4f533dd 100644 --- a/test/integration/smoke/test_network.py +++ b/test/integration/smoke/test_network.py @@ -871,7 +871,7 @@ class TestReleaseIP(cloudstackTestCase): @attr(tags=["advanced", "advancedns", "smoke", "dvs"], required_hardware="false") def test_releaseIP(self): - """Test for release public IP address""" + """Test for release public IP address using the ID""" logger.debug("Deleting Public IP : %s" % self.ip_addr.id) @@ -930,6 +930,66 @@ class TestReleaseIP(cloudstackTestCase): ) return + @attr(tags=["advanced", "advancedns", "smoke", "dvs"], required_hardware="false") + def test_releaseIP_using_IP(self): + """Test for release public IP address using the address""" + + logger.debug("Deleting Public IP : %s" % self.ip_addr.ipaddress) + self.ip_address.delete_by_ip(self.apiclient) + + retriesCount = 10 + isIpAddressDisassociated = False + while retriesCount > 0: + listResponse = list_publicIP( + self.apiclient, + id=self.ip_addr.id, + state="Allocated" + ) + if listResponse is None: + isIpAddressDisassociated = True + break + retriesCount -= 1 + time.sleep(60) + # End while + + self.assertTrue( + isIpAddressDisassociated, + "Failed to disassociate IP address") + + # ListPortForwardingRules should not list + # associated rules with Public IP address + try: + list_nat_rule = list_nat_rules( + self.apiclient, + id=self.nat_rule.id + ) + logger.debug("List NAT Rule response" + str(list_nat_rule)) + except CloudstackAPIException: + logger.debug("Port Forwarding Rule is deleted") + + # listLoadBalancerRules should not list + # associated rules with Public IP address + try: + list_lb_rule = list_lb_rules( + self.apiclient, + id=self.lb_rule.id + ) + logger.debug("List LB Rule response" + str(list_lb_rule)) + except CloudstackAPIException: + logger.debug("Port Forwarding Rule is deleted") + + # SSH Attempt though public IP should fail + with self.assertRaises(Exception): + SshClient( + self.ip_addr.ipaddress, + self.services["natrule"]["publicport"], + self.virtual_machine.username, + self.virtual_machine.password, + retries=2, + delay=0 + ) + return + class TestDeleteAccount(cloudstackTestCase): diff --git a/test/integration/smoke/test_object_stores.py b/test/integration/smoke/test_object_stores.py new file mode 100644 index 00000000000..710fce646e1 --- /dev/null +++ b/test/integration/smoke/test_object_stores.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" BVT tests for Object Storage Pool""" + +#Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from nose.plugins.attrib import attr +from marvin.lib.base import (ObjectStoragePool) +from marvin.lib.utils import (cleanup_resources) + +_multiprocess_shared_ = True + +class TestObjectStore(cloudstackTestCase): + + def setUp(self): + self.services = self.testClient.getParsedTestDataConfig() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + #Clean up, terminate the created resources + cleanup_resources(self.apiclient, self.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["smoke"], required_hardware="false") + def test_01_create_object_store(self): + """Test to create object store + """ + + object_store = ObjectStoragePool.create( + self.apiclient, + "testOS-10", + "http://192.168.0.1", + "Simulator", + None + ) + + self.debug("Created Object Store with ID: %s" % object_store.id) + + list_object_stores_response = ObjectStoragePool.list( + self.apiclient, + id=object_store.id + ) + + self.assertNotEqual( + len(list_object_stores_response), + 0, + "Check List Object Store response" + ) + + object_store_response = list_object_stores_response[0] + self.assertEqual( + "Simulator", + object_store_response.providername, + "Check Provider of the created Object Store" + ) + self.assertEqual( + "testOS-10", + object_store_response.name, + "Check Name of the created Object Store" + ) + self.assertEqual( + "http://192.168.0.1", + object_store_response.url, + "Check URL of the created Object Store" + ) + + object_store.update( + self.apiclient, + name="updated_name" + ) + + list_object_stores_response_updated = ObjectStoragePool.list( + self.apiclient, + id=object_store.id + ) + + object_store_response_updated = list_object_stores_response_updated[0] + + self.assertEqual( + "updated_name", + object_store_response_updated.name, + "Check Name of the updated Object Store name" + ) + + self.cleanup.append(object_store) + + return diff --git a/test/integration/smoke/test_quarantined_ips.py b/test/integration/smoke/test_quarantined_ips.py new file mode 100644 index 00000000000..42349fd2a53 --- /dev/null +++ b/test/integration/smoke/test_quarantined_ips.py @@ -0,0 +1,329 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import time + +from nose.plugins.attrib import attr + +from marvin.cloudstackAPI import updateConfiguration +from marvin.cloudstackException import CloudstackAPIException +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.base import Network, NetworkOffering, VpcOffering, VPC, PublicIPAddress +from marvin.lib.common import get_domain, get_zone + + +class Services: + """ Test Quarantine for public IPs + """ + + def __init__(self): + self.services = { + "root_domain": { + "name": "ROOT", + }, + "domain_admin": { + "username": "Domain admin", + "roletype": 2, + }, + "root_admin": { + "username": "Root admin", + "roletype": 1, + }, + "domain_vpc": { + "name": "domain-vpc", + "displaytext": "domain-vpc", + "cidr": "10.1.1.0/24", + }, + "domain_network": { + "name": "domain-network", + "displaytext": "domain-network", + }, + "root_vpc": { + "name": "root-vpc", + "displaytext": "root-vpc", + "cidr": "10.2.1.0/24", + }, + "root_network": { + "name": "root-network", + "displaytext": "root-network", + } + } + + +class TestQuarantineIPs(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super(TestQuarantineIPs, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + + cls.services = Services().services + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + return + + def setUp(self): + self.domain_admin_apiclient = self.testClient.getUserApiClient(self.services["domain_admin"]["username"], + self.services["root_domain"]["name"], + self.services["domain_admin"]["roletype"]) + + self.admin_apiclient = self.testClient.getUserApiClient(self.services["root_admin"]["username"], + self.services["root_domain"]["name"], + self.services["root_admin"]["roletype"]) + + """ + Set public.ip.address.quarantine.duration to 60 minutes + """ + update_configuration_cmd = updateConfiguration.updateConfigurationCmd() + update_configuration_cmd.name = "public.ip.address.quarantine.duration" + update_configuration_cmd.value = "1" + self.apiclient.updateConfiguration(update_configuration_cmd) + + self.cleanup = [] + return + + def tearDown(self): + """ + Reset public.ip.address.quarantine.duration to 0 minutes + """ + update_configuration_cmd = updateConfiguration.updateConfigurationCmd() + update_configuration_cmd.name = "public.ip.address.quarantine.duration" + update_configuration_cmd.value = "0" + self.apiclient.updateConfiguration(update_configuration_cmd) + + super(TestQuarantineIPs, self).tearDown() + + def create_vpc(self, api_client, services): + # Get network offering + network_offering = NetworkOffering.list(api_client, name="DefaultIsolatedNetworkOfferingForVpcNetworks") + self.assertTrue(network_offering is not None and len(network_offering) > 0, "No VPC network offering") + + # Getting VPC offering + vpc_offering = VpcOffering.list(api_client, name="Default VPC offering") + self.assertTrue(vpc_offering is not None and len(vpc_offering) > 0, "No VPC offerings found") + + # Creating VPC + vpc = VPC.create( + apiclient=api_client, + services=services, + networkDomain="vpc.networkacl", + vpcofferingid=vpc_offering[0].id, + zoneid=self.zone.id, + domainid=self.domain.id, + start=False + ) + + self.cleanup.append(vpc) + self.assertTrue(vpc is not None, "VPC creation failed") + return vpc + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_only_owner_can_allocate_ip_in_quarantine_vpc(self): + """ Test allocate IP in quarantine to VPC. + """ + # Creating Domain Admin VPC + domain_vpc = self.create_vpc(self.domain_admin_apiclient, self.services["domain_vpc"]) + + # Allocating source nat first + PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + vpcid=domain_vpc.id) + + # Getting available public IP address + ip_address = PublicIPAddress.list(self.domain_admin_apiclient, state="Free", listall=True)[0].ipaddress + + self.debug( + f"creating public address with zone {self.zone.id} and vpc id {domain_vpc.id} and ip address {ip_address}.") + # Associating public IP address to Domain Admin account + public_ip = PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + vpcid=domain_vpc.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip, "Failed to Associate IP Address") + self.assertEqual(public_ip.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") + + self.debug(f"Disassociating public IP {public_ip.ipaddress.ipaddress}.") + public_ip.delete(self.domain_admin_apiclient) + + # Creating Root Admin VPC + root_vpc = self.create_vpc(self.admin_apiclient, self.services["root_vpc"]) + + self.debug(f"Trying to allocate the same IP address {ip_address} that is still in quarantine.") + + with self.assertRaises(CloudstackAPIException) as exception: + PublicIPAddress.create(self.admin_apiclient, + zoneid=self.zone.id, + vpcid=root_vpc.id, + ipaddress=ip_address) + self.assertIn(f"Failed to allocate public IP address [{ip_address}] as it is in quarantine.", + exception.exception.errorMsg) + + # Owner should be able to allocate its IP in quarantine + public_ip = PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + vpcid=domain_vpc.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip, "Failed to Associate IP Address") + self.assertEqual(public_ip.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_another_user_can_allocate_ip_after_quarantined_has_ended_vpc(self): + """ Test allocate IP to VPC after quarantine has ended. + """ + # Creating Domain Admin VPC + domain_vpc = self.create_vpc(self.domain_admin_apiclient, self.services["domain_vpc"]) + + # Allocating source nat first + PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + vpcid=domain_vpc.id) + + # Getting available public IP address + ip_address = PublicIPAddress.list(self.domain_admin_apiclient, state="Free", listall=True)[0].ipaddress + + self.debug( + f"creating public address with zone {self.zone.id} and vpc id {domain_vpc.id} and ip address {ip_address}.") + # Associating public IP address to Domain Admin account + public_ip = PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + vpcid=domain_vpc.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip, "Failed to Associate IP Address") + self.assertEqual(public_ip.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") + + self.debug(f"Disassociating public IP {public_ip.ipaddress.ipaddress}.") + public_ip.delete(self.domain_admin_apiclient) + + # Creating Root Admin VPC + root_vpc = self.create_vpc(self.admin_apiclient, self.services["root_vpc"]) + + self.debug(f"Trying to allocate the same IP address {ip_address} after the quarantine duration.") + + time.sleep(60) + + public_ip_2 = PublicIPAddress.create(self.admin_apiclient, + zoneid=self.zone.id, + vpcid=root_vpc.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip_2, "Failed to Associate IP Address") + self.assertEqual(public_ip_2.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_only_owner_can_allocate_ip_in_quarantine_network(self): + """ Test allocate IP in quarantine to network. + """ + network_offering = NetworkOffering.list(self.domain_admin_apiclient, + name="DefaultIsolatedNetworkOfferingWithSourceNatService") + domain_network = Network.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + services=self.services["domain_network"], + networkofferingid=network_offering[0].id) + self.cleanup.append(domain_network) + + # Allocating source nat first + PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + networkid=domain_network.id) + + # Getting available public IP address + ip_address = PublicIPAddress.list(self.domain_admin_apiclient, state="Free", listall=True)[0].ipaddress + + self.debug( + f"creating public address with zone {self.zone.id} and network id {domain_network.id} and ip address {ip_address}.") + # Associating public IP address to Domain Admin account + public_ip = PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + networkid=domain_network.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip, "Failed to Associate IP Address") + self.assertEqual(public_ip.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") + + self.debug(f"Disassociating public IP {public_ip.ipaddress.ipaddress}.") + public_ip.delete(self.domain_admin_apiclient) + + # Creating Root Admin network + root_network = Network.create(self.admin_apiclient, + zoneid=self.zone.id, + services=self.services["root_network"], + networkofferingid=network_offering[0].id) + self.cleanup.append(root_network) + self.debug(f"Trying to allocate the same IP address {ip_address} that is still in quarantine.") + + with self.assertRaises(CloudstackAPIException) as exception: + PublicIPAddress.create(self.admin_apiclient, + zoneid=self.zone.id, + networkid=root_network.id, + ipaddress=ip_address) + self.assertIn(f"Failed to allocate public IP address [{ip_address}] as it is in quarantine.", + exception.exception.errorMsg) + + # Owner should be able to allocate its IP in quarantine + public_ip = PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + networkid=domain_network.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip, "Failed to Associate IP Address") + self.assertEqual(public_ip.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_another_user_can_allocate_ip_after_quarantined_has_ended_network(self): + """ Test allocate IP to network after quarantine has ended. + """ + network_offering = NetworkOffering.list(self.domain_admin_apiclient, + name="DefaultIsolatedNetworkOfferingWithSourceNatService") + domain_network = Network.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + services=self.services["domain_network"], + networkofferingid=network_offering[0].id) + self.cleanup.append(domain_network) + # Allocating source nat first + PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + networkid=domain_network.id) + + # Getting available public IP address + ip_address = PublicIPAddress.list(self.domain_admin_apiclient, state="Free", listall=True)[0].ipaddress + + self.debug( + f"creating public address with zone {self.zone.id} and network id {domain_network.id} and ip address {ip_address}.") + # Associating public IP address to Domain Admin account + public_ip = PublicIPAddress.create(self.domain_admin_apiclient, + zoneid=self.zone.id, + networkid=domain_network.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip, "Failed to Associate IP Address") + self.assertEqual(public_ip.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") + + self.debug(f"Disassociating public IP {public_ip.ipaddress.ipaddress}.") + public_ip.delete(self.domain_admin_apiclient) + + # Creating Root Admin VPC + root_network = Network.create(self.admin_apiclient, + zoneid=self.zone.id, + services=self.services["root_network"], + networkofferingid=network_offering[0].id) + self.cleanup.append(root_network) + + self.debug(f"Trying to allocate the same IP address {ip_address} after the quarantine duration.") + + time.sleep(60) + + public_ip_2 = PublicIPAddress.create(self.admin_apiclient, + zoneid=self.zone.id, + networkid=domain_network.id, + ipaddress=ip_address) + self.assertIsNotNone(public_ip_2, "Failed to Associate IP Address") + self.assertEqual(public_ip_2.ipaddress.ipaddress, ip_address, "Associated IP is not same as specified") diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 56f631d4d51..301aece23bd 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -259,7 +259,18 @@ known_categories = { 'importVsphereStoragePolicies' : 'vSphere storage policies', 'listVsphereStoragePolicies' : 'vSphere storage policies', 'ConsoleEndpoint': 'Console Endpoint', - 'Shutdown': 'Shutdown' + 'listQuarantinedIp': 'IP Quarantine', + 'updateQuarantinedIp': 'IP Quarantine', + 'removeQuarantinedIp': 'IP Quarantine', + 'Shutdown': 'Shutdown', + 'addObjectStoragePool': 'Object Store', + 'listObjectStoragePools': 'Object Store', + 'deleteObjectStoragePool': 'Object Store', + 'updateObjectStoragePool': 'Object Store', + 'createBucket': 'Object Store', + 'updateBucket': 'Object Store', + 'deleteBucket': 'Object Store', + 'listBuckets': 'Object Store' } diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index b0fd3198301..39af1d4303b 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -1854,7 +1854,7 @@ class PublicIPAddress: if zoneid: cmd.zoneid = zoneid - elif "zoneid" in services: + elif services and "zoneid" in services: cmd.zoneid = services["zoneid"] if domainid: @@ -1879,12 +1879,19 @@ class PublicIPAddress: return PublicIPAddress(apiclient.associateIpAddress(cmd).__dict__) def delete(self, apiclient): - """Dissociate Public IP address""" + """Dissociate Public IP address using the given ID""" cmd = disassociateIpAddress.disassociateIpAddressCmd() cmd.id = self.ipaddress.id apiclient.disassociateIpAddress(cmd) return + def delete_by_ip(self, apiclient): + """Dissociate Public IP address using the given IP address""" + cmd = disassociateIpAddress.disassociateIpAddressCmd() + cmd.ipaddress = self.ipaddress.ipaddress + apiclient.disassociateIpAddress(cmd) + return + @classmethod def list(cls, apiclient, **kwargs): """List all Public IPs matching criteria""" @@ -5105,7 +5112,7 @@ class VPC: @classmethod def create(cls, apiclient, services, vpcofferingid, zoneid, networkDomain=None, account=None, - domainid=None, **kwargs): + domainid=None, start=True, **kwargs): """Creates the virtual private connection (VPC)""" cmd = createVPC.createVPCCmd() @@ -5113,6 +5120,7 @@ class VPC: cmd.displaytext = "-".join([services["displaytext"], random_gen()]) cmd.vpcofferingid = vpcofferingid cmd.zoneid = zoneid + cmd.start = start if "cidr" in services: cmd.cidr = services["cidr"] if account: @@ -7064,3 +7072,82 @@ class VnfAppliance: cmd.expunge = expunge [setattr(cmd, k, v) for k, v in list(kwargs.items())] apiclient.destroyVirtualMachine(cmd) + +class ObjectStoragePool: + + def __init__(self, items): + self.__dict__.update(items) + + """Manage Object Stores""" + @classmethod + def create(cls, apiclient, name, url, provider, services=None): + """Add Object Store""" + cmd = addObjectStoragePool.addObjectStoragePoolCmd() + cmd.name = name + cmd.url = url + cmd.provider = provider + if services: + if "details" in services: + cmd.details = services["details"] + + return ObjectStoragePool(apiclient.addObjectStoragePool(cmd).__dict__) + + def delete(self, apiclient): + """Delete Object Store""" + cmd = deleteObjectStoragePool.deleteObjectStoragePoolCmd() + cmd.id = self.id + apiclient.deleteObjectStoragePool(cmd) + + @classmethod + def list(cls, apiclient, **kwargs): + cmd = listObjectStoragePools.listObjectStoragePoolsCmd() + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + if 'account' in list(kwargs.keys()) and 'domainid' in list(kwargs.keys()): + cmd.listall = True + return (apiclient.listObjectStoragePools(cmd)) + + def update(self, apiclient, **kwargs): + """Update the Object Store""" + + cmd = updateObjectStoragePool.updateObjectStoragePoolCmd() + cmd.id = self.id + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + return apiclient.updateObjectStoragePool(cmd) + +class Bucket: + """Manage Bucket Life cycle""" + + def __init__(self, items): + self.__dict__.update(items) + + @classmethod + def create(cls, apiclient, name, objectstorageid, **kwargs): + """Create Bucket""" + cmd = createBucket.createBucketCmd() + cmd.name = name + cmd.objectstorageid = objectstorageid + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + + return Bucket(apiclient.createBucket(cmd).__dict__) + + def delete(self, apiclient): + """Delete Bucket""" + cmd = deleteBucket.deleteBucketCmd() + cmd.id = self.id + apiclient.deleteBucket(cmd) + + @classmethod + def list(cls, apiclient, **kwargs): + cmd = listBuckets.listBucketsCmd() + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + if 'account' in list(kwargs.keys()) and 'domainid' in list(kwargs.keys()): + cmd.listall = True + return (apiclient.listBuckets(cmd)) + + def update(self, apiclient, **kwargs): + """Update Bucket""" + + cmd = updateBucket.updateBucketCmd() + cmd.id = self.id + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + return apiclient.updateBucket(cmd) diff --git a/ui/package-lock.json b/ui/package-lock.json index 0cbf08bd4a4..d0e0744f3ca 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -113,8 +113,7 @@ "version": "0.5.4", "resolved": "https://registry.npmjs.org/@apollographql/apollo-tools/-/apollo-tools-0.5.4.tgz", "integrity": "sha512-shM3q7rUbNyXVVRkQJQseXv6bnYM3BUma/eZhwXR4xsuM+bqWnJKvW7SAfRjP7LuSCocrexa5AXhjjawNHrIlw==", - "dev": true, - "requires": {} + "dev": true }, "@apollographql/graphql-playground-html": { "version": "1.6.27", @@ -1534,8 +1533,7 @@ "@fortawesome/vue-fontawesome": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@fortawesome/vue-fontawesome/-/vue-fontawesome-3.0.3.tgz", - "integrity": "sha512-KCPHi9QemVXGMrfuwf3nNnNo129resAIQWut9QTAMXmXqL2ErABC6ohd2yY5Ipq0CLWNbKHk8TMdTXL/Zf3ZhA==", - "requires": {} + "integrity": "sha512-KCPHi9QemVXGMrfuwf3nNnNo129resAIQWut9QTAMXmXqL2ErABC6ohd2yY5Ipq0CLWNbKHk8TMdTXL/Zf3ZhA==" }, "@gar/promisify": { "version": "1.1.3", @@ -3349,8 +3347,7 @@ "version": "4.5.19", "resolved": "https://registry.npmjs.org/@vue/cli-plugin-vuex/-/cli-plugin-vuex-4.5.19.tgz", "integrity": "sha512-DUmfdkG3pCdkP7Iznd87RfE9Qm42mgp2hcrNcYQYSru1W1gX2dG/JcW8bxmeGSa06lsxi9LEIc/QD1yPajSCZw==", - "dev": true, - "requires": {} + "dev": true }, "@vue/cli-service": { "version": "4.5.19", @@ -3861,8 +3858,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/@vue/preload-webpack-plugin/-/preload-webpack-plugin-1.1.2.tgz", "integrity": "sha512-LIZMuJk38pk9U9Ur4YzHjlIyMuxPlACdBIHH9/nGYVTsaGKOSnSuELiE8vS9wa+dJpIYspYUOqk+L1Q4pgHQHQ==", - "dev": true, - "requires": {} + "dev": true }, "@vue/reactivity": { "version": "3.2.37", @@ -3921,8 +3917,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/@vue/test-utils/-/test-utils-2.0.0.tgz", "integrity": "sha512-zL5kygNq7hONrO1CzaUGprEAklAX+pH8J1MPMCU3Rd2xtSYkZ+PmKU3oEDRg8VAGdL5lNJHzDgrud5amFPtirw==", - "dev": true, - "requires": {} + "dev": true }, "@vue/web-component-wrapper": { "version": "1.3.0", @@ -3934,6 +3929,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", "integrity": "sha512-C6wW5L+b7ogSDVqymbkkvuW9kruN//YisMED04xzeBBqjHa2FYnmvOlS6Xj68xWQRgWvI9cIglsjFowH/RJyEA==", + "dev": true, "requires": { "@webassemblyjs/helper-module-context": "1.9.0", "@webassemblyjs/helper-wasm-bytecode": "1.9.0", @@ -3943,22 +3939,26 @@ "@webassemblyjs/floating-point-hex-parser": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.9.0.tgz", - "integrity": "sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA==" + "integrity": "sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA==", + "dev": true }, "@webassemblyjs/helper-api-error": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.9.0.tgz", - "integrity": "sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw==" + "integrity": "sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw==", + "dev": true }, "@webassemblyjs/helper-buffer": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.9.0.tgz", - "integrity": "sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA==" + "integrity": "sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA==", + "dev": true }, "@webassemblyjs/helper-code-frame": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.9.0.tgz", "integrity": "sha512-ERCYdJBkD9Vu4vtjUYe8LZruWuNIToYq/ME22igL+2vj2dQ2OOujIZr3MEFvfEaqKoVqpsFKAGsRdBSBjrIvZA==", + "dev": true, "requires": { "@webassemblyjs/wast-printer": "1.9.0" } @@ -3966,12 +3966,14 @@ "@webassemblyjs/helper-fsm": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.9.0.tgz", - "integrity": "sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw==" + "integrity": "sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw==", + "dev": true }, "@webassemblyjs/helper-module-context": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.9.0.tgz", "integrity": "sha512-MJCW8iGC08tMk2enck1aPW+BE5Cw8/7ph/VGZxwyvGbJwjktKkDK7vy7gAmMDx88D7mhDTCNKAW5tED+gZ0W8g==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0" } @@ -3979,12 +3981,14 @@ "@webassemblyjs/helper-wasm-bytecode": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.9.0.tgz", - "integrity": "sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw==" + "integrity": "sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw==", + "dev": true }, "@webassemblyjs/helper-wasm-section": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.9.0.tgz", "integrity": "sha512-XnMB8l3ek4tvrKUUku+IVaXNHz2YsJyOOmz+MMkZvh8h1uSJpSen6vYnw3IoQ7WwEuAhL8Efjms1ZWjqh2agvw==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-buffer": "1.9.0", @@ -3996,6 +4000,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.9.0.tgz", "integrity": "sha512-dcX8JuYU/gvymzIHc9DgxTzUUTLexWwt8uCTWP3otys596io0L5aW02Gb1RjYpx2+0Jus1h4ZFqjla7umFniTg==", + "dev": true, "requires": { "@xtuc/ieee754": "^1.2.0" } @@ -4004,6 +4009,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.9.0.tgz", "integrity": "sha512-ENVzM5VwV1ojs9jam6vPys97B/S65YQtv/aanqnU7D8aSoHFX8GyhGg0CMfyKNIHBuAVjy3tlzd5QMMINa7wpw==", + "dev": true, "requires": { "@xtuc/long": "4.2.2" } @@ -4011,12 +4017,14 @@ "@webassemblyjs/utf8": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.9.0.tgz", - "integrity": "sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w==" + "integrity": "sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w==", + "dev": true }, "@webassemblyjs/wasm-edit": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.9.0.tgz", "integrity": "sha512-FgHzBm80uwz5M8WKnMTn6j/sVbqilPdQXTWraSjBwFXSYGirpkSWE2R9Qvz9tNiTKQvoKILpCuTjBKzOIm0nxw==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-buffer": "1.9.0", @@ -4032,6 +4040,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.9.0.tgz", "integrity": "sha512-cPE3o44YzOOHvlsb4+E9qSqjc9Qf9Na1OO/BHFy4OI91XDE14MjFN4lTMezzaIWdPqHnsTodGGNP+iRSYfGkjA==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-wasm-bytecode": "1.9.0", @@ -4044,6 +4053,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.9.0.tgz", "integrity": "sha512-Qkjgm6Anhm+OMbIL0iokO7meajkzQD71ioelnfPEj6r4eOFuqm4YC3VBPqXjFyyNwowzbMD+hizmprP/Fwkl2A==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-buffer": "1.9.0", @@ -4055,6 +4065,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.9.0.tgz", "integrity": "sha512-9+wkMowR2AmdSWQzsPEjFU7njh8HTO5MqO8vjwEHuM+AMHioNqSBONRdr0NQQ3dVQrzp0s8lTcYqzUdb7YgELA==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-api-error": "1.9.0", @@ -4068,6 +4079,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.9.0.tgz", "integrity": "sha512-qsqSAP3QQ3LyZjNC/0jBJ/ToSxfYJ8kYyuiGvtn/8MK89VrNEfwj7BPQzJVHi0jGTRK2dGdJ5PRqhtjzoww+bw==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/floating-point-hex-parser": "1.9.0", @@ -4081,6 +4093,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.9.0.tgz", "integrity": "sha512-2J0nE95rHXHyQ24cWjMKJ1tqB/ds8z/cyeOZxJhcb+rW+SQASVjuznUSmdz5GpVJTzU8JkhYut0D3siFDD6wsA==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/wast-parser": "1.9.0", @@ -4099,12 +4112,20 @@ "@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", - "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==" + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "dev": true }, "@xtuc/long": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", - "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "dev": true + }, + "@zxing/text-encoding": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", + "integrity": "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==", + "optional": true }, "abab": { "version": "2.0.6", @@ -4130,7 +4151,8 @@ "acorn": { "version": "6.4.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", - "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==" + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", + "dev": true }, "acorn-globals": { "version": "4.3.4", @@ -4146,8 +4168,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "requires": {} + "dev": true }, "acorn-walk": { "version": "6.2.0", @@ -4203,13 +4224,13 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz", "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==", - "requires": {} + "dev": true }, "ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", - "requires": {} + "dev": true }, "alphanum-sort": { "version": "1.0.2", @@ -4377,14 +4398,14 @@ "resolved": "https://registry.npmjs.org/antd-theme-generator/-/antd-theme-generator-1.2.11.tgz", "integrity": "sha512-7A3lXyLb7eD7MXK7aSgZZ4DxQEdhZwyKhzIm70orUZPQJ8N8TWhZphyOWSGCe8yUqGQhi8PcpM2pLmTriZyKBw==", "requires": { - "glob": "*", - "hash.js": "*", - "less": "*", + "glob": "^7.1.3", + "hash.js": "^1.1.5", + "less": "^3.9.0", "less-bundle-promise": "^1.0.11", - "less-plugin-npm-import": "*", - "postcss": "*", + "less-plugin-npm-import": "^2.1.0", + "postcss": "^6.0.21", "postcss-less": "^3.1.4", - "strip-css-comments": "*" + "strip-css-comments": "^4.1.0" }, "dependencies": { "ansi-styles": { @@ -4985,7 +5006,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "devOptional": true, + "dev": true, "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -5107,8 +5128,7 @@ "version": "2.5.0", "resolved": "https://registry.npmjs.org/apollo-server-errors/-/apollo-server-errors-2.5.0.tgz", "integrity": "sha512-lO5oTjgiC3vlVg2RKr3RiXIIQ5pGXBFxYGGUkKDhTud3jMIhs+gel8L8zsEjKaKxkjHhCQAA/bcEfYiKkGQIvA==", - "dev": true, - "requires": {} + "dev": true }, "apollo-server-express": { "version": "2.25.4", @@ -5225,17 +5245,20 @@ "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", - "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==" + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", + "dev": true }, "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==" + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true }, "arr-union": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", - "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==" + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "dev": true }, "array-equal": { "version": "1.0.0", @@ -5291,7 +5314,8 @@ "array-unique": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", - "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==" + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", + "dev": true }, "array.prototype.flat": { "version": "1.3.0", @@ -5354,6 +5378,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.0.tgz", "integrity": "sha512-EDsgawzwoun2CZkCgtxJbv392v4nbk9XDD06zI+kQYoBM/3RBWLlEyJARDOmhAAosBjWACEkKL6S+lIZtcAubA==", + "dev": true, "requires": { "object-assign": "^4.1.1", "util": "0.10.3" @@ -5362,12 +5387,14 @@ "inherits": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", - "integrity": "sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==" + "integrity": "sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==", + "dev": true }, "util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha512-5KiHfsmkqacuKjkRkdV7SsfDJ2EGiPsK92s2MhNSY0craxjTdKTtqKsJaCWp4LW33ZZ0OPUv1WO/TFvNQRiQxQ==", + "dev": true, "requires": { "inherits": "2.0.1" } @@ -5382,7 +5409,8 @@ "assign-symbols": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", - "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==" + "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", + "dev": true }, "ast-types": { "version": "0.13.3", @@ -5409,7 +5437,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==", - "devOptional": true + "dev": true }, "async-limiter": { "version": "1.0.1", @@ -5447,7 +5475,8 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true }, "autoprefixer": { "version": "9.8.8", @@ -5464,6 +5493,11 @@ "postcss-value-parser": "^4.1.0" } }, + "available-typed-arrays": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", + "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -5530,8 +5564,7 @@ "version": "7.0.0-bridge.0", "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", "integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==", - "dev": true, - "requires": {} + "dev": true }, "babel-eslint": { "version": "10.1.0", @@ -6140,6 +6173,7 @@ "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, "requires": { "cache-base": "^1.0.1", "class-utils": "^0.3.5", @@ -6154,6 +6188,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, "requires": { "is-descriptor": "^1.0.0" } @@ -6162,6 +6197,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -6170,6 +6206,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -6178,6 +6215,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -6189,7 +6227,8 @@ "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true }, "batch": { "version": "0.6.1", @@ -6226,13 +6265,13 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "devOptional": true + "dev": true }, "bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "devOptional": true, + "dev": true, "requires": { "file-uri-to-path": "1.0.0" } @@ -6247,10 +6286,31 @@ "safe-buffer": "^5.1.1" } }, + "block-stream2": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz", + "integrity": "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==", + "requires": { + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true }, "bn.js": { "version": "5.2.1", @@ -6432,6 +6492,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, "requires": { "arr-flatten": "^1.1.0", "array-unique": "^0.3.2", @@ -6449,6 +6510,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -6460,6 +6522,11 @@ "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" }, + "browser-or-node": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-1.3.0.tgz", + "integrity": "sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg==" + }, "browser-process-hrtime": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", @@ -6563,6 +6630,7 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "dev": true, "requires": { "pako": "~1.0.5" } @@ -6626,8 +6694,7 @@ "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "dev": true + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" }, "buffer-fill": { "version": "1.0.0", @@ -6638,7 +6705,8 @@ "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true }, "buffer-indexof": { "version": "1.1.1", @@ -6660,7 +6728,8 @@ "builtin-status-codes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", - "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==" + "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", + "dev": true }, "builtins": { "version": "1.0.3", @@ -6711,6 +6780,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, "requires": { "collection-visit": "^1.0.0", "component-emitter": "^1.2.1", @@ -6821,7 +6891,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "requires": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -6958,8 +7027,7 @@ "chartjs-adapter-moment": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/chartjs-adapter-moment/-/chartjs-adapter-moment-1.0.0.tgz", - "integrity": "sha512-PqlerEvQcc5hZLQ/NQWgBxgVQ4TRdvkW3c/t+SUEQSj78ia3hgLkf2VZ2yGJtltNbEEFyYGm+cA6XXevodYvWA==", - "requires": {} + "integrity": "sha512-PqlerEvQcc5hZLQ/NQWgBxgVQ4TRdvkW3c/t+SUEQSj78ia3hgLkf2VZ2yGJtltNbEEFyYGm+cA6XXevodYvWA==" }, "check-types": { "version": "8.0.3", @@ -6971,7 +7039,7 @@ "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "devOptional": true, + "dev": true, "requires": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -6987,7 +7055,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "devOptional": true, + "dev": true, "requires": { "fill-range": "^7.0.1" } @@ -6996,7 +7064,7 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "devOptional": true, + "dev": true, "requires": { "to-regex-range": "^5.0.1" } @@ -7005,7 +7073,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "devOptional": true, + "dev": true, "requires": { "is-glob": "^4.0.1" } @@ -7014,13 +7082,13 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "devOptional": true + "dev": true }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "devOptional": true, + "dev": true, "requires": { "is-number": "^7.0.0" } @@ -7035,7 +7103,8 @@ "chrome-trace-event": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", - "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==" + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", + "dev": true }, "ci-info": { "version": "2.0.0", @@ -7060,6 +7129,7 @@ "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, "requires": { "arr-union": "^3.1.0", "define-property": "^0.2.5", @@ -7071,6 +7141,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -7364,6 +7435,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==", + "dev": true, "requires": { "map-visit": "^1.0.0", "object-visit": "^1.0.0" @@ -7440,12 +7512,14 @@ "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", - "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==" + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", - "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", + "dev": true }, "compressible": { "version": "2.0.18", @@ -7508,6 +7582,7 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "dev": true, "requires": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", @@ -7578,7 +7653,8 @@ "console-browserify": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", - "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==" + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==", + "dev": true }, "console-control-strings": { "version": "1.1.0", @@ -7597,7 +7673,8 @@ "constants-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", - "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==" + "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==", + "dev": true }, "content-disposition": { "version": "0.5.4", @@ -7656,6 +7733,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "dev": true, "requires": { "aproba": "^1.1.1", "fs-write-stream-atomic": "^1.0.8", @@ -7669,6 +7747,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -7677,6 +7756,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -7686,7 +7766,8 @@ "copy-descriptor": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", - "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==" + "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==", + "dev": true }, "copy-to-clipboard": { "version": "3.3.1", @@ -8289,7 +8370,8 @@ "cyclist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz", - "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==" + "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==", + "dev": true }, "dashdash": { "version": "1.14.1", @@ -8375,7 +8457,8 @@ "decode-uri-component": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==" + "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==", + "dev": true }, "decompress": { "version": "4.2.1", @@ -8686,6 +8769,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, "requires": { "is-descriptor": "^1.0.2", "isobject": "^3.0.1" @@ -8695,6 +8779,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -8703,6 +8788,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -8711,6 +8797,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -8946,7 +9033,8 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==" + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "dev": true }, "domelementtype": { "version": "1.3.1", @@ -9186,6 +9274,7 @@ "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "dev": true, "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -9326,6 +9415,7 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.5.0.tgz", "integrity": "sha512-Nv9m36S/vxpsI+Hc4/ZGRs0n9mXqSWGGq49zxb/cJfPAQMbUtttJAlNPS4AQzaBdw/pKskw5bMbekT/Y7W/Wlg==", + "dev": true, "requires": { "graceful-fs": "^4.1.2", "memory-fs": "^0.5.0", @@ -9336,6 +9426,7 @@ "version": "0.5.0", "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.5.0.tgz", "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "dev": true, "requires": { "errno": "^0.1.3", "readable-stream": "^2.0.1" @@ -9374,6 +9465,7 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, "requires": { "prr": "~1.0.1" } @@ -9453,6 +9545,11 @@ "is-symbol": "^1.0.2" } }, + "es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==" + }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -9669,8 +9766,7 @@ "version": "14.1.1", "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-14.1.1.tgz", "integrity": "sha512-Z9B+VR+JIXRxz21udPTL9HpFMyoMUEeX1G251EQ6e05WD9aPVtVBn09XUmZ259wCMlCDmYDSZG62Hhm+ZTJcUg==", - "dev": true, - "requires": {} + "dev": true }, "eslint-import-resolver-node": { "version": "0.3.6", @@ -10036,8 +10132,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz", "integrity": "sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ==", - "dev": true, - "requires": {} + "dev": true }, "eslint-plugin-vue": { "version": "7.20.0", @@ -10072,6 +10167,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, "requires": { "esrecurse": "^4.1.0", "estraverse": "^4.1.1" @@ -10137,6 +10233,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, "requires": { "estraverse": "^5.2.0" }, @@ -10144,14 +10241,16 @@ "estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true } } }, "estraverse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==" + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true }, "estree-walker": { "version": "2.0.2", @@ -10185,7 +10284,8 @@ "events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true }, "eventsource": { "version": "2.0.2", @@ -10236,6 +10336,7 @@ "version": "2.1.4", "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "dev": true, "requires": { "debug": "^2.3.3", "define-property": "^0.2.5", @@ -10250,6 +10351,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, "requires": { "ms": "2.0.0" } @@ -10258,6 +10360,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -10266,6 +10369,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -10273,7 +10377,8 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true } } }, @@ -10456,6 +10561,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", + "dev": true, "requires": { "assign-symbols": "^1.0.0", "is-extendable": "^1.0.1" @@ -10465,6 +10571,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, "requires": { "is-plain-object": "^2.0.4" } @@ -10473,6 +10580,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -10505,6 +10613,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, "requires": { "array-unique": "^0.3.2", "define-property": "^1.0.0", @@ -10520,6 +10629,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, "requires": { "is-descriptor": "^1.0.0" } @@ -10528,6 +10638,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -10536,6 +10647,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -10544,6 +10656,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -10552,6 +10665,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -10604,6 +10718,14 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-xml-parser": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.0.tgz", + "integrity": "sha512-5Wln/SBrtlN37aboiNNFHfSALwLzpUx1vJhDgDVPKKG3JrNe8BWTUoNKqkeKk/HqNbKxC8nEAJaBydq30yHoLA==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -10705,7 +10827,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "devOptional": true + "dev": true }, "filename-reserved-regex": { "version": "2.0.0", @@ -10734,6 +10856,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", + "dev": true, "requires": { "extend-shallow": "^2.0.1", "is-number": "^3.0.0", @@ -10745,12 +10868,18 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } } } }, + "filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==" + }, "finalhandler": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", @@ -10817,6 +10946,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, "requires": { "commondir": "^1.0.1", "make-dir": "^2.0.0", @@ -10827,6 +10957,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, "requires": { "pify": "^4.0.1", "semver": "^5.6.0" @@ -10835,12 +10966,14 @@ "pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true } } }, @@ -10911,6 +11044,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz", "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==", + "dev": true, "requires": { "inherits": "^2.0.3", "readable-stream": "^2.3.6" @@ -10925,7 +11059,6 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dev": true, "requires": { "is-callable": "^1.1.3" } @@ -10933,7 +11066,8 @@ "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", - "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==" + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "dev": true }, "forever-agent": { "version": "0.6.1", @@ -10960,6 +11094,7 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==", + "dev": true, "requires": { "map-cache": "^0.2.2" } @@ -10974,6 +11109,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", + "dev": true, "requires": { "inherits": "^2.0.1", "readable-stream": "^2.0.0" @@ -11020,6 +11156,7 @@ "version": "1.0.10", "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", "integrity": "sha512-gehEzmPn2nAwr39eay+x3X34Ra+M2QlVUTLhkXPjWdeO8RF9kszk116avgBJM3ZyNHgHXBNx+VmPaFC36k0PzA==", + "dev": true, "requires": { "graceful-fs": "^4.1.2", "iferr": "^0.1.5", @@ -11036,6 +11173,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, "optional": true }, "fswin": { @@ -11047,8 +11185,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "function.prototype.name": { "version": "1.1.5", @@ -11120,7 +11257,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", - "dev": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -11168,7 +11304,8 @@ "get-value": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", - "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==" + "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==", + "dev": true }, "getpass": { "version": "0.1.7", @@ -11223,7 +11360,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", "integrity": "sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==", - "devOptional": true, + "dev": true, "requires": { "is-glob": "^3.1.0", "path-dirname": "^1.0.0" @@ -11233,7 +11370,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==", - "devOptional": true, + "dev": true, "requires": { "is-extglob": "^2.1.0" } @@ -11305,6 +11442,27 @@ "delegate": "^3.1.2" } }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "requires": { + "get-intrinsic": "^1.1.3" + }, + "dependencies": { + "get-intrinsic": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", + "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3" + } + } + } + }, "got": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", @@ -11391,8 +11549,7 @@ "version": "0.3.2", "resolved": "https://registry.npmjs.org/graphql-type-json/-/graphql-type-json-0.3.2.tgz", "integrity": "sha512-J+vjof74oMlCWXSvt0DOf2APEdZOCdubEvGDUAlqH//VBYcOYsGgRW7Xzorr44LvkjiuvecWc8fChxuZZbChtg==", - "dev": true, - "requires": {} + "dev": true }, "growly": { "version": "1.3.0", @@ -11442,7 +11599,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -11475,6 +11631,11 @@ "get-intrinsic": "^1.1.1" } }, + "has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" + }, "has-symbol-support-x": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz", @@ -11484,8 +11645,7 @@ "has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" }, "has-to-string-tag-x": { "version": "1.4.1", @@ -11500,7 +11660,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dev": true, "requires": { "has-symbols": "^1.0.2" } @@ -11514,6 +11673,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==", + "dev": true, "requires": { "get-value": "^2.0.6", "has-values": "^1.0.0", @@ -11524,6 +11684,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==", + "dev": true, "requires": { "is-number": "^3.0.0", "kind-of": "^4.0.0" @@ -11533,6 +11694,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -11945,7 +12107,8 @@ "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", - "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==" + "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==", + "dev": true }, "https-proxy-agent": { "version": "5.0.1", @@ -11991,12 +12154,14 @@ "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true }, "iferr": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz", - "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==" + "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==", + "dev": true }, "ignore": { "version": "4.0.6", @@ -12301,6 +12466,7 @@ "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -12309,6 +12475,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -12319,7 +12486,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -12344,7 +12510,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "devOptional": true, + "dev": true, "requires": { "binary-extensions": "^2.0.0" } @@ -12367,8 +12533,7 @@ "is-callable": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", - "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", - "dev": true + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==" }, "is-ci": { "version": "2.0.0", @@ -12405,6 +12570,7 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -12413,6 +12579,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -12432,6 +12599,7 @@ "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", @@ -12441,7 +12609,8 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==" + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true } } }, @@ -12460,13 +12629,14 @@ "is-extendable": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", - "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==" + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true }, "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "devOptional": true + "dev": true }, "is-fullwidth-code-point": { "version": "1.0.0", @@ -12482,11 +12652,19 @@ "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "dev": true }, + "is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, "is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "devOptional": true, + "dev": true, "requires": { "is-extglob": "^2.1.1" } @@ -12526,6 +12704,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -12534,6 +12713,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -12669,6 +12849,14 @@ "has-symbols": "^1.0.2" } }, + "is-typed-array": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", + "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", + "requires": { + "which-typed-array": "^1.1.11" + } + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -12698,12 +12886,14 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==" + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true }, "is-wsl": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz", - "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==" + "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==", + "dev": true }, "is-yarn-global": { "version": "0.3.0", @@ -12729,7 +12919,8 @@ "isobject": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==" + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true }, "isstream": { "version": "0.1.2", @@ -13794,8 +13985,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz", "integrity": "sha512-olV41bKSMm8BdnuMsewT4jqlZ8+3TCARAXjZGT9jcoSnrfUnRCqnMoF9XEeoWjbzObpqF9dRhHQj0Xb9QdF6/w==", - "dev": true, - "requires": {} + "dev": true }, "jest-regex-util": { "version": "24.9.0", @@ -14840,7 +15030,8 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==" + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true }, "json-parse-even-better-errors": { "version": "2.3.1", @@ -14871,6 +15062,11 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "json-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-stream/-/json-stream-1.0.0.tgz", + "integrity": "sha512-H/ZGY0nIAg3QcOwE1QN/rK/Fa7gJn7Ii5obwp6zyPO4xiPNwpIMjqy2gwjBEGqzkF/vSWEIBQCBuN19hYiL6Qg==" + }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", @@ -14931,7 +15127,8 @@ "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true }, "kleur": { "version": "3.0.3", @@ -15224,7 +15421,8 @@ "loader-runner": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.4.0.tgz", - "integrity": "sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==" + "integrity": "sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==", + "dev": true }, "loader-utils": { "version": "2.0.2", @@ -15491,12 +15689,14 @@ "map-cache": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", - "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==" + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==", + "dev": true }, "map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", "integrity": "sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==", + "dev": true, "requires": { "object-visit": "^1.0.0" } @@ -15542,6 +15742,7 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz", "integrity": "sha512-cda4JKCxReDXFXRqOHPQscuIYg1PvxbE2S2GP45rnwfEK+vZaXC8C1OFvdHIbgw0DLzowXGVoxLaAmlgRy14GQ==", + "dev": true, "requires": { "errno": "^0.1.3", "readable-stream": "^2.0.1" @@ -15590,6 +15791,7 @@ "version": "3.1.10", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, "requires": { "arr-diff": "^4.0.0", "array-unique": "^0.3.2", @@ -15758,6 +15960,80 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, + "minio": { + "version": "7.0.33", + "resolved": "https://registry.npmjs.org/minio/-/minio-7.0.33.tgz", + "integrity": "sha512-8wXGH98nZiLPe2xZhMV7UJ+48L1UlhgekxgpUhJWMO1h24TvZ0wUjtIt9e7DfNACopXh1spL8iuDQD7Lrq8Upw==", + "requires": { + "async": "^3.1.0", + "block-stream2": "^2.0.0", + "browser-or-node": "^1.3.0", + "buffer-crc32": "^0.2.13", + "crypto-browserify": "^3.12.0", + "es6-error": "^4.1.1", + "fast-xml-parser": "^4.1.3", + "ipaddr.js": "^2.0.1", + "json-stream": "^1.0.0", + "lodash": "^4.17.21", + "mime-types": "^2.1.14", + "mkdirp": "^0.5.1", + "query-string": "^7.1.1", + "through2": "^3.0.1", + "web-encoding": "^1.1.5", + "xml": "^1.0.0", + "xml2js": "^0.4.15" + }, + "dependencies": { + "async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, + "decode-uri-component": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==" + }, + "ipaddr.js": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", + "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==" + }, + "mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "requires": { + "minimist": "^1.2.6" + } + }, + "query-string": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", + "integrity": "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==", + "requires": { + "decode-uri-component": "^0.2.2", + "filter-obj": "^1.1.0", + "split-on-first": "^1.0.0", + "strict-uri-encode": "^2.0.0" + } + }, + "strict-uri-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==" + }, + "through2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", + "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "2 || 3" + } + } + } + }, "minipass": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", @@ -15831,6 +16107,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz", "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==", + "dev": true, "requires": { "concat-stream": "^1.5.0", "duplexify": "^3.4.2", @@ -15853,6 +16130,7 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, "requires": { "for-in": "^1.0.2", "is-extendable": "^1.0.1" @@ -15862,6 +16140,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, "requires": { "is-plain-object": "^2.0.4" } @@ -15870,6 +16149,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -15905,6 +16185,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", "integrity": "sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==", + "dev": true, "requires": { "aproba": "^1.1.1", "copy-concurrently": "^1.0.0", @@ -15918,6 +16199,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -15926,6 +16208,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -15979,6 +16262,7 @@ "version": "2.16.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.16.0.tgz", "integrity": "sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==", + "dev": true, "optional": true }, "nanoid": { @@ -15990,6 +16274,7 @@ "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, "requires": { "arr-diff": "^4.0.0", "array-unique": "^0.3.2", @@ -16065,7 +16350,8 @@ "neo-async": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true }, "nested-error-stacks": { "version": "2.0.1", @@ -16195,6 +16481,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", "integrity": "sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==", + "dev": true, "requires": { "assert": "^1.1.1", "browserify-zlib": "^0.2.0", @@ -16225,6 +16512,7 @@ "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dev": true, "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4", @@ -16234,7 +16522,8 @@ "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", + "dev": true } } }, @@ -16324,7 +16613,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "devOptional": true + "dev": true }, "normalize-range": { "version": "0.1.2", @@ -16505,6 +16794,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", "integrity": "sha512-79LYn6VAb63zgtmAteVOWo9Vdj71ZVBy3Pbse+VqxDpEP83XuujMrGqHIwAXJ5I/aM0zU7dIyIAhifVTPrNItQ==", + "dev": true, "requires": { "copy-descriptor": "^0.1.0", "define-property": "^0.2.5", @@ -16515,6 +16805,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -16523,6 +16814,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -16567,6 +16859,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", "integrity": "sha512-GBaMwwAVK9qbQN3Scdo0OyvgPW7l3lnaVMj84uTOZlswkX0KpF6fyDBJhtTthf7pymztoN36/KEr1DyhF96zEA==", + "dev": true, "requires": { "isobject": "^3.0.0" } @@ -16599,6 +16892,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -16774,7 +17068,8 @@ "os-browserify": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", - "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==" + "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==", + "dev": true }, "os-tmpdir": { "version": "1.0.2", @@ -16917,12 +17212,14 @@ "pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true }, "parallel-transform": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz", "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==", + "dev": true, "requires": { "cyclist": "^1.0.1", "inherits": "^2.0.3", @@ -17020,18 +17317,20 @@ "pascalcase": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", - "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==" + "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", + "dev": true }, "path-browserify": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz", - "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==" + "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==", + "dev": true }, "path-dirname": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz", "integrity": "sha512-ALzNPpyNq9AqXMBjeymIjFDAkAFH06mHJH/cSBHAgU0s4vfpBn6b2nf8tiRLvagKD8RbTpq2FKTBg7cl9l3c7Q==", - "devOptional": true + "dev": true }, "path-exists": { "version": "4.0.0", @@ -17107,7 +17406,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "devOptional": true + "dev": true }, "pid-from-port": { "version": "1.1.3", @@ -17208,6 +17507,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, "requires": { "find-up": "^3.0.0" }, @@ -17216,6 +17516,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, "requires": { "locate-path": "^3.0.0" } @@ -17224,6 +17525,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, "requires": { "p-locate": "^3.0.0", "path-exists": "^3.0.0" @@ -17233,6 +17535,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, "requires": { "p-limit": "^2.0.0" } @@ -17240,7 +17543,8 @@ "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==" + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true } } }, @@ -17293,7 +17597,8 @@ "posix-character-classes": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", - "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==" + "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==", + "dev": true }, "postcss": { "version": "7.0.39", @@ -18001,7 +18306,8 @@ "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true }, "process-exists": { "version": "3.1.0", @@ -18064,7 +18370,8 @@ "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==" + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true }, "ps-list": { "version": "4.1.0", @@ -18120,6 +18427,7 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "dev": true, "requires": { "duplexify": "^3.6.0", "inherits": "^2.0.3", @@ -18130,6 +18438,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dev": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -18180,12 +18489,14 @@ "querystring": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==" + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", + "dev": true }, "querystring-es3": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", - "integrity": "sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA==" + "integrity": "sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA==", + "dev": true }, "querystringify": { "version": "2.2.0", @@ -18804,7 +19115,7 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "devOptional": true, + "dev": true, "requires": { "picomatch": "^2.2.1" } @@ -18863,6 +19174,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, "requires": { "extend-shallow": "^3.0.2", "safe-regex": "^1.1.0" @@ -18948,7 +19260,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", "integrity": "sha512-/hS+Y0u3aOfIETiaiirUFwDBDzmXPvO+jAfKTitUngIPzdKc6Z0LoFjM/CK5PL4C+eKwHohlHAb6H0VFfmmUsw==", - "devOptional": true + "dev": true }, "renderkid": { "version": "2.0.7", @@ -19024,12 +19336,14 @@ "repeat-element": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.4.tgz", - "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==" + "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==", + "dev": true }, "repeat-string": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==" + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "dev": true }, "request": { "version": "2.88.2", @@ -19142,7 +19456,8 @@ "resolve-url": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==" + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", + "dev": true }, "responselike": { "version": "1.0.2", @@ -19165,7 +19480,8 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true }, "retry": { "version": "0.12.0", @@ -19242,6 +19558,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", "integrity": "sha512-ntymy489o0/QQplUDnpYAYUsO50K9SBrIVaKCWDOJzYJts0f9WH9RFJkyagebkw5+y1oi00R7ynNW/d12GBumg==", + "dev": true, "requires": { "aproba": "^1.1.1" } @@ -19264,6 +19581,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "dev": true, "requires": { "ret": "~0.1.10" } @@ -19382,8 +19700,7 @@ "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", - "dev": true + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "saxes": { "version": "3.1.11", @@ -19559,6 +19876,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz", "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==", + "dev": true, "requires": { "randombytes": "^2.1.0" } @@ -19640,6 +19958,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, "requires": { "extend-shallow": "^2.0.1", "is-extendable": "^0.1.1", @@ -19651,6 +19970,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -19659,6 +19979,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -19668,7 +19989,8 @@ "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true }, "setprototypeof": { "version": "1.2.0", @@ -19850,6 +20172,7 @@ "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, "requires": { "base": "^0.11.1", "debug": "^2.2.0", @@ -19865,6 +20188,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, "requires": { "ms": "2.0.0" } @@ -19873,6 +20197,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -19881,6 +20206,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -19888,12 +20214,14 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true } } }, @@ -19901,6 +20229,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, "requires": { "define-property": "^1.0.0", "isobject": "^3.0.0", @@ -19911,6 +20240,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, "requires": { "is-descriptor": "^1.0.0" } @@ -19919,6 +20249,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -19927,6 +20258,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -19935,6 +20267,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -19947,6 +20280,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, "requires": { "kind-of": "^3.2.0" }, @@ -19955,6 +20289,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -20065,6 +20400,7 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "dev": true, "requires": { "atob": "^2.1.2", "decode-uri-component": "^0.2.0", @@ -20077,6 +20413,7 @@ "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, "requires": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" @@ -20085,7 +20422,8 @@ "source-map-url": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.1.tgz", - "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==" + "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==", + "dev": true }, "sourcemap-codec": { "version": "1.4.8", @@ -20169,10 +20507,16 @@ } } }, + "split-on-first": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", + "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==" + }, "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, "requires": { "extend-shallow": "^3.0.0" } @@ -20248,6 +20592,7 @@ "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", "integrity": "sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==", + "dev": true, "requires": { "define-property": "^0.2.5", "object-copy": "^0.1.0" @@ -20257,6 +20602,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -20288,6 +20634,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==", + "dev": true, "requires": { "inherits": "~2.0.1", "readable-stream": "^2.0.2" @@ -20297,6 +20644,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/stream-each/-/stream-each-1.2.3.tgz", "integrity": "sha512-vlMC2f8I2u/bZGqkdfLQW/13Zihpej/7PmSiMQsbYddxuTsJp8vRe2x2FvVExZg7FaOds43ROAuFJwPR4MTZLw==", + "dev": true, "requires": { "end-of-stream": "^1.1.0", "stream-shift": "^1.0.0" @@ -20306,6 +20654,7 @@ "version": "2.8.3", "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-2.8.3.tgz", "integrity": "sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==", + "dev": true, "requires": { "builtin-status-codes": "^3.0.0", "inherits": "^2.0.1", @@ -20317,7 +20666,8 @@ "stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", - "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==", + "dev": true }, "streamsearch": { "version": "0.1.2", @@ -20331,14 +20681,6 @@ "integrity": "sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ==", "dev": true }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "string-convert": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", @@ -20403,6 +20745,14 @@ "es-abstract": "^1.19.5" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -20458,6 +20808,11 @@ "escape-string-regexp": "^1.0.2" } }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "stylehacks": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/stylehacks/-/stylehacks-4.0.3.tgz", @@ -20676,7 +21031,8 @@ "tapable": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz", - "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==" + "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==", + "dev": true }, "tar": { "version": "6.1.11", @@ -20861,6 +21217,7 @@ "version": "4.8.0", "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz", "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==", + "dev": true, "requires": { "commander": "^2.20.0", "source-map": "~0.6.1", @@ -20870,7 +21227,8 @@ "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true } } }, @@ -20878,6 +21236,7 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz", "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==", + "dev": true, "requires": { "cacache": "^12.0.2", "find-cache-dir": "^2.1.0", @@ -20894,6 +21253,7 @@ "version": "12.0.4", "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", + "dev": true, "requires": { "bluebird": "^3.5.5", "chownr": "^1.1.1", @@ -20915,12 +21275,14 @@ "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, "requires": { "yallist": "^3.0.2" } @@ -20929,6 +21291,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -20937,6 +21300,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -20945,6 +21309,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dev": true, "requires": { "ajv": "^6.1.0", "ajv-errors": "^1.0.0", @@ -20955,6 +21320,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz", "integrity": "sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==", + "dev": true, "requires": { "figgy-pudding": "^3.5.1" } @@ -20963,6 +21329,7 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "dev": true, "requires": { "source-list-map": "^2.0.0", "source-map": "~0.6.1" @@ -20971,12 +21338,14 @@ "y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "dev": true }, "yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true } } }, @@ -21065,6 +21434,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dev": true, "requires": { "readable-stream": "~2.3.6", "xtend": "~4.0.1" @@ -21086,6 +21456,7 @@ "version": "2.0.12", "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", + "dev": true, "requires": { "setimmediate": "^1.0.4" } @@ -21119,7 +21490,8 @@ "to-arraybuffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", - "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==" + "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==", + "dev": true }, "to-buffer": { "version": "1.1.1", @@ -21137,6 +21509,7 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -21145,6 +21518,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -21160,6 +21534,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, "requires": { "define-property": "^2.0.2", "extend-shallow": "^3.0.2", @@ -21171,6 +21546,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", + "dev": true, "requires": { "is-number": "^3.0.0", "repeat-string": "^1.6.1" @@ -21332,7 +21708,8 @@ "tty-browserify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", - "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==" + "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==", + "dev": true }, "tunnel-agent": { "version": "0.6.0", @@ -21374,7 +21751,8 @@ "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "dev": true }, "typedarray-to-buffer": { "version": "3.1.5", @@ -21591,6 +21969,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", @@ -21656,6 +22035,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", "integrity": "sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==", + "dev": true, "requires": { "has-value": "^0.3.1", "isobject": "^3.0.0" @@ -21665,6 +22045,7 @@ "version": "0.3.1", "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", "integrity": "sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==", + "dev": true, "requires": { "get-value": "^2.0.3", "has-values": "^0.1.4", @@ -21675,6 +22056,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, "requires": { "isarray": "1.0.0" } @@ -21684,7 +22066,8 @@ "has-values": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", - "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==" + "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==", + "dev": true } } }, @@ -21692,7 +22075,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==", - "devOptional": true + "dev": true }, "update-browserslist-db": { "version": "1.0.4", @@ -21760,12 +22143,14 @@ "urix": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", - "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==" + "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==", + "dev": true }, "url": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", "integrity": "sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==", + "dev": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -21774,7 +22159,8 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", + "dev": true } } }, @@ -21844,12 +22230,14 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==" + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true }, "util": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "dev": true, "requires": { "inherits": "2.0.3" }, @@ -21857,7 +22245,8 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true } } }, @@ -21952,7 +22341,8 @@ "vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", - "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", + "dev": true }, "vue": { "version": "3.2.37", @@ -21969,8 +22359,7 @@ "vue-chartjs": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/vue-chartjs/-/vue-chartjs-4.1.2.tgz", - "integrity": "sha512-QSggYjeFv/L4jFSBQpX8NzrAvX0B+Ha6nDgxkTG8tEXxYOOTwKI4phRLe+B4f+REnkmg7hgPY24R0cixZJyXBg==", - "requires": {} + "integrity": "sha512-QSggYjeFv/L4jFSBQpX8NzrAvX0B+Ha6nDgxkTG8tEXxYOOTwKI4phRLe+B4f+REnkmg7hgPY24R0cixZJyXBg==" }, "vue-clipboard2": { "version": "0.3.3", @@ -22406,8 +22795,7 @@ "vue-web-storage": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/vue-web-storage/-/vue-web-storage-6.1.0.tgz", - "integrity": "sha512-Qsa6QkUyGP+Tj0oxLRc6vEATv6axY89LbwfX8eaMM3i7K/Nl9m0NTELtNp0s8Xhg9F0SkeEP4NC3+LQL3ygMQw==", - "requires": {} + "integrity": "sha512-Qsa6QkUyGP+Tj0oxLRc6vEATv6axY89LbwfX8eaMM3i7K/Nl9m0NTELtNp0s8Xhg9F0SkeEP4NC3+LQL3ygMQw==" }, "vue3-clipboard": { "version": "1.0.0", @@ -22492,6 +22880,7 @@ "version": "1.7.5", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.5.tgz", "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==", + "dev": true, "requires": { "chokidar": "^3.4.1", "graceful-fs": "^4.1.2", @@ -22503,6 +22892,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz", "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==", + "dev": true, "optional": true, "requires": { "chokidar": "^2.1.8" @@ -22512,6 +22902,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==", + "dev": true, "optional": true, "requires": { "micromatch": "^3.1.4", @@ -22522,6 +22913,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", "integrity": "sha512-3pKJwH184Xo/lnH6oyP1q2pMd7HcypqqmRs91/6/i2CGtWwIKGCkOOMTm/zXbgTEWHw1uNpNi/igc3ePOYHb6w==", + "dev": true, "optional": true, "requires": { "remove-trailing-separator": "^1.0.1" @@ -22533,12 +22925,14 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "dev": true, "optional": true }, "chokidar": { "version": "2.1.8", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "dev": true, "optional": true, "requires": { "anymatch": "^2.0.0", @@ -22559,6 +22953,7 @@ "version": "1.2.13", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, "optional": true, "requires": { "bindings": "^1.5.0", @@ -22569,6 +22964,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", "integrity": "sha512-9fRVlXc0uCxEDj1nQzaWONSpbTfx0FmJfzHF7pwlI8DkWGoHBBea4Pg5Ky0ojwwxQmnSifgbKkI06Qv0Ljgj+Q==", + "dev": true, "optional": true, "requires": { "binary-extensions": "^1.0.0" @@ -22578,6 +22974,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "dev": true, "optional": true, "requires": { "graceful-fs": "^4.1.11", @@ -22605,6 +23002,29 @@ "defaults": "^1.0.3" } }, + "web-encoding": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", + "integrity": "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==", + "requires": { + "@zxing/text-encoding": "0.9.0", + "util": "^0.12.3" + }, + "dependencies": { + "util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "requires": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + } + } + }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -22615,6 +23035,7 @@ "version": "4.46.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.46.0.tgz", "integrity": "sha512-6jJuJjg8znb/xRItk7bkT0+Q7AHCYjjFnvKIWQPkNIOyRqoCGvkOs0ipeQzrqz4l5FtN5ZI/ukEHroeX/o1/5Q==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-module-context": "1.9.0", @@ -22645,6 +23066,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, "requires": { "minimist": "^1.2.0" } @@ -22653,6 +23075,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "dev": true, "requires": { "big.js": "^5.2.2", "emojis-list": "^3.0.0", @@ -22663,6 +23086,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -22671,6 +23095,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dev": true, "requires": { "ajv": "^6.1.0", "ajv-errors": "^1.0.0", @@ -22681,6 +23106,7 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "dev": true, "requires": { "source-list-map": "^2.0.0", "source-map": "~0.6.1" @@ -23315,6 +23741,18 @@ "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", "dev": true }, + "which-typed-array": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", + "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", + "requires": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0" + } + }, "wide-align": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", @@ -23371,6 +23809,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", + "dev": true, "requires": { "errno": "~0.1.7" } @@ -23460,14 +23899,18 @@ "version": "7.5.8", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.8.tgz", "integrity": "sha512-ri1Id1WinAX5Jqn9HejiGb8crfRio0Qgu8+MtL36rlTA6RLsMdWt1Az/19A2Qij6uSHUMphEFaTKa4WG+UNHNw==", - "dev": true, - "requires": {} + "dev": true }, "xdg-basedir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==" }, + "xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==" + }, "xml-name-validator": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", @@ -23478,7 +23921,6 @@ "version": "0.4.23", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", - "dev": true, "requires": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" @@ -23487,8 +23929,7 @@ "xmlbuilder": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "dev": true + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" }, "xmlchars": { "version": "2.2.0", @@ -23517,7 +23958,8 @@ "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true }, "y18n": { "version": "5.0.8", diff --git a/ui/package.json b/ui/package.json index 1c6e42a321e..fd9eeb72058 100644 --- a/ui/package.json +++ b/ui/package.json @@ -53,6 +53,7 @@ "js-cookie": "^2.2.1", "lodash": "^4.17.15", "md5": "^2.2.1", + "minio": "^7.0.33", "mitt": "^2.1.0", "moment": "^2.26.0", "moment-timezone": "^0.5.43", diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index afe3c4df7ac..d54d1bba75d 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -25,6 +25,8 @@ "label.access": "Access", "label.access.kubernetes.nodes": "Access Kubernetes nodes", "label.accesskey": "Access key", +"label.access.key": "Access key", +"label.secret.key": "Secret key", "label.account": "Account", "label.account.and.security.group": "Account - security group", "label.account.id": "Account ID", @@ -1083,6 +1085,7 @@ "label.isdedicated": "Dedicated", "label.isdefault": "Is default", "label.isdynamicallyscalable": "Dynamically scalable", +"label.istagarule": "Tag as JS rule", "label.isextractable": "Extractable", "label.isfeatured": "Featured", "label.isforced": "Force delete", @@ -1183,6 +1186,7 @@ "label.limitcpuuse": "CPU cap", "label.limits": "Limits", "label.limits.configure": "Configure limits", +"label.link": "Link", "label.link.domain.to.ldap": "Link domain to LDAP", "label.linklocalip": "Link-local/Control IP address", "label.linux": "Linux", @@ -1279,6 +1283,9 @@ "label.memused": "Memory usage", "label.menu.security.groups": "Security groups", "label.menu.service.offerings": "Service offerings", +"label.metadata": "Metadata", +"label.metadata.description": "Metadata of the Object", +"label.metadata.upload.description": "Set metadata for the object", "label.metrics": "Metrics", "label.migrate.allowed": "Migrate allowed", "label.migrate.data.from.image.store": "Migrate data from image store", @@ -1420,6 +1427,16 @@ "label.oauth.configuration": "OAuth configuration", "label.oauth.verification": "OAuth verification", "label.ocfs2": "OCFS2", +"label.object.storage" : "Object Storage", +"label.object.presigned.url": "Presigned URL", +"label.object.presigned.url.description" : "Presigned URL of the object in order to access it without authentication.", +"label.object.url.description" : "URL of the object", +"label.objectstore" : "Object Storage", +"label.objectstore.search" : "Prefix based search in current directory", +"label.add.object.storage" : "Add Object Storage", +"label.add.key.value": "Add key value pair", +"label.action.update.object.storage" : "Update Object Storage", +"label.action.delete.object.storage" : "Delete Object Storage", "label.of": "of", "label.of.month": "of month", "label.offerha": "Offer HA", @@ -2135,6 +2152,8 @@ "label.updating": "Updating", "label.upgrade.router.newer.template": "Upgrade router to use newer Template", "label.upload": "Upload", +"label.upload.description": "Path to upload objects at", +"label.upload.path": "Upload path", "label.upload.icon": "Upload icon", "label.upload.iso.from.local": "Upload ISO from local", "label.upload.resource.icon": "Upload icon", @@ -2225,7 +2244,7 @@ "label.vnf.appliance": "VNF Appliance", "label.vnf.appliances": "VNF appliances", "label.vnf.appliance.add": "Add VNF Appliance", -"label.vnf.appliance.access.methods": "Management access information of this VNF appliance", +"label.vnf.appliance.access.methods": "Management access information for this VNF appliance", "label.vnf.app.action.destroy": "Destroy VNF appliance", "label.vnf.app.action.edit": "Edit VNF appliance", "label.vnf.app.action.expunge": "Expunge VNF appliance", @@ -2237,24 +2256,25 @@ "label.vnf.app.action.stop": "Stop VNF appliance", "label.vnf.app.action.reboot": "Reboot VNF appliance", "label.vnf.app.action.reinstall": "Reinstall VNF appliance", -"label.vnf.cidr.list": "Source cidr list of rules", +"label.vnf.cidr.list": "CIDR from which access to the VNF appliance’s Management interface should be allowed from", "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.", -"label.vnf.configure.management": "Configure rules for VNF management interfaces", -"label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise.", +"label.vnf.configure.management": "Configure Firewall and Port Forwarding rules for VNF's management interfaces", +"label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", "label.vnf.detail.add": "Add VNF detail", "label.vnf.detail.remove": "Remove VNF detail", -"label.vnf.details": "VNF details", +"label.vnf.details": "VNF Details", "label.vnf.nic.add": "Add VNF nic", "label.vnf.nic.delete": "Delete VNF nic", "label.vnf.nic.description": "Description of VNF nic", -"label.vnf.nic.deviceid": "Device ID of VNF nic. It starts with 0", +"label.vnf.nic.deviceid": "Device ID of VNF nic. It starts with 0. The NIC with deviceid as 0 for VNF appliance will be the default NIC.", "label.vnf.nic.edit": "Edit VNF nic", "label.vnf.nic.management": "Management NIC", "label.vnf.nic.management.description": "True if the VNF nic is a management interface. False otherwise", +"label.vnf.nic.mappings": "VNF NIC mappings", "label.vnf.nic.name": "Name of VNF nic", "label.vnf.nic.remove": "Remove VNF nic", "label.vnf.nic.required": "True if VNF nic is required. Otherwise optional", -"label.vnf.nics": "VNF nics", +"label.vnf.nics": "VNF Nics", "label.vnf.settings": "VNF settings", "label.vnf.templates": "VNF templates", "label.vnf.template.register": "Register VNF template", @@ -2333,6 +2353,17 @@ "label.zonenamelabel": "Zone name", "label.zones": "Zones", "label.zonewizard.traffictype.storage": "Storage: Traffic between primary and secondary storage servers, such as Instance Templates and Snapshots.", +"label.buckets": "Buckets", +"label.objectstorageid": "Object Storage Pool", +"label.bucket.update": "Update Bucket", +"label.bucket.delete": "Delete Bucket", +"label.quotagb": "Quota in GB", +"label.encryption": "Encryption", +"label.versioning": "Versioning", +"label.objectlocking": "Object Lock", +"label.bucket.policy": "Bucket Policy", +"label.usersecretkey": "Secret Key", +"label.create.bucket": "Create Bucket", "message.acquire.ip.failed": "Failed to acquire IP.", "message.action.acquire.ip": "Please confirm that you want to acquire new IP.", "message.action.cancel.maintenance": "Your host has been successfully canceled for maintenance. This process can take up to several minutes.", @@ -2512,6 +2543,8 @@ "message.config.health.monitor.failed": "Configure Health Monitor failed", "message.config.sticky.policy.failed": "Failed to configure sticky policy.", "message.config.sticky.policy.processing": "Updating sticky policy...", +"message.configure.network.ip.and.mac": "Please configure the IP address and mac address of networks if needed.", +"message.configure.network.select.default.network": "Please configure the IP address and mac address of networks if needed. Please select a network as the default network.", "message.configuring.guest.traffic": "Configuring guest traffic", "message.configuring.physical.networks": "Configuring physical Networks", "message.configuring.public.traffic": "Configuring public traffic", @@ -2568,6 +2601,8 @@ "message.confirm.type": "To confirm, please type", "message.confirm.upgrade.router.newer.template": "Please confirm that you want to upgrade router to use newer Template.", "message.cpu.usage.info": "The CPU usage percentage can exceed 100% if the Instance has more than 1 vCPU or when CPU Cap is not enabled. This behavior happens according to the hypervisor being used (e.g: in KVM), due to how they account the stats", +"message.create.bucket.failed": "Failed to create bucket.", +"message.create.bucket.processing": "Bucket creation in progress", "message.create.compute.offering": "Compute offering created", "message.create.tungsten.public.network": "Create Tungsten-Fabric public Network", "message.create.internallb": "Creating internal LB", @@ -3059,6 +3094,7 @@ "message.success.add.network.static.route": "Successfully added Network Static Route", "message.success.add.network.permissions": "Successfully added Network permissions", "message.success.add.physical.network": "Successfully added Physical Network", +"message.success.add.object.storage": "Successfully added Object Storage", "message.success.add.policy.rule": "Successfully added Policy rule", "message.success.add.port.forward": "Successfully added new port forwarding rule", "message.success.add.private.gateway": "Successfully added private gateway", @@ -3087,6 +3123,7 @@ "message.success.config.vm.schedule": "Successfully configured Instance schedule", "message.success.copy.clipboard": "Successfully copied to clipboard", "message.success.create.account": "Successfully created Account", +"message.success.create.bucket": "Successfully created bucket", "message.success.create.internallb": "Successfully created Internal Load Balancer", "message.success.create.isolated.network": "Successfully created isolated Network", "message.success.create.keypair": "Successfully created SSH key pair", @@ -3127,6 +3164,8 @@ "message.success.register.user.data": "Successfully registered Userdata", "message.success.release.ip": "Successfully released IP", "message.success.remove.egress.rule": "Successfully removed egress rule", +"message.success.remove.objectstore.objects": "Successfully removed selected object(s)", +"message.success.remove.objectstore.directory": "Successfully removed selected directory", "message.success.remove.firewall.rule": "Successfully removed firewall rule", "message.success.remove.instance.rule": "Successfully removed Instance from rule", "message.success.remove.ip": "Successfully removed IP", @@ -3145,6 +3184,7 @@ "message.success.resize.volume": "Successfully resized volume", "message.success.scale.kubernetes": "Successfully scaled Kubernetes cluster", "message.success.unmanage.instance": "Successfully unmanaged Instance", +"message.success.update.bucket": "Successfully updated bucket", "message.success.update.condition": "Successfully updated condition", "message.success.update.ipaddress": "Successfully updated IP address", "message.success.update.iprange": "Successfully updated IP range", @@ -3223,17 +3263,19 @@ "message.vm.state.stopped": "Instance is stopped.", "message.vm.state.stopping": "Instance is being stopped.", "message.vm.state.unknown": "Instance state is unknown.", -"message.vnf.appliance.networks": "Please select Networks for the new VNF appliance.", +"message.vnf.appliance.networks": "Please select the networks for the new VNF appliance.", "message.vnf.credentials.change": "Please change the password(s) of the VNF appliance immediately.", "message.vnf.credentials.default": "The default credentials(s) of the VNF appliance", "message.vnf.credentials.in.template.vnf.details": "Please find the default credentials for this VNF in the details of the VNF template.", -"message.vnf.error.deviceid.should.be.continuous": "The deviceid of selected VNF NICs should be continuous.", +"message.vnf.error.deviceid.should.be.consecutive": "The deviceid of selected VNF NICs should be consecutive.", "message.vnf.error.network.is.already.used": "Network has been used by multiple NICs of the new VNF appliance.", -"message.vnf.error.no.networks": "Please select networks for NICs of the new VNF appliance.", +"message.vnf.error.network.should.be.used": "All selected networks should be used as VNF Nics", +"message.vnf.error.no.networks": "Please select the networks for the new VNF appliance.", "message.vnf.error.no.network.for.required.deviceid": "Please select a Network for required NIC of the new VNF appliance.", "message.vnf.nic.move.up.fail": "Failed to move up this NIC", "message.vnf.nic.move.down.fail": "Failed to move down this NIC", -"message.vnf.select.networks": "Please select a Network for each VNF nic. ", +"message.vnf.no.credentials": "No credentials found for the VNF appliance.", +"message.vnf.select.networks": "Please select the relevant network for each VNF NIC.", "message.volume.state.allocated": "The volume is allocated but has not been created yet.", "message.volume.state.attaching": "The volume is attaching to a volume from Ready state.", "message.volume.state.copying": "The volume is being copied from the image store to primary storage, in case it's an uploaded volume.", @@ -3263,6 +3305,8 @@ "message.zone.detail.description": "Populate zone details.", "message.zone.detail.hint": "A zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", "message.validate.min": "Please enter a value greater than or equal to {0}.", +"message.action.delete.object.storage": "Please confirm that you want to delete this Object Store", +"message.bucket.delete": "Please confirm that you want to delete this Bucket", "migrate.from": "Migrate from", "migrate.to": "Migrate to", "migrationPolicy": "Migration policy", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 5b275a12602..c36a5c762a7 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -844,6 +844,7 @@ "label.isdedicated": "Dedicado", "label.isdefault": "\u00c9\u0089 padr\u00e3o", "label.isdynamicallyscalable": "Dinamicamente escal\u00e1vel", +"label.istagarule": "Tag como regra JS", "label.isextractable": "Extra\u00edvel", "label.isfeatured": "Em destaque", "label.isforced": "For\u00e7ar exclus\u00e3o", diff --git a/ui/src/components/KeyValuePairInput.vue b/ui/src/components/KeyValuePairInput.vue new file mode 100644 index 00000000000..03989de50de --- /dev/null +++ b/ui/src/components/KeyValuePairInput.vue @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/components/header/CreateMenu.vue b/ui/src/components/header/CreateMenu.vue index a477c98b1a1..5ff6894c48c 100644 --- a/ui/src/components/header/CreateMenu.vue +++ b/ui/src/components/header/CreateMenu.vue @@ -133,6 +133,25 @@ + + + + + + + + + +

+ {{ $t('label.vnf.appliance') }} +

+ {{ $t('label.vnf.appliance.add') }} + + + + diff --git a/ui/src/components/view/AnnotationsTab.vue b/ui/src/components/view/AnnotationsTab.vue index 97cf926dee3..5b42af8eabc 100644 --- a/ui/src/components/view/AnnotationsTab.vue +++ b/ui/src/components/view/AnnotationsTab.vue @@ -193,6 +193,7 @@ export default { case 'VirtualRouter': return 'VR' case 'AutoScaleVmGroup': return 'AUTOSCALE_VM_GROUP' case 'ManagementServer': return 'MANAGEMENT_SERVER' + case 'ObjectStorage': return 'OBJECT_STORAGE' default: return '' } }, diff --git a/ui/src/components/view/DetailsTab.vue b/ui/src/components/view/DetailsTab.vue index 45723c18aeb..d758355adbd 100644 --- a/ui/src/components/view/DetailsTab.vue +++ b/ui/src/components/view/DetailsTab.vue @@ -207,9 +207,11 @@ export default { } const managementDeviceIds = [] - for (const vnfnic of this.resource.vnfnics) { - if (vnfnic.management) { - managementDeviceIds.push(vnfnic.deviceid) + if (this.resource.vnfnics) { + for (const vnfnic of this.resource.vnfnics) { + if (vnfnic.management) { + managementDeviceIds.push(vnfnic.deviceid) + } } } const managementIps = [] diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index cf52f058075..53c6efb321d 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -279,6 +279,9 @@ + @@ -585,7 +588,7 @@ export default { '/volume', '/snapshot', '/vmsnapshot', '/backup', '/guestnetwork', '/vpc', '/vpncustomergateway', '/vnfapp', '/template', '/iso', - '/project', '/account', + '/project', '/account', 'buckets', 'objectstore', '/zone', '/pod', '/cluster', '/host', '/storagepool', '/imagestore', '/systemvm', '/router', '/ilbvm', '/annotation', '/computeoffering', '/systemoffering', '/diskoffering', '/backupoffering', '/networkoffering', '/vpcoffering', '/tungstenfabric', '/oauthsetting', '/guestos', '/guestoshypervisormapping'].join('|')) @@ -595,7 +598,7 @@ export default { return ['vm', 'alert', 'vmgroup', 'ssh', 'userdata', 'affinitygroup', 'autoscalevmgroup', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork', 'vpc', 'publicip', 'vpnuser', 'vpncustomergateway', 'vnfapp', 'project', 'account', 'systemvm', 'router', 'computeoffering', 'systemoffering', - 'diskoffering', 'backupoffering', 'networkoffering', 'vpcoffering', 'ilbvm', 'kubernetes', 'comment' + 'diskoffering', 'backupoffering', 'networkoffering', 'vpcoffering', 'ilbvm', 'kubernetes', 'comment', 'buckets' ].includes(this.$route.name) }, getDateAtTimeZone (date, timezone) { diff --git a/ui/src/components/view/ObjectStoreBrowser.vue b/ui/src/components/view/ObjectStoreBrowser.vue new file mode 100644 index 00000000000..f2e68c8b954 --- /dev/null +++ b/ui/src/components/view/ObjectStoreBrowser.vue @@ -0,0 +1,541 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/config/section/infra.js b/ui/src/config/section/infra.js index 41b08fc6f51..0b3b1e7ae4f 100644 --- a/ui/src/config/section/infra.js +++ b/ui/src/config/section/infra.js @@ -23,6 +23,7 @@ import clusters from '@/config/section/infra/clusters' import hosts from '@/config/section/infra/hosts' import primaryStorages from '@/config/section/infra/primaryStorages' import secondaryStorages from '@/config/section/infra/secondaryStorages' +import objectStorages from '@/config/section/infra/objectStorages' import systemVms from '@/config/section/infra/systemVms' import routers from '@/config/section/infra/routers' import ilbvms from '@/config/section/infra/ilbvms' @@ -49,6 +50,7 @@ export default { hosts, primaryStorages, secondaryStorages, + objectStorages, systemVms, routers, ilbvms, diff --git a/ui/src/config/section/infra/hosts.js b/ui/src/config/section/infra/hosts.js index af3ad1dbcb5..9a6fc021152 100644 --- a/ui/src/config/section/infra/hosts.js +++ b/ui/src/config/section/infra/hosts.js @@ -68,7 +68,7 @@ export default { icon: 'edit-outlined', label: 'label.edit', dataView: true, - args: ['name', 'hosttags', 'oscategoryid'], + args: ['name', 'hosttags', 'istagarule', 'oscategoryid'], mapping: { oscategoryid: { api: 'listOsCategories' diff --git a/ui/src/config/section/infra/objectStorages.js b/ui/src/config/section/infra/objectStorages.js new file mode 100644 index 00000000000..821c1b2d948 --- /dev/null +++ b/ui/src/config/section/infra/objectStorages.js @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { shallowRef, defineAsyncComponent } from 'vue' +import store from '@/store' + +export default { + name: 'objectstore', + title: 'label.object.storage', + icon: 'gold-outlined', + docHelp: 'adminguide/storage.html#object-storage', + permission: ['listObjectStoragePools'], + columns: () => { + var fields = ['name', 'url', 'providername'] + return fields + }, + details: () => { + var fields = ['name', 'id', 'url', 'providername'] + return fields + }, + resourceType: 'ObjectStorage', + tabs: [{ + name: 'details', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) + }, { + name: 'events', + resourceType: 'ObjectStore', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/EventsTab.vue'))), + show: () => { return 'listEvents' in store.getters.apis } + }, { + name: 'comments', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) + }], + actions: [ + { + api: 'addObjectStoragePool', + icon: 'plus-outlined', + docHelp: 'installguide/configuration.html#add-object-storage', + label: 'label.add.object.storage', + listView: true, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/AddObjectStorage.vue'))) + }, + { + api: 'updateObjectStoragePool', + icon: 'edit-outlined', + label: 'label.action.update.object.storage', + args: ['name', 'url'], + dataView: true + }, + { + api: 'deleteObjectStoragePool', + icon: 'delete-outlined', + label: 'label.action.delete.object.storage', + message: 'message.action.delete.object.storage', + dataView: true, + displayName: (record) => { return record.name || record.displayName || record.id } + } + ] +} diff --git a/ui/src/config/section/infra/primaryStorages.js b/ui/src/config/section/infra/primaryStorages.js index 6f51c7bd0a3..f222edeaf70 100644 --- a/ui/src/config/section/infra/primaryStorages.js +++ b/ui/src/config/section/infra/primaryStorages.js @@ -89,7 +89,7 @@ export default { icon: 'edit-outlined', label: 'label.edit', dataView: true, - args: ['name', 'tags', 'capacitybytes', 'capacityiops'] + args: ['name', 'tags', 'istagarule', 'capacitybytes', 'capacityiops'] }, { api: 'updateStoragePool', diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index 5c78a4d0e68..0362b424d1d 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -325,6 +325,7 @@ export default { title: 'label.vnf.appliances', icon: 'gateway-outlined', permission: ['listVirtualMachinesMetrics'], + resourceType: 'UserVm', params: () => { return { details: 'servoff,tmpl,nics', isvnf: true } }, diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 269c1f81788..0fbb930e750 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -164,8 +164,8 @@ export default { dataView: true, show: (record, store) => { return record.state === 'Ready' && (record.hypervisor !== 'KVM' || - record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || - record.hypervisor === 'KVM' && record.vmstate !== 'Running') + record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || + record.hypervisor === 'KVM' && record.vmstate !== 'Running') }, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/TakeSnapshot.vue'))) @@ -178,8 +178,8 @@ export default { dataView: true, show: (record, store) => { return record.state === 'Ready' && (record.hypervisor !== 'KVM' || - record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || - record.hypervisor === 'KVM' && record.vmstate !== 'Running') + record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || + record.hypervisor === 'KVM' && record.vmstate !== 'Running') }, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/RecurringSnapshotVolume.vue'))), @@ -250,10 +250,17 @@ export default { dataView: true, show: (record) => { return !['Destroy', 'Destroyed', 'Expunging', 'Expunged', 'Migrating', 'Uploading', 'UploadError', 'Creating'].includes(record.state) && - ((record.type === 'ROOT' && record.vmstate === 'Stopped') || - (record.type !== 'ROOT' && !record.virtualmachineid && !['Allocated', 'Uploaded'].includes(record.state))) + ((record.type === 'ROOT' && record.vmstate === 'Stopped') || + (record.type !== 'ROOT' && !record.virtualmachineid && !['Allocated', 'Uploaded'].includes(record.state))) + }, + args: (record, store) => { + var fields = ['volumeid', 'name', 'displaytext', 'ostypeid', 'ispublic', 'isfeatured', 'isdynamicallyscalable', 'requireshvm', 'passwordenabled'] + if (['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + fields.push('domainid') + fields.push('account') + } + return fields }, - args: ['volumeid', 'name', 'displaytext', 'ostypeid', 'ispublic', 'isfeatured', 'isdynamicallyscalable', 'requireshvm', 'passwordenabled'], mapping: { volumeid: { value: (record) => { return record.id } @@ -278,8 +285,8 @@ export default { dataView: true, show: (record, store) => { return ['Expunging', 'Expunged', 'UploadError'].includes(record.state) || - ['Allocated', 'Uploaded'].includes(record.state) && record.type !== 'ROOT' && !record.virtualmachineid || - ((['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) || store.features.allowuserexpungerecovervolume) && record.state === 'Destroy') + ['Allocated', 'Uploaded'].includes(record.state) && record.type !== 'ROOT' && !record.virtualmachineid || + ((['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) || store.features.allowuserexpungerecovervolume) && record.state === 'Destroy') }, groupAction: true, popup: true, @@ -440,6 +447,119 @@ export default { show: (record) => { return record.state !== 'Destroyed' } } ] + }, + { + name: 'backup', + title: 'label.backup', + icon: 'cloud-upload-outlined', + permission: ['listBackups'], + columns: [{ name: (record) => { return record.virtualmachinename } }, 'virtualmachinename', 'status', 'type', 'created', 'account', 'zone'], + details: ['virtualmachinename', 'id', 'type', 'externalid', 'size', 'virtualsize', 'volumes', 'backupofferingname', 'zone', 'account', 'domain', 'created'], + actions: [ + { + api: 'restoreBackup', + icon: 'sync-outlined', + docHelp: 'adminguide/virtual_machines.html#restoring-vm-backups', + label: 'label.backup.restore', + message: 'message.backup.restore', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' } + }, + { + api: 'restoreVolumeFromBackupAndAttachToVM', + icon: 'paper-clip-outlined', + label: 'label.backup.attach.restore', + message: 'message.backup.attach.restore', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' }, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/RestoreAttachBackupVolume.vue'))) + }, + { + api: 'removeVirtualMachineFromBackupOffering', + icon: 'scissor-outlined', + label: 'label.backup.offering.remove', + message: 'message.backup.offering.remove', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' }, + args: ['forced', 'virtualmachineid'], + mapping: { + forced: { + value: (record) => { return true } + }, + virtualmachineid: { + value: (record) => { return record.virtualmachineid } + } + } + }, + { + api: 'deleteBackup', + icon: 'delete-outlined', + label: 'label.delete.backup', + message: 'message.delete.backup', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' } + } + ] + }, + { + name: 'buckets', + title: 'label.buckets', + icon: 'funnel-plot-outlined', + permission: ['listBuckets'], + columns: ['name', 'state', 'objectstore', 'size', 'account'], + details: ['id', 'name', 'state', 'objectstore', 'size', 'url', 'accesskey', 'usersecretkey', 'account', 'domain', 'created', 'quota', 'encryption', 'versioning', 'objectlocking', 'policy'], + tabs: [ + { + name: 'details', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) + }, + { + name: 'browser', + resourceType: 'Bucket', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/ObjectStoreBrowser.vue'))), + show: (record) => { return record.provider !== 'Simulator' } + + }, + { + name: 'events', + resourceType: 'Bucket', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/EventsTab.vue'))), + show: () => { return 'listEvents' in store.getters.apis } + } + ], + actions: [ + { + api: 'createBucket', + icon: 'plus-outlined', + docHelp: 'installguide/configuration.html#create-bucket', + label: 'label.create.bucket', + listView: true, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/CreateBucket.vue'))) + }, + { + api: 'updateBucket', + icon: 'edit-outlined', + docHelp: 'adminguide/object_storage.html#update-bucket', + label: 'label.bucket.update', + dataView: true, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/UpdateBucket.vue'))), + show: (record) => { return record.state !== 'Destroyed' } + }, + { + api: 'deleteBucket', + icon: 'delete-outlined', + label: 'label.bucket.delete', + message: 'message.bucket.delete', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' }, + groupAction: true, + popup: true, + groupMap: (selection) => { return selection.map(x => { return { id: x } }) } + } + ] } ] } diff --git a/ui/src/core/lazy_lib/icons_use.js b/ui/src/core/lazy_lib/icons_use.js index 0a95facdd9d..dc960de1bad 100644 --- a/ui/src/core/lazy_lib/icons_use.js +++ b/ui/src/core/lazy_lib/icons_use.js @@ -88,6 +88,7 @@ import { FormOutlined, ForwardOutlined, FullscreenOutlined, + FunnelPlotOutlined, GatewayOutlined, GithubOutlined, GlobalOutlined, @@ -244,6 +245,7 @@ export default { app.component('FormOutlined', FormOutlined) app.component('ForwardOutlined', ForwardOutlined) app.component('FullscreenOutlined', FullscreenOutlined) + app.component('FunnelPlotOutlined', FunnelPlotOutlined) app.component('GatewayOutlined', GatewayOutlined) app.component('GithubOutlined', GithubOutlined) app.component('GlobalOutlined', GlobalOutlined) diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index 5c725b7dd98..1f18f3cbb18 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -342,7 +342,6 @@ export default { const listZones = json.listzonesresponse.zone if (listZones) { this.zones = this.zones.concat(listZones) - this.zones = this.zones.filter(zone => zone.type !== 'Edge') } }).finally(() => { this.zoneLoading = false @@ -355,6 +354,7 @@ export default { handleZoneChange (zone) { this.selectedZone = zone this.fetchKubernetesVersionData() + this.fetchNetworkData() }, fetchKubernetesVersionData () { this.kubernetesVersions = [] @@ -413,10 +413,14 @@ export default { }, fetchNetworkData () { const params = {} + if (!this.isObjectEmpty(this.selectedZone)) { + params.zoneid = this.selectedZone.id + } this.networkLoading = true api('listNetworks', params).then(json => { - const listNetworks = json.listnetworksresponse.network + var listNetworks = json.listnetworksresponse.network if (this.arrayHasItems(listNetworks)) { + listNetworks = listNetworks.filter(n => n.type !== 'L2') this.networks = this.networks.concat(listNetworks) } }).finally(() => { diff --git a/ui/src/views/compute/DeployVnfAppliance.vue b/ui/src/views/compute/DeployVnfAppliance.vue index e571f4e23c4..5e1baac3d8f 100644 --- a/ui/src/views/compute/DeployVnfAppliance.vue +++ b/ui/src/views/compute/DeployVnfAppliance.vue @@ -365,7 +365,7 @@