diff --git a/.asf.yaml b/.asf.yaml index ce89a03d9ce..1772623f1b1 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -41,6 +41,7 @@ github: features: wiki: true issues: true + discussions: true projects: true enabled_merge_buttons: @@ -49,15 +50,21 @@ github: rebase: false collaborators: + - acs-robot - kiranchavala - rajujith - alexandremattioli - vishesh92 - GaOrtiga - - acs-robot - BryanMLima - SadiJr - JoaoJandre - winterhazel protected_branches: ~ + +notifications: + commits: commits@cloudstack.apache.org + issues: commits@cloudstack.apache.org + pullrequests: commits@cloudstack.apache.org + discussions: users@cloudstack.apache.org diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 457dd2e1af4..1be892f4577 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0422cd4bd0..c6edc7bdb20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: if: github.repository == 'apache/cloudstack' @@ -90,6 +93,7 @@ jobs: smoke/test_nic smoke/test_nic_adapter_type smoke/test_non_contigiousvlan + smoke/test_object_stores smoke/test_outofbandmanagement smoke/test_outofbandmanagement_nestedplugin smoke/test_over_provisioning @@ -108,7 +112,8 @@ jobs: smoke/test_reset_configuration_settings smoke/test_reset_vm_on_reboot smoke/test_resource_accounting - smoke/test_resource_detail", + smoke/test_resource_detail + smoke/test_global_acls", "smoke/test_router_dhcphosts smoke/test_router_dns smoke/test_router_dnsservice diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index d243fa863fe..64fa4c3da0c 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 280024b5a91..4d89977adf9 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 3f07ba16237..b15534a0be6 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -419,3 +419,6 @@ iscsi.session.cleanup.enabled=false # Timeout (in milliseconds) of the KVM heartbeat checker. # kvm.heartbeat.checker.timeout=360000 + +# Instance Conversion from Vmware to KVM through virt-v2v. Enable verbose mode +# virtv2v.verbose.enabled=false diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 8f51d470f07..0e979d370f4 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -14,6 +14,8 @@ */ package com.cloud.agent.properties; +import org.apache.cloudstack.utils.security.KeyStoreUtils; + /** * Class of constant agent's properties available to configure on * "agent.properties". @@ -733,6 +735,13 @@ public class AgentProperties{ */ public static final Property IOTHREADS = new Property<>("iothreads", 1); + /** + * Enable verbose mode for virt-v2v Instance Conversion from Vmware to KVM + * Data type: Boolean.
+ * Default value: false + */ + public static final Property VIRTV2V_VERBOSE_ENABLED = new Property<>("virtv2v.verbose.enabled", false); + /** * BGP controll CIDR * Data type: String.
@@ -772,6 +781,13 @@ public class AgentProperties{ */ public static final Property KVM_HEARTBEAT_CHECKER_TIMEOUT = new Property<>("kvm.heartbeat.checker.timeout", 360000L); + /** + * Keystore passphrase + * Data type: String.
+ * Default value: null + */ + public static final Property KEYSTORE_PASSPHRASE = new Property<>(KeyStoreUtils.KS_PASSPHRASE_PROPERTY, null, String.class); + public static class Property { private String name; private T defaultValue; diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java new file mode 100644 index 00000000000..6e7aa8b21e2 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.agent.api.to; + +import com.cloud.agent.api.LogLevel; +import com.cloud.hypervisor.Hypervisor; + +import java.io.Serializable; + +public class RemoteInstanceTO implements Serializable { + + private Hypervisor.HypervisorType hypervisorType; + private String hostName; + private String instanceName; + + // Vmware Remote Instances parameters + // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes + private String vcenterUsername; + @LogLevel(LogLevel.Log4jLevel.Off) + private String vcenterPassword; + private String vcenterHost; + private String datacenterName; + private String clusterName; + + public RemoteInstanceTO() { + } + + public RemoteInstanceTO(String hostName, String instanceName, String vcenterHost, + String datacenterName, String clusterName, + String vcenterUsername, String vcenterPassword) { + this.hypervisorType = Hypervisor.HypervisorType.VMware; + this.hostName = hostName; + this.instanceName = instanceName; + this.vcenterHost = vcenterHost; + this.datacenterName = datacenterName; + this.clusterName = clusterName; + this.vcenterUsername = vcenterUsername; + this.vcenterPassword = vcenterPassword; + } + + public Hypervisor.HypervisorType getHypervisorType() { + return this.hypervisorType; + } + + public String getInstanceName() { + return this.instanceName; + } + + public String getHostName() { + return this.hostName; + } + + public String getVcenterUsername() { + return vcenterUsername; + } + + public String getVcenterPassword() { + return vcenterPassword; + } + + public String getVcenterHost() { + return vcenterHost; + } + + public String getDatacenterName() { + return datacenterName; + } + + public String getClusterName() { + return clusterName; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java b/api/src/main/java/com/cloud/dc/VmwareDatacenter.java similarity index 96% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java rename to api/src/main/java/com/cloud/dc/VmwareDatacenter.java index b1f233c3606..859ff190b65 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenter.java +++ b/api/src/main/java/com/cloud/dc/VmwareDatacenter.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware; +package com.cloud.dc; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 1ac8d79b9d5..67fed5500ee 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -29,6 +29,8 @@ import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.ha.HAConfig; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.vm.schedule.VMSchedule; @@ -318,6 +320,7 @@ public class EventTypes { public static final String EVENT_DOMAIN_CREATE = "DOMAIN.CREATE"; public static final String EVENT_DOMAIN_DELETE = "DOMAIN.DELETE"; public static final String EVENT_DOMAIN_UPDATE = "DOMAIN.UPDATE"; + public static final String EVENT_DOMAIN_MOVE = "DOMAIN.MOVE"; // Snapshots public static final String EVENT_SNAPSHOT_COPY = "SNAPSHOT.COPY"; @@ -714,6 +717,16 @@ public class EventTypes { // SystemVM public static final String EVENT_LIVE_PATCH_SYSTEMVM = "LIVE.PATCH.SYSTEM.VM"; + // OBJECT STORE + public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE"; + public static final String EVENT_OBJECT_STORE_DELETE = "OBJECT.STORE.DELETE"; + public static final String EVENT_OBJECT_STORE_UPDATE = "OBJECT.STORE.UPDATE"; + + // BUCKETS + public static final String EVENT_BUCKET_CREATE = "BUCKET.CREATE"; + public static final String EVENT_BUCKET_DELETE = "BUCKET.DELETE"; + public static final String EVENT_BUCKET_UPDATE = "BUCKET.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -866,6 +879,7 @@ public class EventTypes { entityEventDetails.put(EVENT_DOMAIN_CREATE, Domain.class); entityEventDetails.put(EVENT_DOMAIN_DELETE, Domain.class); entityEventDetails.put(EVENT_DOMAIN_UPDATE, Domain.class); + entityEventDetails.put(EVENT_DOMAIN_MOVE, Domain.class); // Snapshots entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class); @@ -1151,6 +1165,16 @@ public class EventTypes { entityEventDetails.put(EVENT_IMAGE_STORE_DATA_MIGRATE, ImageStore.class); entityEventDetails.put(EVENT_IMAGE_STORE_OBJECT_DOWNLOAD, ImageStore.class); entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs"); + + //Object Store + entityEventDetails.put(EVENT_OBJECT_STORE_CREATE, ObjectStore.class); + entityEventDetails.put(EVENT_OBJECT_STORE_UPDATE, ObjectStore.class); + entityEventDetails.put(EVENT_OBJECT_STORE_DELETE, ObjectStore.class); + + //Buckets + entityEventDetails.put(EVENT_BUCKET_CREATE, Bucket.class); + entityEventDetails.put(EVENT_BUCKET_UPDATE, Bucket.class); + entityEventDetails.put(EVENT_BUCKET_DELETE, Bucket.class); } public static String getEntityForEvent(String eventName) { diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java index 2dfa707b57b..3c7dbac6442 100644 --- a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java @@ -33,6 +33,7 @@ import com.cloud.utils.component.Adapter; import com.cloud.vm.NicProfile; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; public interface HypervisorGuru extends Adapter { @@ -104,4 +105,25 @@ public interface HypervisorGuru extends Adapter { * @return a list of commands to perform for a successful migration */ List finalizeMigrate(VirtualMachine vm, Map volumeToPool); + + + /** + * Will perform a clone of a VM on an external host (if the guru can handle) + * @param hostIp VM's source host IP + * @param vmName name of the source VM to clone from + * @param params hypervisor specific additional parameters + * @return a reference to the cloned VM + */ + UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName, + Map params); + + /** + * Removes a VM created as a clone of a VM on an external host + * @param hostIp VM's source host IP + * @param vmName name of the VM to remove + * @param params hypervisor specific additional parameters + * @return true if the operation succeeds, false if not + */ + boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, + Map params); } diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index d959e3abce0..82d229da459 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -114,6 +114,8 @@ public interface NetworkService { IpAddress getIp(long id); + IpAddress getIp(String ipAddress); + Network updateGuestNetwork(final UpdateNetworkCmd cmd); /** diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index 2304241f7e5..9bbb5d43eae 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -69,7 +69,8 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit GuestOs(false, true), NetworkOffering(false, true), VpcOffering(true, false), - Domain(false, false, true); + Domain(false, false, true), + ObjectStore(false, false, true); ResourceObjectType(boolean resourceTagsSupport, boolean resourceMetadataSupport) { diff --git a/api/src/main/java/com/cloud/storage/DataStoreRole.java b/api/src/main/java/com/cloud/storage/DataStoreRole.java index cc20cc0ce96..185e370159c 100644 --- a/api/src/main/java/com/cloud/storage/DataStoreRole.java +++ b/api/src/main/java/com/cloud/storage/DataStoreRole.java @@ -21,7 +21,7 @@ package com.cloud.storage; import com.cloud.utils.exception.CloudRuntimeException; public enum DataStoreRole { - Primary("primary"), Image("image"), ImageCache("imagecache"), Backup("backup"); + Primary("primary"), Image("image"), ImageCache("imagecache"), Backup("backup"), Object("object"); public boolean isImageStore() { return (role.equalsIgnoreCase("image") || role.equalsIgnoreCase("imagecache")) ? true : false; @@ -45,6 +45,8 @@ public enum DataStoreRole { return ImageCache; } else if (role.equalsIgnoreCase("backup")) { return Backup; + } else if (role.equalsIgnoreCase("object")) { + return Object; } else { throw new CloudRuntimeException("can't identify the role"); } diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 1ee7200a313..8a2ec1a8905 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -77,13 +77,18 @@ public class Storage { } public static enum Capability { - HARDWARE_ACCELERATION("HARDWARE_ACCELERATION"); + HARDWARE_ACCELERATION("HARDWARE_ACCELERATION"), + ALLOW_MIGRATE_OTHER_POOLS("ALLOW_MIGRATE_OTHER_POOLS"); private final String capability; private Capability(String capability) { this.capability = capability; } + + public String toString() { + return this.capability; + } } public static enum ProvisioningType { @@ -150,7 +155,8 @@ public class Storage { ManagedNFS(true, false, false), Linstor(true, true, false), DatastoreCluster(true, true, false), // for VMware, to abstract pool of clusters - StorPool(true, true, true); + StorPool(true, true, true), + FiberChannel(true, true, false); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) private final boolean shared; private final boolean overprovisioning; diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index bb086ad05cb..c3609cfd8ee 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -24,9 +24,11 @@ import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaint import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import com.cloud.exception.DiscoveryException; @@ -34,6 +36,11 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.storage.object.ObjectStore; public interface StorageService { /** @@ -109,4 +116,15 @@ public interface StorageService { StoragePool syncStoragePool(SyncStoragePoolCmd cmd); + Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd); + + Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd); + + void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd); + + ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; + + boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd); + + ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd); } diff --git a/api/src/main/java/com/cloud/user/DomainService.java b/api/src/main/java/com/cloud/user/DomainService.java index 3ccfcbcea4c..06109cf5ff7 100644 --- a/api/src/main/java/com/cloud/user/DomainService.java +++ b/api/src/main/java/com/cloud/user/DomainService.java @@ -20,9 +20,11 @@ import java.util.List; import org.apache.cloudstack.api.command.admin.domain.ListDomainChildrenCmd; import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd; +import org.apache.cloudstack.api.command.admin.domain.MoveDomainCmd; import com.cloud.domain.Domain; import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.utils.Pair; public interface DomainService { @@ -66,4 +68,5 @@ public interface DomainService { */ Domain findDomainByIdOrPath(Long id, String domainPath); + Domain moveDomainAndChildrenToNewParentDomain(MoveDomainCmd cmd) throws ResourceAllocationException; } diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index d58b75b0dca..c32c099ed3a 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.vm; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -518,7 +519,8 @@ public interface UserVmService { UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceName, final String displayName, final Account owner, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKey, - final String hostName, final HypervisorType hypervisorType, final Map customParameters, final VirtualMachine.PowerState powerState) throws InsufficientCapacityException; + final String hostName, final HypervisorType hypervisorType, final Map customParameters, + final VirtualMachine.PowerState powerState, final LinkedHashMap> networkNicMap) throws InsufficientCapacityException; /** * Unmanage a guest VM from CloudStack diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 83c7529b22b..9338cc11cd4 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -39,6 +39,7 @@ public interface VmDetailConstants { // KVM specific (internal) String KVM_VNC_PORT = "kvm.vnc.port"; String KVM_VNC_ADDRESS = "kvm.vnc.address"; + String KVM_VNC_PASSWORD = "kvm.vnc.password"; // KVM specific, custom virtual GPU hardware String VIDEO_HARDWARE = "video.hardware"; @@ -86,4 +87,16 @@ public interface VmDetailConstants { String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; String KEY_PAIR_NAMES = "keypairnames"; String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; + + // VMware to KVM VM migrations specific + String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm"; + String VMWARE_VCENTER_HOST = String.format("%s-vcenter", VMWARE_TO_KVM_PREFIX); + String VMWARE_DATACENTER_NAME = String.format("%s-datacenter", VMWARE_TO_KVM_PREFIX); + String VMWARE_CLUSTER_NAME = String.format("%s-cluster", VMWARE_TO_KVM_PREFIX); + String VMWARE_VCENTER_USERNAME = String.format("%s-username", VMWARE_TO_KVM_PREFIX); + String VMWARE_VCENTER_PASSWORD = String.format("%s-password", VMWARE_TO_KVM_PREFIX); + String VMWARE_VM_NAME = String.format("%s-vmname", VMWARE_TO_KVM_PREFIX); + String VMWARE_HOST_NAME = String.format("%s-host", VMWARE_TO_KVM_PREFIX); + String VMWARE_DISK = String.format("%s-disk", VMWARE_TO_KVM_PREFIX); + String VMWARE_MAC_ADDRESSES = String.format("%s-mac-addresses", VMWARE_TO_KVM_PREFIX); } diff --git a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java index 51c6286a9d5..64bd96d9a9b 100644 --- a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java +++ b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java @@ -45,7 +45,7 @@ public interface AnnotationService { SERVICE_OFFERING(false), DISK_OFFERING(false), NETWORK_OFFERING(false), ZONE(false), POD(false), CLUSTER(false), HOST(false), DOMAIN(false), PRIMARY_STORAGE(false), SECONDARY_STORAGE(false), VR(false), SYSTEM_VM(false), - AUTOSCALE_VM_GROUP(true), MANAGEMENT_SERVER(false),; + AUTOSCALE_VM_GROUP(true), MANAGEMENT_SERVER(false), OBJECT_STORAGE(false); private final boolean usersAllowed; @@ -78,6 +78,7 @@ public interface AnnotationService { list.add(EntityType.VR); list.add(EntityType.SYSTEM_VM); list.add(EntityType.MANAGEMENT_SERVER); + list.add(EntityType.OBJECT_STORAGE); if (roleType != RoleType.DomainAdmin) { list.add(EntityType.DOMAIN); list.add(EntityType.SERVICE_OFFERING); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 9267ca6fa96..affceb4e3f9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -78,7 +78,9 @@ public enum ApiCommandResourceType { VmSnapshot(com.cloud.vm.snapshot.VMSnapshot.class), Role(org.apache.cloudstack.acl.Role.class), VpnCustomerGateway(com.cloud.network.Site2SiteCustomerGateway.class), - ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class); + ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class), + ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class), + Bucket(org.apache.cloudstack.storage.object.Bucket.class); private final Class clazz; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 5b6647991ef..d7767721667 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -69,6 +69,8 @@ public class ApiConstants { public static final String CERTIFICATE_SERIALNUM = "serialnum"; public static final String CERTIFICATE_SUBJECT = "subject"; public static final String CERTIFICATE_VALIDITY = "validity"; + public static final String CONVERT_INSTANCE_HOST_ID = "convertinstancehostid"; + public static final String CONVERT_INSTANCE_STORAGE_POOL_ID = "convertinstancepoolid"; public static final String ENABLED_REVOCATION_CHECK = "enabledrevocationcheck"; public static final String CONTROLLER = "controller"; public static final String CONTROLLER_UNIT = "controllerunit"; @@ -120,6 +122,7 @@ public class ApiConstants { public static final String MIN_IOPS = "miniops"; public static final String MAX_IOPS = "maxiops"; public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve"; + public static final String DATACENTER_NAME = "datacentername"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DESCRIPTION = "description"; @@ -207,7 +210,9 @@ public class ApiConstants { public static final String HIDE_IP_ADDRESS_USAGE = "hideipaddressusage"; public static final String HOST_ID = "hostid"; public static final String HOST_IDS = "hostids"; + public static final String HOST_IP = "hostip"; public static final String HOST_NAME = "hostname"; + public static final String HOST = "host"; public static final String HOST_CONTROL_STATE = "hostcontrolstate"; public static final String HOSTS_MAP = "hostsmap"; public static final String HYPERVISOR = "hypervisor"; @@ -779,6 +784,7 @@ public class ApiConstants { public static final String VSM_CONFIG_STATE = "vsmconfigstate"; public static final String VSM_DEVICE_STATE = "vsmdevicestate"; public static final String VCENTER = "vcenter"; + public static final String EXISTING_VCENTER_ID = "existingvcenterid"; public static final String ADD_VSM_FLAG = "addvsmflag"; public static final String END_POINT = "endpoint"; public static final String REGION_ID = "regionid"; @@ -1049,10 +1055,23 @@ public class ApiConstants { public static final String MTU = "mtu"; public static final String AUTO_ENABLE_KVM_HOST = "autoenablekvmhost"; public static final String LIST_APIS = "listApis"; + public static final String OBJECT_STORAGE_ID = "objectstorageid"; + public static final String VERSIONING = "versioning"; + public static final String OBJECT_LOCKING = "objectlocking"; + public static final String ENCRYPTION = "encryption"; + public static final String QUOTA = "quota"; + public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; public static final String SOURCE_NAT_IP_ID = "sourcenatipaddressid"; public static final String HAS_RULES = "hasrules"; + public static final String DISK_PATH = "diskpath"; + public static final String IMPORT_SOURCE = "importsource"; + public static final String TEMP_PATH = "temppath"; + public static final String OBJECT_STORAGE = "objectstore"; + + public static final String HEURISTIC_RULE = "heuristicrule"; + public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME."; public static final String MANAGEMENT = "management"; public static final String IS_VNF = "isvnf"; @@ -1066,6 +1085,10 @@ public class ApiConstants { public static final String CLIENT_ID = "clientid"; public static final String REDIRECT_URI = "redirecturi"; + public static final String IS_TAG_A_RULE = "istagarule"; + + public static final String PARAMETER_DESCRIPTION_IS_TAG_A_RULE = "Whether the informed tag is a JS interpretable rule or not."; + /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index 0b80cfc8229..f32922819b0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.network.element.InternalLoadBalancerElementService; import org.apache.cloudstack.network.lb.ApplicationLoadBalancerService; import org.apache.cloudstack.network.lb.InternalLoadBalancerVMService; import org.apache.cloudstack.query.QueryService; +import org.apache.cloudstack.storage.object.BucketApiService; import org.apache.cloudstack.storage.ImageStoreService; import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.usage.UsageService; @@ -216,6 +217,9 @@ public abstract class BaseCmd { public Ipv6Service ipv6Service; @Inject public VnfTemplateManager vnfTemplateManager; + @Inject + public BucketApiService _bucketService; + public abstract void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 030f70805d2..ef759aaf9c3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ApiConstants.HostDetails; @@ -37,6 +38,7 @@ import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.CapacityResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConditionResponse; @@ -82,6 +84,7 @@ import org.apache.cloudstack.api.response.NetworkPermissionsResponse; import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.NicResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.OvsProviderResponse; import org.apache.cloudstack.api.response.PhysicalNetworkResponse; import org.apache.cloudstack.api.response.PodResponse; @@ -101,6 +104,7 @@ import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RollingMaintenanceResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SSHKeyPairResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; @@ -119,6 +123,7 @@ import org.apache.cloudstack.api.response.TemplatePermissionsResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.TrafficMonitorResponse; import org.apache.cloudstack.api.response.TrafficTypeResponse; +import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.UpgradeRouterTemplateResponse; import org.apache.cloudstack.api.response.UsageRecordResponse; import org.apache.cloudstack.api.response.UserDataResponse; @@ -145,6 +150,8 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import com.cloud.capacity.Capacity; @@ -231,6 +238,7 @@ import com.cloud.vm.Nic; import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.VirtualMachine; import com.cloud.vm.snapshot.VMSnapshot; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; public interface ResponseGenerator { UserResponse createUserResponse(UserAccount user); @@ -532,5 +540,13 @@ public interface ResponseGenerator { FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl); + UnmanagedInstanceResponse createUnmanagedInstanceResponse(UnmanagedInstanceTO instance, Cluster cluster, Host host); + + SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic); + IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp); + + ObjectStoreResponse createObjectStoreResponse(ObjectStore os); + + BucketResponse createBucketResponse(Bucket bucket); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/MoveDomainCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/MoveDomainCmd.java new file mode 100644 index 00000000000..586345b2de7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/MoveDomainCmd.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.domain; + +import com.cloud.domain.Domain; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.DomainResponse; + +@APICommand(name = "moveDomain", description = "Moves a domain and its children to a new parent domain.", since = "4.19.0.0", responseObject = DomainResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class MoveDomainCmd extends BaseCmd { + + private static final String APINAME = "moveDomain"; + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "The ID of the domain to be moved.") + private Long domainId; + + @Parameter(name = ApiConstants.PARENT_DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, + description = "The ID of the new parent domain of the domain to be moved.") + private Long parentDomainId; + + public Long getDomainId() { + return domainId; + } + + public Long getParentDomainId() { + return parentDomainId; + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() throws ResourceAllocationException { + Domain domain = _domainService.moveDomainAndChildrenToNewParentDomain(this); + + if (domain != null) { + DomainResponse response = _responseGenerator.createDomainResponse(domain); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to move the domain."); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java index e3ff130e2d4..9cf47a9c4b9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java @@ -60,6 +60,9 @@ public class UpdateHostCmd extends BaseCmd { @Parameter(name = ApiConstants.HOST_TAGS, type = CommandType.LIST, collectionType = CommandType.STRING, description = "list of tags to be added to the host") private List hostTags; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the new uri for the secondary storage: nfs://host/path") private String url; @@ -90,6 +93,10 @@ public class UpdateHostCmd extends BaseCmd { return hostTags; } + public Boolean getIsTagARule() { + return isTagARule; + } + public String getUrl() { return url; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java new file mode 100644 index 00000000000..a538962e076 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.storage.object.ObjectStore; +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.log4j.Logger; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +@APICommand(name = "addObjectStoragePool", description = "Adds a object storage pool", responseObject = ObjectStoreResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class AddObjectStoragePoolCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(AddObjectStoragePoolCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "the name for the object store") + private String name; + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, length = 2048, required = true, description = "the URL for the object store") + private String url; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, required = true, description = "the object store provider name") + private String providerName; + + @Parameter(name = ApiConstants.DETAILS, + type = CommandType.MAP, + description = "the details for the object store. Example: details[0].key=accesskey&details[0].value=s389ddssaa&details[1].key=secretkey&details[1].value=8dshfsss") + private Map details; + + @Parameter(name = ApiConstants.TAGS, type = CommandType.STRING, description = "the tags for the storage pool") + private String tags; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getUrl() { + return url; + } + + public String getName() { + return name; + } + + public Map getDetails() { + Map detailsMap = null; + if (details != null && !details.isEmpty()) { + detailsMap = new HashMap(); + Collection props = details.values(); + Iterator iter = props.iterator(); + while (iter.hasNext()) { + HashMap detail = (HashMap)iter.next(); + String key = detail.get(ApiConstants.KEY); + String value = detail.get(ApiConstants.VALUE); + detailsMap.put(key, value); + } + } + return detailsMap; + } + + public String getProviderName() { + return providerName; + } + + public void setUrl(String url) { + this.url = url; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public void setDetails(Map details) { + this.details = details; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute(){ + try{ + ObjectStore result = _storageService.discoverObjectStore(getName(), getUrl(), getProviderName(), getDetails()); + ObjectStoreResponse storeResponse = null; + if (result != null) { + storeResponse = _responseGenerator.createObjectStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("objectstore"); + setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add object storage"); + } + } catch (Exception ex) { + s_logger.error("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java index 0eacc5cda6b..477d7570dfa 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java @@ -90,6 +90,9 @@ public class CreateStoragePoolCmd extends BaseCmd { description = "hypervisor type of the hosts in zone that will be attached to this storage pool. KVM, VMware supported as of now.") private String hypervisor; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -146,6 +149,10 @@ public class CreateStoragePoolCmd extends BaseCmd { return hypervisor; } + public Boolean isTagARule() { + return this.isTagARule; + } + @Override public long getEntityOwnerId() { return Account.ACCOUNT_ID_SYSTEM; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java new file mode 100644 index 00000000000..ed305d9689d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "deleteObjectStoragePool", description = "Deletes an Object Storage Pool", responseObject = SuccessResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class DeleteObjectStoragePoolCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(DeleteObjectStoragePoolCmd.class.getName()); + + // /////////////////////////////////////////////////// + // ////////////// API parameters ///////////////////// + // /////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, required = true, description = "The Object Storage ID.") + private Long id; + + // /////////////////////////////////////////////////// + // ///////////////// Accessors /////////////////////// + // /////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + // /////////////////////////////////////////////////// + // ///////////// API Implementation/////////////////// + // /////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + boolean result = _storageService.deleteObjectStore(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete object store"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java new file mode 100644 index 00000000000..9d8d8eccc3c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "listObjectStoragePools", description = "Lists object storage pools.", responseObject = ObjectStoreResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListObjectStoragePoolsCmd extends BaseListCmd { + public static final Logger s_logger = Logger.getLogger(ListObjectStoragePoolsCmd.class.getName()); + + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the object store") + private String storeName; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, description = "the object store provider") + private String provider; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, description = "the ID of the storage pool") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public String getStoreName() { + return storeName; + } + + public Long getId() { + return id; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ListResponse response = _queryService.searchForObjectStores(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java new file mode 100644 index 00000000000..497179d25ef --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = UpdateObjectStoragePoolCmd.APINAME, description = "Updates object storage pool", responseObject = ObjectStoreResponse.class, entityType = {ObjectStore.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0") +public class UpdateObjectStoragePoolCmd extends BaseCmd { + public static final String APINAME = "updateObjectStoragePool"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, required = true, description = "Object Store ID") + private Long id; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name for the object store") + private String name; + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the url for the object store") + private String url; + + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getUrl() { + return url; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ObjectStore result = _storageService.updateObjectStore(getId(), this); + + ObjectStoreResponse storeResponse = null; + if (result != null) { + storeResponse = _responseGenerator.createObjectStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("objectstore"); + setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update object storage"); + } + } + + @Override + public String getCommandName() { + return APINAME; + } + + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java index 9e34684a09d..7a907e0f76a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.admin.storage; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.log4j.Logger; @@ -32,6 +33,7 @@ import org.apache.cloudstack.api.response.StoragePoolResponse; import com.cloud.storage.StoragePool; import com.cloud.user.Account; +@SuppressWarnings("rawtypes") @APICommand(name = "updateStoragePool", description = "Updates a storage pool.", responseObject = StoragePoolResponse.class, since = "3.0.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class UpdateStoragePoolCmd extends BaseCmd { @@ -61,6 +63,23 @@ public class UpdateStoragePoolCmd extends BaseCmd { " enable it back.") private Boolean enabled; + @Parameter(name = ApiConstants.DETAILS, + type = CommandType.MAP, + required = false, + description = "the details for the storage pool", + since = "4.19.0") + private Map details; + + @Parameter(name = ApiConstants.URL, + type = CommandType.STRING, + required = false, + description = "the URL of the storage pool", + since = "4.19.0") + private String url; + + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -89,6 +108,10 @@ public class UpdateStoragePoolCmd extends BaseCmd { return enabled; } + public Boolean isTagARule() { + return isTagARule; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -108,6 +131,22 @@ public class UpdateStoragePoolCmd extends BaseCmd { return ApiCommandResourceType.StoragePool; } + public Map getDetails() { + return details; + } + + public void setDetails(Map details) { + this.details = details; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + @Override public void execute() { StoragePool result = _storageService.updateStoragePool(this); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..a0e99b55971 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "createSecondaryStorageSelector", description = "Creates a secondary storage selector, described by the heuristic rule.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, entityType = {Heuristic.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class CreateSecondaryStorageSelectorCmd extends BaseCmd{ + + @Parameter(name = ApiConstants.NAME, required = true, type = CommandType.STRING, description = "The name identifying the heuristic rule.") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, required = true, type = BaseCmd.CommandType.STRING, description = "The description of the heuristic rule.") + private String description; + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = BaseCmd.CommandType.UUID, description = "The zone in which the heuristic " + + "rule will be applied.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, required = true, type = BaseCmd.CommandType.STRING, description = + "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.createSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create a secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java new file mode 100644 index 00000000000..c437cd660e7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "listSecondaryStorageSelectors", description = "Lists the secondary storage selectors and their rules.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class ListSecondaryStorageSelectorsCmd extends BaseListCmd { + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = CommandType.UUID, description = "The zone ID to be used in the search filter.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = + "Whether to filter the selectors by type and, if so, which one. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Show removed heuristics.") + private boolean showRemoved = false; + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public boolean isShowRemoved() { + return showRemoved; + } + + @Override + public void execute() { + ListResponse response = _queryService.listSecondaryStorageSelectors(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..79554f44782 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "removeSecondaryStorageSelector", description = "Removes an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class RemoveSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector to be removed.") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() { + _storageService.removeSecondaryStorageHeuristic(this); + final SuccessResponse response = new SuccessResponse(); + response.setResponseName(getCommandName()); + response.setSuccess(true); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..e63d1cbfa27 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "updateSecondaryStorageSelector", description = "Updates an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class UpdateSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector.") + private Long id; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public Long getId() { + return id; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.updateSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java index 70233317cc5..d632c786a16 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java @@ -84,7 +84,7 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, - description = "the hypervisor name of the instance") + description = "the name of the instance as it is known to the hypervisor") private String name; @Parameter(name = ApiConstants.DISPLAY_NAME, @@ -124,7 +124,7 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { type = CommandType.UUID, entityType = ServiceOfferingResponse.class, required = true, - description = "the ID of the service offering for the virtual machine") + description = "the service offering for the virtual machine") private Long serviceOfferingId; @Parameter(name = ApiConstants.NIC_NETWORK_LIST, @@ -154,7 +154,7 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, - description = "VM is imported despite some of its NIC's MAC addresses are already present") + description = "VM is imported despite some of its NIC's MAC addresses are already present, in case the MAC address exists then a new MAC address is generated") private Boolean forced; ///////////////////////////////////////////////////// @@ -279,7 +279,8 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { @Override public String getEventDescription() { - return "Importing unmanaged VM"; + String vmName = this.name; + return String.format("Importing unmanaged VM: %s", vmName); } public boolean isForced() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java new file mode 100644 index 00000000000..e8b9f3addde --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -0,0 +1,258 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.NetworkResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.api.response.VmwareDatacenterResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.vm.VmImportService; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import javax.inject.Inject; + +@APICommand(name = "importVm", + description = "Import virtual machine from a unmanaged host into CloudStack", + responseObject = UserVmResponse.class, + responseView = ResponseObject.ResponseView.Full, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin}, + since = "4.19.0") +public class ImportVmCmd extends ImportUnmanagedInstanceCmd { + public static final Logger LOGGER = Logger.getLogger(ImportVmCmd.class); + + @Inject + public VmImportService vmImportService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + + @Parameter(name = ApiConstants.ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + required = true, + description = "the zone ID") + private Long zoneId; + + @Parameter(name = ApiConstants.USERNAME, + type = CommandType.STRING, + description = "the username for the host") + private String username; + + @Parameter(name = ApiConstants.PASSWORD, + type = CommandType.STRING, + description = "the password for the host") + private String password; + + @Parameter(name = ApiConstants.HOST, + type = CommandType.STRING, + description = "the host name or IP address") + private String host; + + @Parameter(name = ApiConstants.HYPERVISOR, + type = CommandType.STRING, + required = true, + description = "hypervisor type of the host") + private String hypervisor; + + @Parameter(name = ApiConstants.DISK_PATH, + type = CommandType.STRING, + description = "path of the disk image") + private String diskPath; + + @Parameter(name = ApiConstants.IMPORT_SOURCE, + type = CommandType.STRING, + required = true, + description = "Source location for Import" ) + private String importSource; + + @Parameter(name = ApiConstants.NETWORK_ID, + type = CommandType.UUID, + entityType = NetworkResponse.class, + description = "the network ID") + private Long networkId; + + @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "Host where local disk is located") + private Long hostId; + + @Parameter(name = ApiConstants.STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "Shared storage pool where disk is located") + private Long storagePoolId; + + @Parameter(name = ApiConstants.TEMP_PATH, + type = CommandType.STRING, + description = "Temp Path on external host for disk image copy" ) + private String tmpPath; + + // Import from Vmware to KVM migration parameters + + @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, + type = CommandType.UUID, + entityType = VmwareDatacenterResponse.class, + description = "(only for importing migrated VMs from Vmware to KVM) UUID of a linked existing vCenter") + private Long existingVcenterId; + + @Parameter(name = ApiConstants.HOST_IP, + type = BaseCmd.CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) VMware ESXi host IP/Name.") + private String hostip; + + @Parameter(name = ApiConstants.VCENTER, + type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + private String vcenter; + + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware datacenter.") + private String datacenterName; + + @Parameter(name = ApiConstants.CLUSTER_NAME, type = CommandType.STRING, + description = "(only for importing migrated VMs from Vmware to KVM) Name of VMware cluster.") + private String clusterName; + + @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, + description = "(only for importing migrated VMs from Vmware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") + private Long convertInstanceHostId; + + @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, + description = "(only for importing migrated VMs from Vmware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") + private Long convertStoragePoolId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getZoneId() { + return zoneId; + } + + public Long getExistingVcenterId() { + return existingVcenterId; + } + + public String getHostIp() { + return hostip; + } + + public String getVcenter() { + return vcenter; + } + + public String getDatacenterName() { + return datacenterName; + } + + public String getClusterName() { + return clusterName; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getHost() { + return host; + } + + public Long getConvertInstanceHostId() { + return convertInstanceHostId; + } + + public Long getConvertStoragePoolId() { + return convertStoragePoolId; + } + + public String getHypervisor() { + return hypervisor; + } + + public String getDiskPath() { + return diskPath; + } + + public String getImportSource() { + return importSource; + } + + public Long getHostId() { + return hostId; + } + + public Long getStoragePoolId() { + return storagePoolId; + } + + public String getTmpPath() { + return tmpPath; + } + + public Long getNetworkId() { + return networkId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_VM_IMPORT; + } + + @Override + public String getEventDescription() { + String vmName = getName(); + if (ObjectUtils.anyNotNull(vcenter, existingVcenterId)) { + String msg = StringUtils.isNotBlank(vcenter) ? + String.format("external vCenter: %s - datacenter: %s", vcenter, datacenterName) : + String.format("existing vCenter Datacenter with ID: %s", existingVcenterId); + return String.format("Importing unmanaged VM: %s from %s - VM: %s", getDisplayName(), msg, vmName); + } + return String.format("Importing unmanaged VM: %s", vmName); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + UserVmResponse response = vmImportService.importVm(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListVmsForImportCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListVmsForImportCmd.java new file mode 100644 index 00000000000..88df04d9ef5 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListVmsForImportCmd.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.vm; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.cloudstack.vm.VmImportService; +import org.apache.log4j.Logger; + +import javax.inject.Inject; + +@APICommand(name = "listVmsForImport", + description = "Lists virtual machines on a unmanaged host", + responseObject = UnmanagedInstanceResponse.class, + responseView = ResponseObject.ResponseView.Full, + entityType = {UnmanagedInstanceTO.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin}, + since = "4.19.0") +public class ListVmsForImportCmd extends BaseListCmd { + public static final Logger LOGGER = Logger.getLogger(ListVmsForImportCmd.class.getName()); + + @Inject + public VmImportService vmImportService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + required = true, + description = "the zone ID") + private Long zoneId; + + @Parameter(name = ApiConstants.USERNAME, + type = CommandType.STRING, + description = "the username for the host") + private String username; + + @Parameter(name = ApiConstants.PASSWORD, + type = CommandType.STRING, + description = "the password for the host") + private String password; + + @Parameter(name = ApiConstants.HOST, + type = CommandType.STRING, + required = true, + description = "the host name or IP address") + private String host; + + @Parameter(name = ApiConstants.HYPERVISOR, + type = CommandType.STRING, + required = true, + description = "hypervisor type of the host") + private String hypervisor; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getZoneId() { + return zoneId; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getHost() { + return host; + } + + public String getHypervisor() { + return hypervisor; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + ListResponse response = vmImportService.listVmsForImport(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + if (account != null) { + return account.getId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java index b31520e1b88..f9bfcb253b4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java @@ -46,10 +46,14 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, required = true, description = "the ID of the public IP address" - + " to disassociate") + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, description = "the ID of the public IP address" + + " to disassociate. Mutually exclusive with the ipaddress parameter") private Long id; + @Parameter(name=ApiConstants.IP_ADDRESS, type=CommandType.STRING, since="4.19.0", description="IP Address to be disassociated." + + " Mutually exclusive with the id parameter") + private String ipAddress; + // unexposed parameter needed for events logging @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, expose = false) private Long ownerId; @@ -59,7 +63,18 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// public Long getIpAddressId() { - return id; + if (id != null & ipAddress != null) { + throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter"); + } + + if (id != null) { + return id; + } else if (ipAddress != null) { + IpAddress ip = getIpAddressByIp(ipAddress); + return ip.getId(); + } + + throw new InvalidParameterValueException("Please specify either IP address or IP address ID"); } ///////////////////////////////////////////////////// @@ -68,12 +83,13 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public void execute() throws InsufficientAddressCapacityException { - CallContext.current().setEventDetails("IP ID: " + getIpAddressId()); + Long ipAddressId = getIpAddressId(); + CallContext.current().setEventDetails("IP ID: " + ipAddressId); boolean result = false; - if (!isPortable(id)) { - result = _networkService.releaseIpAddress(getIpAddressId()); + if (!isPortable()) { + result = _networkService.releaseIpAddress(ipAddressId); } else { - result = _networkService.releasePortableIpAddress(getIpAddressId()); + result = _networkService.releasePortableIpAddress(ipAddressId); } if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); @@ -85,7 +101,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public String getEventType() { - if (!isPortable(id)) { + if (!isPortable()) { return EventTypes.EVENT_NET_IP_RELEASE; } else { return EventTypes.EVENT_PORTABLE_IP_RELEASE; @@ -100,10 +116,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { if (ownerId == null) { - IpAddress ip = getIpAddress(id); - if (ip == null) { - throw new InvalidParameterValueException("Unable to find IP address by ID=" + id); - } + IpAddress ip = getIpAddress(); ownerId = ip.getAccountId(); } @@ -120,11 +133,11 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public Long getSyncObjId() { - IpAddress ip = getIpAddress(id); + IpAddress ip = getIpAddress(); return ip.getAssociatedWithNetworkId(); } - private IpAddress getIpAddress(long id) { + private IpAddress getIpAddressById(Long id) { IpAddress ip = _entityMgr.findById(IpAddress.class, id); if (ip == null) { @@ -134,6 +147,29 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { } } + private IpAddress getIpAddressByIp(String ipAddress) { + IpAddress ip = _networkService.getIp(ipAddress); + if (ip == null) { + throw new InvalidParameterValueException("Unable to find IP address by IP address=" + ipAddress); + } else { + return ip; + } + } + + private IpAddress getIpAddress() { + if (id != null & ipAddress != null) { + throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter"); + } + + if (id != null) { + return getIpAddressById(id); + } else if (ipAddress != null){ + return getIpAddressByIp(ipAddress); + } + + throw new InvalidParameterValueException("Please specify either IP address or IP address ID"); + } + @Override public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.IpAddress; @@ -144,8 +180,8 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { return getIpAddressId(); } - private boolean isPortable(long id) { - IpAddress ip = getIpAddress(id); + private boolean isPortable() { + IpAddress ip = getIpAddress(); return ip.isPortable(); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java new file mode 100644 index 00000000000..e9a140cf46e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceAllocationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "createBucket", responseObject = BucketResponse.class, + description = "Creates a bucket in the specified object storage pool. ", responseView = ResponseView.Restricted, + entityType = {Bucket.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(CreateBucketCmd.class.getName()); + private static final String s_name = "createbucketresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ACCOUNT, + type = CommandType.STRING, + description = "the account associated with the bucket. Must be used with the domainId parameter.") + private String accountName; + + @Parameter(name = ApiConstants.PROJECT_ID, + type = CommandType.UUID, + entityType = ProjectResponse.class, + description = "the project associated with the bucket. Mutually exclusive with account parameter") + private Long projectId; + + @Parameter(name = ApiConstants.DOMAIN_ID, + type = CommandType.UUID, + entityType = DomainResponse.class, + description = "the domain ID associated with the bucket. If used with the account parameter" + + " returns the bucket associated with the account for the specified domain.") + private Long domainId; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true,description = "the name of the bucket") + private String bucketName; + + @Parameter(name = ApiConstants.OBJECT_STORAGE_ID, type = CommandType.UUID, + entityType = ObjectStoreResponse.class, required = true, + description = "Id of the Object Storage Pool where bucket is created") + private long objectStoragePoolId; + + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + private Integer quota; + + @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable bucket encryption") + private boolean encryption; + + @Parameter(name = ApiConstants.VERSIONING, type = CommandType.BOOLEAN, description = "Enable bucket versioning") + private boolean versioning; + + @Parameter(name = ApiConstants.OBJECT_LOCKING, type = CommandType.BOOLEAN, description = "Enable object locking in bucket") + private boolean objectLocking; + + @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING,description = "The Bucket access policy") + private String policy; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getAccountName() { + return accountName; + } + + public Long getDomainId() { + return domainId; + } + + public String getBucketName() { + return bucketName; + } + + private Long getProjectId() { + return projectId; + } + + public long getObjectStoragePoolId() { + return objectStoragePoolId; + } + + public Integer getQuota() { + return quota; + } + + public boolean isEncryption() { + return encryption; + } + + public boolean isVersioning() { + return versioning; + } + + public boolean isObjectLocking() { + return objectLocking; + } + + public String getPolicy() { + return policy; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public String getCommandName() { + return s_name; + } + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + + return accountId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_BUCKET_CREATE; + } + + @Override + public String getEventDescription() { + return "creating bucket: " + getBucketName(); + } + + @Override + public void create() throws ResourceAllocationException { + Bucket bucket = _bucketService.allocBucket(this); + if (bucket != null) { + setEntityId(bucket.getId()); + setEntityUuid(bucket.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create bucket"); + } + } + + @Override + public void execute() { + CallContext.current().setEventDetails("Bucket Id: " + getEntityUuid()); + + Bucket bucket; + try { + bucket = _bucketService.createBucket(this); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + if (bucket != null) { + BucketResponse response = _responseGenerator.createBucketResponse(bucket); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create bucket with name: "+getBucketName()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java new file mode 100644 index 00000000000..bf9552b779e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "deleteBucket", description = "Deletes an empty Bucket.", responseObject = SuccessResponse.class, entityType = {Bucket.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class DeleteBucketCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(DeleteBucketCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=BucketResponse.class, + required=true, description="The ID of the Bucket") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public long getEntityOwnerId() { + Bucket Bucket = _entityMgr.findById(Bucket.class, getId()); + if (Bucket != null) { + return Bucket.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public Long getApiResourceId() { + return id; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() throws ConcurrentOperationException { + CallContext.current().setEventDetails("Bucket Id: " + this._uuidMgr.getUuid(Bucket.class, getId())); + boolean result = _bucketService.deleteBucket(id, CallContext.current().getCallingAccount()); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java new file mode 100644 index 00000000000..897b9fc6696 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.log4j.Logger; + +import java.util.List; + +@APICommand(name = "listBuckets", description = "Lists all Buckets.", responseObject = BucketResponse.class, responseView = ResponseView.Restricted, entityType = { + Bucket.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListBucketsCmd extends BaseListTaggedResourcesCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(ListBucketsCmd.class.getName()); + + private static final String s_name = "listbucketsresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = BucketResponse.class, description = "the ID of the bucket") + private Long id; + + @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = BucketResponse.class, description = "the IDs of the Buckets, mutually exclusive with id") + private List ids; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the bucket") + private String bucketName; + + @Parameter(name = ApiConstants.OBJECT_STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "the ID of the object storage pool, available to ROOT admin only", authorized = { + RoleType.Admin}) + private Long objectStorageId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getBucketName() { + return bucketName; + } + + public Long getObjectStorageId() { + return objectStorageId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() { + ListResponse response = _queryService.searchForBuckets(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + public List getIds() { + return ids; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java new file mode 100644 index 00000000000..b3b7e00770d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "updateBucket", description = "Updates Bucket properties", responseObject = SuccessResponse.class, entityType = {Bucket.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class UpdateBucketCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(UpdateBucketCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=BucketResponse.class, + required=true, description="The ID of the Bucket") + private Long id; + + @Parameter(name = ApiConstants.VERSIONING, type = CommandType.BOOLEAN, description = "Enable/Disable Bucket Versioning") + private Boolean versioning; + + @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable/Disable Bucket encryption") + private Boolean encryption; + + @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING, description = "Bucket Access Policy") + private String policy; + + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + private Integer quota; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Boolean getVersioning() { + return versioning; + } + + public Boolean getEncryption() { + return encryption; + } + + public String getPolicy() { + return policy; + } + + public Integer getQuota() { + return quota; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public long getEntityOwnerId() { + Bucket Bucket = _entityMgr.findById(Bucket.class, getId()); + if (Bucket != null) { + return Bucket.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public Long getApiResourceId() { + return id; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() throws ConcurrentOperationException { + CallContext.current().setEventDetails("Bucket Id: " + this._uuidMgr.getUuid(Bucket.class, getId())); + boolean result = false; + try { + result = _bucketService.updateBucket(this, CallContext.current().getCallingAccount()); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Error while updating bucket. "+e.getMessage()); + } + if(result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update bucket"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java index a7a418f5a86..e1759566299 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -54,7 +55,6 @@ public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { @Parameter(name = ApiConstants.DISPLAY_TEXT, type = BaseCmd.CommandType.STRING, - required = true, description = "the display text of the ISO. This is usually used for display purposes.", length = 4096) private String displayText; @@ -85,7 +85,7 @@ public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { } public String getDisplayText() { - return displayText; + return StringUtils.isBlank(displayText) ? getName() : displayText; } public Boolean isFeatured() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/ListLoadBalancerRuleInstancesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/ListLoadBalancerRuleInstancesCmd.java index 77aaa6bc1d3..723e0efec12 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/ListLoadBalancerRuleInstancesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/ListLoadBalancerRuleInstancesCmd.java @@ -97,52 +97,44 @@ public class ListLoadBalancerRuleInstancesCmd extends BaseListCmd implements Use public void execute() { Pair, List> vmServiceMap = _lbService.listLoadBalancerInstances(this); List result = vmServiceMap.first(); + s_logger.debug(String.format("A total of [%s] user VMs were obtained when listing the load balancer instances: [%s].", result.size(), result)); + List serviceStates = vmServiceMap.second(); + s_logger.debug(String.format("A total of [%s] service states were obtained when listing the load balancer instances: [%s].", serviceStates.size(), serviceStates)); if (!isListLbVmip()) { - // list lb instances - ListResponse response = new ListResponse(); - List vmResponses = new ArrayList(); - if (result != null) { - vmResponses = _responseGenerator.createUserVmResponse(ResponseView.Restricted, "loadbalancerruleinstance", result.toArray(new UserVm[result.size()])); + ListResponse response = new ListResponse<>(); + List vmResponses = _responseGenerator.createUserVmResponse(ResponseView.Restricted, "loadbalancerruleinstance", result.toArray(new UserVm[0])); - - for (int i = 0; i < result.size(); i++) { - vmResponses.get(i).setServiceState(serviceStates.get(i)); - } + for (int i = 0; i < result.size(); i++) { + vmResponses.get(i).setServiceState(serviceStates.get(i)); } + response.setResponses(vmResponses); response.setResponseName(getCommandName()); setResponseObject(response); - - - } else { - ListResponse lbRes = new ListResponse(); - - List vmResponses = new ArrayList(); - List listlbVmRes = new ArrayList(); - - if (result != null) { - vmResponses = _responseGenerator.createUserVmResponse(getResponseView(), "loadbalancerruleinstance", result.toArray(new UserVm[result.size()])); - - - List ipaddr = null; - - for (int i=0;i lbRes = new ListResponse<>(); + + List vmResponses = _responseGenerator.createUserVmResponse(getResponseView(), "loadbalancerruleinstance", result.toArray(new UserVm[0])); + List lbRuleVmMapList = new ArrayList<>(); + + for (int i=0; i(); + } + + @Override + public String getObjectId() { + return this.getId(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setCreated(Date created) { + this.created = created; + } + + @Override + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + @Override + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + @Override + public void setDomainName(String domainName) { + this.domainName = domainName; + } + + @Override + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + @Override + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setTags(Set tags) { + this.tags = tags; + } + + public void setObjectStoragePoolId(String objectStoragePoolId) { + this.objectStoragePoolId = objectStoragePoolId; + } + + public String getName() { + return name; + } + + public Date getCreated() { + return created; + } + + public String getAccountName() { + return accountName; + } + + public String getProjectId() { + return projectId; + } + + public String getProjectName() { + return projectName; + } + + public String getDomainId() { + return domainId; + } + + public String getDomainName() { + return domainName; + } + + public String getObjectStoragePoolId() { + return objectStoragePoolId; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + public long getQuota() { + return quota; + } + + public void setQuota(Integer quota) { + this.quota = quota; + } + + public boolean isVersioning() { + return versioning; + } + + public void setVersioning(boolean versioning) { + this.versioning = versioning; + } + + public boolean isEncryption() { + return encryption; + } + + public void setEncryption(boolean encryption) { + this.encryption = encryption; + } + + public boolean isObjectLock() { + return objectLock; + } + + public void setObjectLock(boolean objectLock) { + this.objectLock = objectLock; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(String policy) { + this.policy = policy; + } + + public String getBucketURL() { + return bucketURL; + } + + public void setBucketURL(String bucketURL) { + this.bucketURL = bucketURL; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setState(Bucket.State state) { + this.state = state.toString(); + } + + public String getState() { + return state; + } + + public void setObjectStoragePool(String objectStoragePool) { + this.objectStoragePool = objectStoragePool; + } + + public String getObjectStoragePool() { + return objectStoragePool; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index e1f1e5ee241..d72d23b99c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -221,6 +221,10 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "comma-separated list of tags for the host") private String hostTags; + @SerializedName(ApiConstants.IS_TAG_A_RULE) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @SerializedName("hasenoughcapacity") @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise") private Boolean hasEnoughCapacity; @@ -732,4 +736,12 @@ public class HostResponse extends BaseResponseWithAnnotations { public void setEncryptionSupported(Boolean encryptionSupported) { this.encryptionSupported = encryptionSupported; } + + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean tagARule) { + isTagARule = tagARule; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java index 7e3adc4ba01..e2bf6ef5c60 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java @@ -128,7 +128,7 @@ public class IPAddressResponse extends BaseResponseWithAnnotations implements Co private String networkId; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing, Reserved and Free") private String state; @SerializedName(ApiConstants.PHYSICAL_NETWORK_ID) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java new file mode 100644 index 00000000000..e4030799aa7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import org.apache.cloudstack.storage.object.ObjectStore; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.BaseResponseWithAnnotations; +import org.apache.cloudstack.api.EntityReference; + +@EntityReference(value = ObjectStore.class) +public class ObjectStoreResponse extends BaseResponseWithAnnotations { + @SerializedName("id") + @Param(description = "the ID of the object store") + private String id; + + @SerializedName("name") + @Param(description = "the name of the object store") + private String name; + + @SerializedName("url") + @Param(description = "the url of the object store") + private String url; + + @SerializedName("providername") + @Param(description = "the provider name of the object store") + private String providerName; + + @SerializedName("storagetotal") + @Param(description = "the total size of the object store") + private Long storageTotal; + + @SerializedName("storageused") + @Param(description = "the object store currently used size") + private Long storageUsed; + + public ObjectStoreResponse() { + } + + @Override + public String getObjectId() { + return this.getId(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getProviderName() { + return providerName; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public Long getStorageTotal() { + return storageTotal; + } + + public void setStorageTotal(Long storageTotal) { + this.storageTotal = storageTotal; + } + + public Long getStorageUsed() { + return storageUsed; + } + + public void setStorageUsed(Long storageUsed) { + this.storageUsed = storageUsed; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java index 73008b01ccd..e477b111561 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java @@ -68,7 +68,7 @@ public class PortableIpResponse extends BaseResponse { private Date allocated; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing and Free") private String state; public void setRegionId(Integer regionId) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java new file mode 100644 index 00000000000..25a6b2eca75 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java @@ -0,0 +1,141 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import java.util.Date; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@EntityReference(value = {Heuristic.class}) +public class SecondaryStorageHeuristicsResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the heuristic.") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the heuristic.") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "Description of the heuristic.") + private String description; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "The zone which the heuristic is valid upon.") + private String zoneId; + + @SerializedName(ApiConstants.TYPE) + @Param(description = "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @SerializedName(ApiConstants.HEURISTIC_RULE) + @Param(description = "The heuristic rule, in JavaScript language, used to select a secondary storage to be directed.") + private String heuristicRule; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "When the heuristic was created.") + private Date created; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "When the heuristic was removed.") + private Date removed; + + + public SecondaryStorageHeuristicsResponse(String id, String name, String description, String zoneId, String type, String heuristicRule, Date created, Date removed) { + super("heuristics"); + this.id = id; + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + this.created = created; + this.removed = removed; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java index 89256c26473..183290ec9eb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -101,6 +101,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "the tags for the storage pool") private String tags; + @SerializedName(ApiConstants.IS_TAG_A_RULE) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @SerializedName(ApiConstants.STATE) @Param(description = "the state of the storage pool") private StoragePoolStatus state; @@ -304,6 +308,14 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { this.tags = tags; } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean tagARule) { + isTagARule = tagARule; + } + public StoragePoolStatus getState() { return state; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java index e866b19e1c1..7a26b178591 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java @@ -39,6 +39,10 @@ public class UnmanagedInstanceResponse extends BaseResponse { @Param(description = "the ID of the cluster to which virtual machine belongs") private String clusterId; + @SerializedName(ApiConstants.CLUSTER_NAME) + @Param(description = "the name of the cluster to which virtual machine belongs") + private String clusterName; + @SerializedName(ApiConstants.HOST_ID) @Param(description = "the ID of the host to which virtual machine belongs") private String hostId; @@ -104,6 +108,14 @@ public class UnmanagedInstanceResponse extends BaseResponse { this.clusterId = clusterId; } + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + public String getHostId() { return hostId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index 906529c13ab..763265e109d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -166,11 +166,11 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co private String serviceOfferingName; @SerializedName(ApiConstants.DISK_OFFERING_ID) - @Param(description = "the ID of the disk offering of the virtual machine", since = "4.4") + @Param(description = "the ID of the disk offering of the virtual machine. This parameter should not be used for retrieving disk offering details of DATA volumes. Use listVolumes API instead", since = "4.4") private String diskOfferingId; @SerializedName("diskofferingname") - @Param(description = "the name of the disk offering of the virtual machine", since = "4.4") + @Param(description = "the name of the disk offering of the virtual machine. This parameter should not be used for retrieving disk offering details of DATA volumes. Use listVolumes API instead", since = "4.4") private String diskOfferingName; @SerializedName(ApiConstants.BACKUP_OFFERING_ID) diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java similarity index 97% rename from plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java rename to api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java index ddd4a364a0a..3cf06f38242 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VmwareDatacenterResponse.java @@ -23,7 +23,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.serializer.Param; @EntityReference(value = VmwareDatacenter.class) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java index 3a0e84285aa..d3e4d941678 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java @@ -57,7 +57,7 @@ public class VpnUsersResponse extends BaseResponse implements ControlledEntityRe private String projectName; @SerializedName(ApiConstants.STATE) - @Param(description = "the state of the Vpn User") + @Param(description = "the state of the Vpn User, can be 'Add', 'Revoke' or 'Active'.") private String state; public void setId(String id) { diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 6f404452598..3299e7537a2 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -28,14 +28,17 @@ import org.apache.cloudstack.api.command.admin.resource.icon.ListResourceIconCmd import org.apache.cloudstack.api.command.admin.router.GetRouterHealthCheckResultsCmd; import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; import org.apache.cloudstack.api.command.admin.user.ListUsersCmd; import org.apache.cloudstack.api.command.user.account.ListAccountsCmd; import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; import org.apache.cloudstack.api.command.user.address.ListQuarantinedIpsCmd; import org.apache.cloudstack.api.command.user.affinitygroup.ListAffinityGroupsCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; import org.apache.cloudstack.api.command.user.iso.ListIsosCmd; import org.apache.cloudstack.api.command.user.job.ListAsyncJobsCmd; @@ -56,6 +59,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; @@ -68,6 +72,7 @@ import org.apache.cloudstack.api.response.InstanceGroupResponse; import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -75,6 +80,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; @@ -185,9 +191,15 @@ public interface QueryService { List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); + ListResponse listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd); + ListResponse listQuarantinedIps(ListQuarantinedIpsCmd cmd); ListResponse listSnapshots(ListSnapshotsCmd cmd); SnapshotResponse listSnapshot(CopySnapshotCmd cmd); + + ListResponse searchForObjectStores(ListObjectStoragePoolsCmd listObjectStoragePoolsCmd); + + ListResponse searchForBuckets(ListBucketsCmd listBucketsCmd); } diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java new file mode 100644 index 00000000000..2a0b8d6ea24 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface Heuristic extends InternalIdentity, Identity { + + String getName(); + + String getDescription(); + + Long getZoneId(); + + String getType(); + + String getHeuristicRule(); + + Date getCreated(); + + Date getRemoved(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java new file mode 100644 index 00000000000..f23e4b0b633 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +/** + * The type of the heuristic used in the allocation process of secondary storage resources. + * Valid options are: {@link #ISO}, {@link #SNAPSHOT}, {@link #TEMPLATE} and {@link #VOLUME} + */ +public enum HeuristicType { + ISO, SNAPSHOT, TEMPLATE, VOLUME +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java b/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java new file mode 100644 index 00000000000..c821dbac589 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface Bucket extends ControlledEntity, Identity, InternalIdentity { + + long getObjectStoreId(); + + Date getCreated(); + + State getState(); + + void setName(String name); + + Long getSize(); + + Integer getQuota(); + + boolean isVersioning(); + + boolean isEncryption(); + + boolean isObjectLock(); + + String getPolicy(); + + String getBucketURL(); + + String getAccessKey(); + + String getSecretKey(); + + public enum State { + Allocated, Created, Destroyed; + @Override + public String toString() { + return this.name(); + } + + public boolean equals(String status) { + return this.toString().equalsIgnoreCase(status); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java new file mode 100644 index 00000000000..7e1361d1e71 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.cloud.exception.ResourceAllocationException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; + +public interface BucketApiService { + + + /** + * Creates the database object for a Bucket based on the given criteria + * + * @param cmd + * the API command wrapping the criteria (account/domainId [admin only], zone, diskOffering, snapshot, + * name) + * @return the Bucket object + */ + Bucket allocBucket(CreateBucketCmd cmd) throws ResourceAllocationException; + + /** + * Creates the Bucket based on the given criteria + * + * @param cmd + * the API command wrapping the criteria (account/domainId [admin only], zone, diskOffering, snapshot, + * name) + * @return the Bucket object + */ + Bucket createBucket(CreateBucketCmd cmd); + + boolean deleteBucket(long bucketId, Account caller); + + boolean updateBucket(UpdateBucketCmd cmd, Account caller); + + void getBucketUsage(); +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java b/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java new file mode 100644 index 00000000000..47741fc67f4 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface ObjectStore extends Identity, InternalIdentity { + + /** + * @return name of the object store. + */ + String getName(); + + /** + * @return object store provider name + */ + String getProviderName(); + + /** + * + * @return uri + */ + String getUrl(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java index 48cff3076fd..5e0f03ff583 100644 --- a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java +++ b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java @@ -45,6 +45,7 @@ public class UsageTypes { public static final int VOLUME_SECONDARY = 26; public static final int VM_SNAPSHOT_ON_PRIMARY = 27; public static final int BACKUP = 28; + public static final int BUCKET = 29; public static List listUsageTypes() { List responseList = new ArrayList(); @@ -70,6 +71,7 @@ public class UsageTypes { responseList.add(new UsageTypeResponse(VOLUME_SECONDARY, "Volume on secondary storage usage")); responseList.add(new UsageTypeResponse(VM_SNAPSHOT_ON_PRIMARY, "VM Snapshot on primary storage usage")); responseList.add(new UsageTypeResponse(BACKUP, "Backup storage usage")); + responseList.add(new UsageTypeResponse(BUCKET, "Bucket storage usage")); return responseList; } } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index 95675f2bf34..23e0e371714 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -33,6 +33,8 @@ public class UnmanagedInstanceTO { private PowerState powerState; + private PowerState cloneSourcePowerState; + private Integer cpuCores; private Integer cpuCoresPerSocket; @@ -45,10 +47,16 @@ public class UnmanagedInstanceTO { private String operatingSystem; + private String clusterName; + + private String hostName; + private List disks; private List nics; + private String vncPassword; + public String getName() { return name; } @@ -73,6 +81,14 @@ public class UnmanagedInstanceTO { this.powerState = powerState; } + public PowerState getCloneSourcePowerState() { + return cloneSourcePowerState; + } + + public void setCloneSourcePowerState(PowerState cloneSourcePowerState) { + this.cloneSourcePowerState = cloneSourcePowerState; + } + public Integer getCpuCores() { return cpuCores; } @@ -121,6 +137,22 @@ public class UnmanagedInstanceTO { this.operatingSystem = operatingSystem; } + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + public List getDisks() { return disks; } @@ -137,6 +169,14 @@ public class UnmanagedInstanceTO { this.nics = nics; } + public String getVncPassword() { + return vncPassword; + } + + public void setVncPassword(String vncPassword) { + this.vncPassword = vncPassword; + } + public static class Disk { private String diskId; @@ -162,6 +202,8 @@ public class UnmanagedInstanceTO { private String datastorePath; + private int datastorePort; + private String datastoreType; public String getDiskId() { @@ -267,6 +309,14 @@ public class UnmanagedInstanceTO { public void setDatastoreType(String datastoreType) { this.datastoreType = datastoreType; } + + public void setDatastorePort(int datastorePort) { + this.datastorePort = datastorePort; + } + + public int getDatastorePort() { + return datastorePort; + } } public static class Nic { diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java index 2876a0127be..53aece94964 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java @@ -17,13 +17,20 @@ package org.apache.cloudstack.vm; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.component.PluggableService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.KVM; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.VMware; public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService, PluggableService, Configurable { ConfigKey UnmanageVMPreserveNic = new ConfigKey<>("Advanced", Boolean.class, "unmanage.vm.preserve.nics", "false", "If set to true, do not remove VM nics (and its MAC addresses) when unmanaging a VM, leaving them allocated but not reserved. " + "If set to false, nics are removed and MAC addresses can be reassigned", true, ConfigKey.Scope.Zone); + + static boolean isSupported(Hypervisor.HypervisorType hypervisorType) { + return hypervisorType == VMware || hypervisorType == KVM; + } } diff --git a/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java b/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java index cce28474541..04ef248fb8a 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java +++ b/api/src/main/java/org/apache/cloudstack/vm/VmImportService.java @@ -18,12 +18,28 @@ package org.apache.cloudstack.vm; import org.apache.cloudstack.api.command.admin.vm.ImportUnmanagedInstanceCmd; +import org.apache.cloudstack.api.command.admin.vm.ImportVmCmd; import org.apache.cloudstack.api.command.admin.vm.ListUnmanagedInstancesCmd; +import org.apache.cloudstack.api.command.admin.vm.ListVmsForImportCmd; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.UserVmResponse; public interface VmImportService { + + enum ImportSource { + UNMANAGED, VMWARE, EXTERNAL, SHARED, LOCAL; + + @Override + public String toString() { + return name().toLowerCase(); + } + } + ListResponse listUnmanagedInstances(ListUnmanagedInstancesCmd cmd); UserVmResponse importUnmanagedInstance(ImportUnmanagedInstanceCmd cmd); + + UserVmResponse importVm(ImportVmCmd cmd); + + ListResponse listVmsForImport(ListVmsForImportCmd cmd); } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..f64df167e25 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.exception.DiscoveryException; +import com.cloud.storage.StorageService; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.anyObject; + +@RunWith(MockitoJUnitRunner.class) +public class AddObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(AddObjectStoragePoolCmdTest.class.getName()); + + @Mock + StorageService storageService; + + @Mock + ObjectStore objectStore; + + @Mock + ResponseGenerator responseGenerator; + + @Spy + AddObjectStoragePoolCmd addObjectStoragePoolCmdSpy; + + String name = "testObjStore"; + + String url = "testURL"; + + String provider = "Simulator"; + + Map details; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + details = new HashMap<>(); + addObjectStoragePoolCmdSpy = Mockito.spy(new AddObjectStoragePoolCmd()); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "name", name); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "url", url); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "providerName", provider); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "details", details); + addObjectStoragePoolCmdSpy._storageService = storageService; + addObjectStoragePoolCmdSpy._responseGenerator = responseGenerator; + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testAddObjectStore() throws DiscoveryException { + Mockito.doReturn(objectStore).when(storageService).discoverObjectStore(Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), anyObject()); + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(anyObject()); + addObjectStoragePoolCmdSpy.execute(); + + Mockito.verify(storageService, Mockito.times(1)) + .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..35be56d0c75 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.storage.StorageService; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +public class DeleteObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(DeleteObjectStoragePoolCmdTest.class.getName()); + @Mock + private StorageService storageService; + + @Spy + DeleteObjectStoragePoolCmd deleteObjectStoragePoolCmd; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + deleteObjectStoragePoolCmd = Mockito.spy(new DeleteObjectStoragePoolCmd()); + deleteObjectStoragePoolCmd._storageService = storageService; + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testDeleteObjectStore() { + Mockito.doReturn(true).when(storageService).deleteObjectStore(deleteObjectStoragePoolCmd); + deleteObjectStoragePoolCmd.execute(); + Mockito.verify(storageService, Mockito.times(1)) + .deleteObjectStore(deleteObjectStoragePoolCmd); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..ef66c2a1a64 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.storage.StorageService; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.mockito.ArgumentMatchers.anyObject; + +public class UpdateObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(UpdateObjectStoragePoolCmdTest.class.getName()); + + @Mock + private StorageService storageService; + + @Spy + UpdateObjectStoragePoolCmd updateObjectStoragePoolCmd; + + @Mock + ObjectStore objectStore; + + @Mock + ResponseGenerator responseGenerator; + + private String name = "testObjStore"; + + private String url = "testURL"; + + private String provider = "Simulator"; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + updateObjectStoragePoolCmd = Mockito.spy(new UpdateObjectStoragePoolCmd()); + updateObjectStoragePoolCmd._storageService = storageService; + updateObjectStoragePoolCmd._responseGenerator = responseGenerator; + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "name", name); + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "url", url); + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "id", 1L); + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testUpdateObjectStore() { + Mockito.doReturn(objectStore).when(storageService).updateObjectStore(1L, updateObjectStoragePoolCmd); + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(anyObject()); + updateObjectStoragePoolCmd.execute(); + Mockito.verify(storageService, Mockito.times(1)) + .updateObjectStore(1L, updateObjectStoragePoolCmd); + } + +} diff --git a/client/pom.xml b/client/pom.xml index de9e910b978..f0ba65e0a8c 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -71,6 +71,56 @@ mysql mysql-connector-java + + org.apache.cloudstack + cloud-agent + ${project.version} + + + org.apache.cloudstack + cloud-api + ${project.version} + + + org.apache.cloudstack + cloud-core + ${project.version} + + + org.apache.cloudstack + cloud-framework-cluster + ${project.version} + + + org.apache.cloudstack + cloud-framework-config + ${project.version} + + + org.apache.cloudstack + cloud-framework-db + ${project.version} + + + org.apache.cloudstack + cloud-framework-events + ${project.version} + + + org.apache.cloudstack + cloud-framework-jobs + ${project.version} + + + org.apache.cloudstack + cloud-framework-managed-context + ${project.version} + + + org.apache.cloudstack + cloud-framework-security + ${project.version} + org.apache.cloudstack cloud-framework-spring-module @@ -81,6 +131,11 @@ cloud-framework-spring-lifecycle ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-volume-adaptive + ${project.version} + org.apache.cloudstack cloud-plugin-storage-volume-solidfire @@ -111,6 +166,16 @@ cloud-plugin-storage-volume-storpool ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-volume-primera + ${project.version} + + + org.apache.cloudstack + cloud-plugin-storage-volume-flasharray + ${project.version} + org.apache.cloudstack cloud-server @@ -577,6 +642,31 @@ cloud-plugin-shutdown ${project.version} + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-plugin-storage-object-minio + ${project.version} + + + org.apache.cloudstack + cloud-plugin-storage-object-simulator + ${project.version} + + + org.apache.cloudstack + cloud-usage + ${project.version} + + + org.apache.cloudstack + cloud-utils + ${project.version} + @@ -886,6 +976,7 @@ mysql:mysql-connector-java org.apache.cloudstack:cloud-plugin-storage-volume-storpool org.apache.cloudstack:cloud-plugin-storage-volume-linstor + org.apache.cloudstack:cloud-usage com.linbit.linstor.api:java-linstor diff --git a/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java new file mode 100644 index 00000000000..dd136d8642f --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api; + +@LogLevel(LogLevel.Log4jLevel.Trace) +public class CheckVolumeAnswer extends Answer { + + private long size; + + CheckVolumeAnswer() { + } + + public CheckVolumeAnswer(CheckVolumeCommand cmd, String details, long size) { + super(cmd, true, details); + this.size = size; + } + + public long getSize() { + return size; + } + + public String getString() { + return "CheckVolumeAnswer [size=" + size + "]"; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/CheckVolumeCommand.java b/core/src/main/java/com/cloud/agent/api/CheckVolumeCommand.java new file mode 100644 index 00000000000..b4036bebf3a --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CheckVolumeCommand.java @@ -0,0 +1,59 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.agent.api.to.StorageFilerTO; + +@LogLevel(LogLevel.Log4jLevel.Trace) +public class CheckVolumeCommand extends Command { + + String srcFile; + + StorageFilerTO storageFilerTO; + + + public String getSrcFile() { + return srcFile; + } + + public void setSrcFile(String srcFile) { + this.srcFile = srcFile; + } + + public CheckVolumeCommand() { + } + + @Override + public boolean executeInSequence() { + return false; + } + + public String getString() { + return "CheckVolumeCommand [srcFile=" + srcFile + "]"; + } + + public StorageFilerTO getStorageFilerTO() { + return storageFilerTO; + } + + public void setStorageFilerTO(StorageFilerTO storageFilerTO) { + this.storageFilerTO = storageFilerTO; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java new file mode 100644 index 00000000000..829888570a6 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import org.apache.cloudstack.vm.UnmanagedInstanceTO; + +public class ConvertInstanceAnswer extends Answer { + + public ConvertInstanceAnswer() { + super(); + } + private UnmanagedInstanceTO convertedInstance; + + public ConvertInstanceAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public ConvertInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { + super(command, true, ""); + this.convertedInstance = convertedInstance; + } + + public UnmanagedInstanceTO getConvertedInstance() { + return convertedInstance; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java new file mode 100644 index 00000000000..63234b04480 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.RemoteInstanceTO; +import com.cloud.hypervisor.Hypervisor; + +import java.util.List; + +public class ConvertInstanceCommand extends Command { + + private RemoteInstanceTO sourceInstance; + private Hypervisor.HypervisorType destinationHypervisorType; + private List destinationStoragePools; + private DataStoreTO conversionTemporaryLocation; + + public ConvertInstanceCommand() { + } + + public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, + List destinationStoragePools, DataStoreTO conversionTemporaryLocation) { + this.sourceInstance = sourceInstance; + this.destinationHypervisorType = destinationHypervisorType; + this.destinationStoragePools = destinationStoragePools; + this.conversionTemporaryLocation = conversionTemporaryLocation; + } + + public RemoteInstanceTO getSourceInstance() { + return sourceInstance; + } + + public Hypervisor.HypervisorType getDestinationHypervisorType() { + return destinationHypervisorType; + } + + public List getDestinationStoragePools() { + return destinationStoragePools; + } + + public DataStoreTO getConversionTemporaryLocation() { + return conversionTemporaryLocation; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java new file mode 100644 index 00000000000..f6d7cab4596 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api; + +@LogLevel(LogLevel.Log4jLevel.Trace) +public class CopyRemoteVolumeAnswer extends Answer { + + private String remoteIp; + private String filename; + + private long size; + + CopyRemoteVolumeAnswer() { + } + + public CopyRemoteVolumeAnswer(CopyRemoteVolumeCommand cmd, String details, String filename, long size) { + super(cmd, true, details); + this.remoteIp = cmd.getRemoteIp(); + this.filename = filename; + this.size = size; + } + + public String getRemoteIp() { + return remoteIp; + } + + public void setRemoteIp(String remoteIp) { + this.remoteIp = remoteIp; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public String getFilename() { + return filename; + } + + public long getSize() { + return size; + } + + public String getString() { + return "CopyRemoteVolumeAnswer [remoteIp=" + remoteIp + "]"; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java new file mode 100644 index 00000000000..82bc4d7cb45 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java @@ -0,0 +1,101 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.agent.api.to.StorageFilerTO; + +@LogLevel(LogLevel.Log4jLevel.Trace) +public class CopyRemoteVolumeCommand extends Command { + + String remoteIp; + String username; + String password; + String srcFile; + + String tmpPath; + + StorageFilerTO storageFilerTO; + + public CopyRemoteVolumeCommand(String remoteIp, String username, String password) { + this.remoteIp = remoteIp; + this.username = username; + this.password = password; + } + + public String getRemoteIp() { + return remoteIp; + } + + public void setRemoteIp(String remoteIp) { + this.remoteIp = remoteIp; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getSrcFile() { + return srcFile; + } + + public void setSrcFile(String srcFile) { + this.srcFile = srcFile; + } + + public CopyRemoteVolumeCommand() { + } + + @Override + public boolean executeInSequence() { + return false; + } + + public String getString() { + return "CopyRemoteVolumeCommand [remoteIp=" + remoteIp + "]"; + } + + public void setTempPath(String tmpPath) { + this.tmpPath = tmpPath; + } + + public String getTmpPath() { + return tmpPath; + } + + public StorageFilerTO getStorageFilerTO() { + return storageFilerTO; + } + + public void setStorageFilerTO(StorageFilerTO storageFilerTO) { + this.storageFilerTO = storageFilerTO; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/GetRemoteVmsAnswer.java b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsAnswer.java new file mode 100644 index 00000000000..8cd072f1da1 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsAnswer.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api; + +import org.apache.cloudstack.vm.UnmanagedInstanceTO; + +import java.util.HashMap; +import java.util.List; + +@LogLevel(LogLevel.Log4jLevel.Trace) +public class GetRemoteVmsAnswer extends Answer { + + private String remoteIp; + private HashMap unmanagedInstances; + + List vmNames; + + GetRemoteVmsAnswer() { + } + + public GetRemoteVmsAnswer(GetRemoteVmsCommand cmd, String details, HashMap unmanagedInstances) { + super(cmd, true, details); + this.remoteIp = cmd.getRemoteIp(); + this.unmanagedInstances = unmanagedInstances; + } + + public GetRemoteVmsAnswer(GetRemoteVmsCommand cmd, String details, List vmNames) { + super(cmd, true, details); + this.remoteIp = cmd.getRemoteIp(); + this.vmNames = vmNames; + } + + public String getRemoteIp() { + return remoteIp; + } + + public void setRemoteIp(String remoteIp) { + this.remoteIp = remoteIp; + } + + public HashMap getUnmanagedInstances() { + return unmanagedInstances; + } + + public void setUnmanagedInstances(HashMap unmanagedInstances) { + this.unmanagedInstances = unmanagedInstances; + } + + public List getVmNames() { + return vmNames; + } + + public void setVmNames(List vmNames) { + this.vmNames = vmNames; + } + + public String getString() { + return "GetRemoteVmsAnswer [remoteIp=" + remoteIp + "]"; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/GetRemoteVmsCommand.java b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsCommand.java new file mode 100644 index 00000000000..5c71d12dbd0 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/GetRemoteVmsCommand.java @@ -0,0 +1,70 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +@LogLevel(LogLevel.Log4jLevel.Trace) +public class GetRemoteVmsCommand extends Command { + + String remoteIp; + String username; + String password; + + public GetRemoteVmsCommand(String remoteIp, String username, String password) { + this.remoteIp = remoteIp; + this.username = username; + this.password = password; + } + + public String getRemoteIp() { + return remoteIp; + } + + public void setRemoteIp(String remoteIp) { + this.remoteIp = remoteIp; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public GetRemoteVmsCommand() { + } + + @Override + public boolean executeInSequence() { + return false; + } + + public String getString() { + return "GetRemoteVmsCommand [remoteIp=" + remoteIp + "]"; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java b/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java index 3c6118d426e..771d472be2a 100644 --- a/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/GetUnmanagedInstancesAnswer.java @@ -30,6 +30,10 @@ public class GetUnmanagedInstancesAnswer extends Answer { GetUnmanagedInstancesAnswer() { } + public GetUnmanagedInstancesAnswer(GetUnmanagedInstancesCommand cmd, String details) { + super(cmd, false, details); + } + public GetUnmanagedInstancesAnswer(GetUnmanagedInstancesCommand cmd, String details, HashMap unmanagedInstances) { super(cmd, true, details); this.instanceName = cmd.getInstanceName(); diff --git a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java index 27251f4bb78..3acdb9c351b 100644 --- a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java +++ b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java @@ -40,6 +40,9 @@ public class MigrateCommand extends Command { private boolean executeInSequence = false; private List migrateDiskInfoList = new ArrayList<>(); private Map dpdkInterfaceMapping = new HashMap<>(); + + private int newVmCpuShares; + Map vlanToPersistenceMap = new HashMap<>(); public Map getDpdkInterfaceMapping() { @@ -138,6 +141,14 @@ public class MigrateCommand extends Command { this.migrateDiskInfoList = migrateDiskInfoList; } + public int getNewVmCpuShares() { + return newVmCpuShares; + } + + public void setNewVmCpuShares(int newVmCpuShares) { + this.newVmCpuShares = newVmCpuShares; + } + public static class MigrateDiskInfo { public enum DiskType { FILE, BLOCK; diff --git a/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java b/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java index d0a544ba081..190e844ddc5 100644 --- a/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/PrepareForMigrationAnswer.java @@ -28,6 +28,8 @@ public class PrepareForMigrationAnswer extends Answer { private Map dpdkInterfaceMapping = new HashMap<>(); + private Integer newVmCpuShares = null; + protected PrepareForMigrationAnswer() { } @@ -50,4 +52,12 @@ public class PrepareForMigrationAnswer extends Answer { public Map getDpdkInterfaceMapping() { return this.dpdkInterfaceMapping; } + + public Integer getNewVmCpuShares() { + return newVmCpuShares; + } + + public void setNewVmCpuShares(Integer newVmCpuShares) { + this.newVmCpuShares = newVmCpuShares; + } } diff --git a/debian/control b/debian/control index 7c1ff8e0b1b..9fec540975e 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,7 @@ Description: CloudStack server library Package: cloudstack-agent Architecture: all -Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, aria2, ufw, apparmor +Depends: ${python:Depends}, ${python3:Depends}, openjdk-11-jre-headless | java11-runtime-headless | java11-runtime | openjdk-11-jre-headless | zulu-11, cloudstack-common (= ${source:Version}), lsb-base (>= 9), openssh-client, qemu-kvm (>= 2.5) | qemu-system-x86 (>= 5.2), libvirt-bin (>= 1.3) | libvirt-daemon-system (>= 3.0), iproute2, ebtables, vlan, ipset, python3-libvirt, ethtool, iptables, cryptsetup, rng-tools, lsb-release, ufw, apparmor Recommends: init-system-helpers Conflicts: cloud-agent, cloud-agent-libs, cloud-agent-deps, cloud-agent-scripts Description: CloudStack agent diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 15f5b231be2..01123401fac 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -168,6 +168,9 @@ public interface VolumeOrchestrationService { DiskProfile importVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, Account owner, Long deviceId, Long poolId, String path, String chainInfo); + DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, + Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile); + /** * Unmanage VM volumes */ diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java new file mode 100644 index 00000000000..366c7c8fb41 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +import org.apache.cloudstack.storage.object.Bucket; + +public interface BucketInfo extends DataObject, Bucket { + + void addPayload(Object data); + + Object getPayload(); + + Bucket getBucket(); +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 3ee5803a91a..de26a09c6e5 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -35,6 +35,8 @@ public interface DataStoreManager { List getImageStoresByScopeExcludingReadOnly(ZoneScope scope); + List getImageStoresByZoneIds(Long ... zoneIds); + DataStore getRandomImageStore(long zoneId); DataStore getRandomUsableImageStore(long zoneId); @@ -55,5 +57,7 @@ public interface DataStoreManager { boolean isRegionStore(DataStore store); + DataStore getImageStoreByUuid(String uuid); + Long getStoreZoneId(long storeId, DataStoreRole role); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java index 3e5761ef37f..41c1d940745 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java @@ -31,7 +31,7 @@ public interface DataStoreProvider { String DEFAULT_PRIMARY = "DefaultPrimary"; enum DataStoreProviderType { - PRIMARY, IMAGE, ImageCache + PRIMARY, IMAGE, ImageCache, OBJECT } DataStoreLifeCycle getDataStoreLifeCycle(); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java index e476d8f3b35..379911a54fb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java @@ -33,4 +33,6 @@ public interface DataStoreProviderManager extends Manager, DataStoreProviderApiS DataStoreProvider getDefaultCacheDataStoreProvider(); List getProviders(); + + DataStoreProvider getDefaultObjectStoreProvider(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java new file mode 100644 index 00000000000..691da7ecc8d --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.subsystem.api.storage; + +public interface ObjectStorageService { + +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java new file mode 100644 index 00000000000..71b3762b441 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +public interface ObjectStoreProvider extends DataStoreProvider { + +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java index 1dbff59a891..1b18264df15 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java @@ -43,6 +43,8 @@ public class PrimaryDataStoreParameters { private boolean managed; private Long capacityIops; + private Boolean isTagARule; + /** * @return the userInfo */ @@ -277,4 +279,12 @@ public class PrimaryDataStoreParameters { public void setUsedBytes(long usedBytes) { this.usedBytes = usedBytes; } + + public Boolean isTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java new file mode 100644 index 00000000000..9ee94b083cf --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; + +import java.util.List; +import java.util.Map; + +public interface ObjectStoreEntity extends DataStore, ObjectStore { + Bucket createBucket(Bucket bucket, boolean objectLock); + + List listBuckets(); + + boolean createUser(long accountId); + + boolean deleteBucket(String name); + + boolean setBucketEncryption(String name); + + boolean deleteBucketEncryption(String name); + + boolean setBucketVersioning(String name); + + boolean deleteBucketVersioning(String name); + + void setBucketPolicy(String name, String policy); + + void setQuota(String name, int quota); + + Map getAllBucketsUsage(); +} diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 28e7c89419a..f5cf443211c 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -96,6 +96,14 @@ public interface StorageManager extends StorageService { true, ConfigKey.Scope.Global, null); + ConfigKey ConvertVmwareInstanceToKvmTimeout = new ConfigKey<>(Integer.class, + "convert.vmware.instance.to.kvm.timeout", + "Storage", + "8", + "Timeout (in hours) for the instance conversion process from VMware through the virt-v2v binary on a KVM host", + true, + ConfigKey.Scope.Global, + null); ConfigKey KvmAutoConvergence = new ConfigKey<>(Boolean.class, "kvm.auto.convergence", "Storage", @@ -192,6 +200,9 @@ public interface StorageManager extends StorageService { ConfigKey.Scope.Global, null); + ConfigKey HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000", + "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true); + /** * should we execute in sequence not involving any storages? * @return tru if commands should execute in sequence diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 2c129dfd6a5..3b3537d5488 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -34,6 +34,7 @@ import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachineProfile; @@ -116,7 +117,7 @@ public interface TemplateManager { Long getTemplateSize(long templateId, long zoneId); - DataStore getImageStore(String storeUuid, Long zoneId); + DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume); String getChecksum(DataStore store, String templatePath, String algorithm); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index b2eaaf7ea6e..ccc005c3932 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -48,6 +48,7 @@ import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; import com.cloud.event.ActionEventUtils; +import com.google.gson.Gson; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -2790,23 +2791,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } boolean migrated = false; - Map dpdkInterfaceMapping = null; + Map dpdkInterfaceMapping = new HashMap<>(); try { - final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); - final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType())); - if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { - mc.setVlanToPersistenceMap(vlanToPersistenceMap); - } - - boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); - mc.setAutoConvergence(kvmAutoConvergence); - mc.setHostGuid(dest.getHost().getGuid()); - - dpdkInterfaceMapping = ((PrepareForMigrationAnswer) pfma).getDpdkInterfaceMapping(); - if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { - mc.setDpdkInterfaceMapping(dpdkInterfaceMapping); - } + final MigrateCommand mc = buildMigrateCommand(vm, to, dest, pfma, dpdkInterfaceMapping); try { final Answer ma = _agentMgr.send(vm.getLastHostId(), mc); @@ -2878,6 +2865,43 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } + /** + * Create and set parameters for the {@link MigrateCommand} used in the migration and scaling of VMs. + */ + protected MigrateCommand buildMigrateCommand(VMInstanceVO vmInstance, VirtualMachineTO virtualMachineTO, DeployDestination destination, Answer answer, + Map dpdkInterfaceMapping) { + final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vmInstance.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); + final MigrateCommand migrateCommand = new MigrateCommand(vmInstance.getInstanceName(), destination.getHost().getPrivateIpAddress(), isWindows, virtualMachineTO, + getExecuteInSequence(vmInstance.getHypervisorType())); + + Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vmInstance.getId()); + if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { + s_logger.debug(String.format("Setting VLAN persistence to [%s] as part of migrate command for VM [%s].", new Gson().toJson(vlanToPersistenceMap), virtualMachineTO)); + migrateCommand.setVlanToPersistenceMap(vlanToPersistenceMap); + } + + migrateCommand.setAutoConvergence(StorageManager.KvmAutoConvergence.value()); + migrateCommand.setHostGuid(destination.getHost().getGuid()); + + PrepareForMigrationAnswer prepareForMigrationAnswer = (PrepareForMigrationAnswer) answer; + + Map answerDpdkInterfaceMapping = prepareForMigrationAnswer.getDpdkInterfaceMapping(); + if (MapUtils.isNotEmpty(answerDpdkInterfaceMapping) && dpdkInterfaceMapping != null) { + s_logger.debug(String.format("Setting DPDK interface mapping to [%s] as part of migrate command for VM [%s].", new Gson().toJson(vlanToPersistenceMap), + virtualMachineTO)); + dpdkInterfaceMapping.putAll(answerDpdkInterfaceMapping); + migrateCommand.setDpdkInterfaceMapping(dpdkInterfaceMapping); + } + + Integer newVmCpuShares = prepareForMigrationAnswer.getNewVmCpuShares(); + if (newVmCpuShares != null) { + s_logger.debug(String.format("Setting CPU shares to [%d] as part of migrate command for VM [%s].", newVmCpuShares, virtualMachineTO)); + migrateCommand.setNewVmCpuShares(newVmCpuShares); + } + + return migrateCommand; + } + private void updateVmPod(VMInstanceVO vm, long dstHostId) { // update the VMs pod HostVO host = _hostDao.findById(dstHostId); @@ -2957,6 +2981,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac *
    *
  • If the current storage pool of the volume is not a managed storage, we do not need to validate anything here. *
  • If the current storage pool is a managed storage and the target storage pool ID is different from the current one, we throw an exception. + *
  • If the current storage pool is a managed storage and explicitly declared its capable of migration to alternate storage pools *
*/ protected void executeManagedStorageChecksWhenTargetStoragePoolProvided(StoragePoolVO currentPool, VolumeVO volume, StoragePoolVO targetPool) { @@ -2966,6 +2991,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (currentPool.getId() == targetPool.getId()) { return; } + + Map details = _storagePoolDao.getDetails(currentPool.getId()); + if (details != null && Boolean.parseBoolean(details.get(Storage.Capability.ALLOW_MIGRATE_OTHER_POOLS.toString()))) { + return; + } throw new CloudRuntimeException(String.format("Currently, a volume on managed storage can only be 'migrated' to itself " + "[volumeId=%s, currentStoragePoolId=%s, targetStoragePoolId=%s].", volume.getUuid(), currentPool.getUuid(), targetPool.getUuid())); } @@ -4395,16 +4425,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac boolean migrated = false; try { - Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); - final boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - final MigrateCommand mc = new MigrateCommand(vm.getInstanceName(), dest.getHost().getPrivateIpAddress(), isWindows, to, getExecuteInSequence(vm.getHypervisorType())); - if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { - mc.setVlanToPersistenceMap(vlanToPersistenceMap); - } - - boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); - mc.setAutoConvergence(kvmAutoConvergence); - mc.setHostGuid(dest.getHost().getGuid()); + final MigrateCommand mc = buildMigrateCommand(vm, to, dest, pfma, null); try { final Answer ma = _agentMgr.send(vm.getLastHostId(), mc); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 960dddbe847..30fd35e5f37 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -4588,18 +4587,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public NicVO doInTransaction(TransactionStatus status) { NicVO existingNic = _nicDao.findByNetworkIdAndMacAddress(network.getId(), macAddress); + String macAddressToPersist = macAddress; if (existingNic != null) { - if (!forced) { - throw new CloudRuntimeException("NIC with MAC address = " + macAddress + " exists on network with ID = " + network.getId() + - " and forced flag is disabled"); - } - s_logger.debug("Removing existing NIC with MAC address = " + macAddress + " on network with ID = " + network.getId()); - existingNic.setState(Nic.State.Deallocating); - existingNic.setRemoved(new Date()); - _nicDao.update(existingNic.getId(), existingNic); + macAddressToPersist = generateNewMacAddressIfForced(network, macAddress, forced); } NicVO vo = new NicVO(network.getGuruName(), vm.getId(), network.getId(), vm.getType()); - vo.setMacAddress(macAddress); + vo.setMacAddress(macAddressToPersist); vo.setAddressFormat(Networks.AddressFormat.Ip4); if (NetUtils.isValidIp4(finalGuestIp) && StringUtils.isNotEmpty(network.getGateway())) { vo.setIPv4Address(finalGuestIp); @@ -4643,6 +4636,23 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return new Pair(vmNic, Integer.valueOf(deviceId)); } + private String generateNewMacAddressIfForced(Network network, String macAddress, boolean forced) { + if (!forced) { + throw new CloudRuntimeException("NIC with MAC address = " + macAddress + " exists on network with ID = " + network.getId() + + " and forced flag is disabled"); + } + try { + s_logger.debug(String.format("Generating a new mac address on network %s as the mac address %s already exists", network.getName(), macAddress)); + String newMacAddress = _networkModel.getNextAvailableMacAddressInNetwork(network.getId()); + s_logger.debug(String.format("Successfully generated the mac address %s, using it instead of the conflicting address %s", newMacAddress, macAddress)); + return newMacAddress; + } catch (InsufficientAddressCapacityException e) { + String msg = String.format("Could not generate a new mac address on network %s", network.getName()); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + } + @Override public void unmanageNics(VirtualMachineProfile vm) { if (s_logger.isDebugEnabled()) { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 6f945479bd4..b8f3e5a10e5 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -2224,6 +2224,51 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati return toDiskProfile(vol, offering); } + @Override + public DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, + Long deviceId, Long poolId, String path, String chainInfo, DiskProfile diskProfile) { + + VolumeVO vol = _volsDao.findById(diskProfile.getVolumeId()); + if (vm != null) { + vol.setInstanceId(vm.getId()); + } + + if (deviceId != null) { + vol.setDeviceId(deviceId); + } else if (type.equals(Type.ROOT)) { + vol.setDeviceId(0l); + } else { + vol.setDeviceId(1l); + } + + if (template != null) { + if (ImageFormat.ISO.equals(template.getFormat())) { + vol.setIsoId(template.getId()); + } else if (Storage.TemplateType.DATADISK.equals(template.getTemplateType())) { + vol.setTemplateId(template.getId()); + } + if (type == Type.ROOT) { + vol.setTemplateId(template.getId()); + } + } + + // display flag matters only for the User vms + if (VirtualMachine.Type.User.equals(vm.getType())) { + UserVmVO userVm = _userVmDao.findById(vm.getId()); + vol.setDisplayVolume(userVm.isDisplayVm()); + } + + vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType())); + vol.setPoolId(poolId); + vol.setPath(path); + vol.setChainInfo(chainInfo); + vol.setSize(diskProfile.getSize()); + vol.setState(Volume.State.Ready); + vol.setAttached(new Date()); + _volsDao.update(vol.getId(), vol); + return toDiskProfile(vol, offering); + } + @Override public void unmanageVolumes(long vmId) { if (s_logger.isDebugEnabled()) { diff --git a/engine/pom.xml b/engine/pom.xml index b2f41834403..6ecfb4bebf4 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -55,6 +55,7 @@ storage/configdrive storage/datamotion storage/image + storage/object storage/snapshot storage/volume userdata/cloud-init diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java b/engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java similarity index 99% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java rename to engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java index 86597b2e5c9..6390d923ed8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/VmwareDatacenterVO.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware; +package com.cloud.dc; import java.util.UUID; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java similarity index 96% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java rename to engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java index 2754e91d26c..1ed1f8d5a0f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VmwareDatacenterDao.java @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware.dao; +package com.cloud.dc.dao; import java.util.List; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.utils.db.GenericDao; public interface VmwareDatacenterDao extends GenericDao { diff --git a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java index 6b8b754849c..cd4ac29738d 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java @@ -24,6 +24,7 @@ import javax.persistence.Id; import javax.persistence.Table; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.BooleanUtils; @Entity @Table(name = "host_tags") @@ -39,12 +40,22 @@ public class HostTagVO implements InternalIdentity { @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + protected HostTagVO() { } public HostTagVO(long hostId, String tag) { this.hostId = hostId; this.tag = tag; + this.isTagARule = false; + } + + public HostTagVO(long hostId, String tag, Boolean isTagARule) { + this.hostId = hostId; + this.tag = tag; + this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false); } public long getHostId() { @@ -59,6 +70,11 @@ public class HostTagVO implements InternalIdentity { this.tag = tag; } + public boolean getIsTagARule() { + return isTagARule; + } + + @Override public long getId() { return id; diff --git a/engine/schema/src/main/java/com/cloud/host/HostVO.java b/engine/schema/src/main/java/com/cloud/host/HostVO.java index d6e7ea1fd87..697401ad069 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostVO.java @@ -39,6 +39,7 @@ import javax.persistence.TemporalType; import javax.persistence.Transient; import com.cloud.agent.api.VgpuTypesInfo; +import com.cloud.host.dao.HostTagsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceState; @@ -46,7 +47,10 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.GenericDao; import java.util.Arrays; + +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang3.StringUtils; @Entity @@ -159,6 +163,14 @@ public class HostVO implements Host { @Transient List hostTags; + /** + * This is a delayed load value. + * If the value is null, then this field has not been loaded yet. + * Call host dao to load it. + */ + @Transient + Boolean isTagARule; + // This value is only for saving and current cannot be loaded. @Transient HashMap> groupDetails = new HashMap>(); @@ -322,8 +334,13 @@ public class HostVO implements Host { return hostTags; } - public void setHostTags(List hostTags) { + public void setHostTags(List hostTags, Boolean isTagARule) { this.hostTags = hostTags; + this.isTagARule = isTagARule; + } + + public Boolean getIsTagARule() { + return isTagARule; } public HashMap> getGpuGroupDetails() { @@ -748,6 +765,11 @@ public class HostVO implements Host { if (serviceOffering == null) { return false; } + + if (BooleanUtils.isTrue(this.getIsTagARule())) { + return TagAsRuleHelper.interpretTagAsRule(this.getHostTags().get(0), serviceOffering.getHostTag(), HostTagsDao.hostTagRuleExecutionTimeout.value()); + } + if (StringUtils.isEmpty(serviceOffering.getHostTag())) { return true; } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index ca51ad3f428..fe30722feb1 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -108,6 +108,8 @@ public interface HostDao extends GenericDao, StateDao listAllHostsByZoneAndHypervisorType(long zoneId, HypervisorType hypervisorType); + List listAllHostsThatHaveNoRuleTag(Host.Type type, Long clusterId, Long podId, Long dcId); + List listAllHostsByType(Host.Type type); HostVO findByPublicIp(String publicIp); @@ -161,4 +163,8 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); + + List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + + List findClustersThatMatchHostTagRule(String computeOfferingTags); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 392b623299f..136351527f6 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -22,9 +22,11 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; @@ -32,6 +34,8 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.persistence.TableGenerator; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.VgpuTypesInfo; @@ -80,8 +84,8 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao private static final Logger state_logger = Logger.getLogger(ResourceState.class); private static final String LIST_HOST_IDS_BY_COMPUTETAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count " - + "FROM (SELECT host_id, tag FROM host_tags GROUP BY host_id,tag) AS filtered " - + "WHERE tag IN(%s) " + + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag) AS filtered " + + "WHERE tag IN(%s) AND is_tag_a_rule = 0 " + "GROUP BY host_id " + "HAVING tag_count = %s "; private static final String SEPARATOR = ","; @@ -148,6 +152,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected GenericSearchBuilder AllClustersSearch; protected SearchBuilder HostsInClusterSearch; + protected SearchBuilder searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag; + + protected SearchBuilder searchBuilderFindByRuleTag; + protected Attribute _statusAttr; protected Attribute _resourceStateAttr; protected Attribute _msIdAttr; @@ -455,6 +463,22 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ); HostIdSearch.done(); + searchBuilderFindByRuleTag = _hostTagsDao.createSearchBuilder(); + searchBuilderFindByRuleTag.and("is_tag_a_rule", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.EQ); + searchBuilderFindByRuleTag.or("tagDoesNotExist", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.NULL); + + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag = createSearchBuilder(); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("type", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getType(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("cluster_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getClusterId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("pod_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getPodId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("data_center_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getDataCenterId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.join("id", searchBuilderFindByRuleTag, searchBuilderFindByRuleTag.entity().getHostId(), + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), JoinType.LEFTOUTER); + + searchBuilderFindByRuleTag.done(); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.done(); + _statusAttr = _allAttributes.get("status"); _msIdAttr = _allAttributes.get("managementServerId"); _pingTimeAttr = _allAttributes.get("lastPinged"); @@ -792,9 +816,12 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public List listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag) { - SearchBuilder hostTagSearch = null; + SearchBuilder hostTagSearch = _hostTagsDao.createSearchBuilder(); + hostTagSearch.and(); + hostTagSearch.op("isTagARule", hostTagSearch.entity().getIsTagARule(), Op.EQ); + hostTagSearch.or("tagDoesNotExist", hostTagSearch.entity().getIsTagARule(), Op.NULL); + hostTagSearch.cp(); if (haTag != null && !haTag.isEmpty()) { - hostTagSearch = _hostTagsDao.createSearchBuilder(); hostTagSearch.and().op("tag", hostTagSearch.entity().getTag(), SearchCriteria.Op.NEQ); hostTagSearch.or("tagNull", hostTagSearch.entity().getTag(), SearchCriteria.Op.NULL); hostTagSearch.cp(); @@ -809,12 +836,14 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao hostSearch.and("status", hostSearch.entity().getStatus(), SearchCriteria.Op.EQ); hostSearch.and("resourceState", hostSearch.entity().getResourceState(), SearchCriteria.Op.EQ); - if (haTag != null && !haTag.isEmpty()) { - hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER); - } + + hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER); + SearchCriteria sc = hostSearch.create(); + sc.setJoinParameters("hostTagSearch", "isTagARule", false); + if (haTag != null && !haTag.isEmpty()) { sc.setJoinParameters("hostTagSearch", "tag", haTag); } @@ -846,8 +875,13 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public void loadHostTags(HostVO host) { - List hostTags = _hostTagsDao.getHostTags(host.getId()); - host.setHostTags(hostTags); + List hostTagVOList = _hostTagsDao.getHostTags(host.getId()); + if (CollectionUtils.isNotEmpty(hostTagVOList)) { + List hostTagList = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); + host.setHostTags(hostTagList, hostTagVOList.get(0).getIsTagARule()); + } else { + host.setHostTags(null, null); + } } @DB @@ -881,10 +915,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected void saveHostTags(HostVO host) { List hostTags = host.getHostTags(); - if (hostTags == null || (hostTags != null && hostTags.isEmpty())) { + if (CollectionUtils.isEmpty(hostTags)) { return; } - _hostTagsDao.persist(host.getId(), hostTags); + _hostTagsDao.persist(host.getId(), hostTags, host.getIsTagARule()); } protected void saveGpuRecords(HostVO host) { @@ -1244,6 +1278,26 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public List listAllHostsThatHaveNoRuleTag(Type type, Long clusterId, Long podId, Long dcId) { + SearchCriteria sc = searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.create(); + if (type != null) { + sc.setParameters("type", type); + } + if (clusterId != null) { + sc.setParameters("cluster_id", clusterId); + } + if (podId != null) { + sc.setParameters("pod_id", podId); + } + if (dcId != null) { + sc.setParameters("data_center_id", dcId); + } + sc.setJoinParameters("id", "is_tag_a_rule", false); + + return search(sc, null); + } + @Override public List listClustersByHostTag(String computeOfferingTags) { TransactionLegacy txn = TransactionLegacy.currentTxn(); @@ -1266,9 +1320,6 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao result.add(rs.getLong(1)); } pstmt.close(); - if(result.isEmpty()){ - throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags); - } return result; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + sql, e); @@ -1293,15 +1344,33 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao result.add(rs.getLong(1)); } pstmt.close(); - if(result.isEmpty()){ - throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags); - } return result; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + select, e); } } + public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + List hostTagVOList = _hostTagsDao.findHostRuleTags(); + List result = new ArrayList<>(); + for (HostTagVO rule: hostTagVOList) { + if (TagAsRuleHelper.interpretTagAsRule(rule.getTag(), computeOfferingTags, HostTagsDao.hostTagRuleExecutionTimeout.value())) { + result.add(findById(rule.getHostId())); + } + } + + return result; + } + + public List findClustersThatMatchHostTagRule(String computeOfferingTags) { + Set result = new HashSet<>(); + List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + for (HostVO host: hosts) { + result.add(host.getClusterId()); + } + return new ArrayList<>(result); + } + private String getHostIdsByComputeTags(List offeringTags){ List questionMarks = new ArrayList(); offeringTags.forEach((tag) -> { questionMarks.add("?"); }); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 0fb5370d81a..d134db33403 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -20,15 +20,21 @@ import java.util.List; import com.cloud.host.HostTagVO; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.framework.config.ConfigKey; public interface HostTagsDao extends GenericDao { - void persist(long hostId, List hostTags); + ConfigKey hostTagRuleExecutionTimeout = new ConfigKey<>("Advanced", Long.class, "host.tag.rule.execution.timeout", "2000", "The maximum runtime, in milliseconds, " + + "to execute a host tag rule; if it is reached, a timeout will happen.", true); - List getHostTags(long hostId); + void persist(long hostId, List hostTags, Boolean isTagARule); + + List getHostTags(long hostId); List getDistinctImplicitHostTags(List hostIds, String[] implicitHostTags); void deleteTags(long hostId); + List findHostRuleTags(); + } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index a73899b7b33..65deb1d1c9b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -16,10 +16,10 @@ // under the License. package com.cloud.host.dao; -import java.util.ArrayList; import java.util.List; - +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.springframework.stereotype.Component; import com.cloud.host.HostTagVO; @@ -31,13 +31,14 @@ import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Func; @Component -public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao { +public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao, Configurable { protected final SearchBuilder HostSearch; protected final GenericSearchBuilder DistinctImplictTagsSearch; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ); + HostSearch.and("isTagARule", HostSearch.entity().getIsTagARule(), SearchCriteria.Op.EQ); HostSearch.done(); DistinctImplictTagsSearch = createSearchBuilder(String.class); @@ -48,17 +49,11 @@ public class HostTagsDaoImpl extends GenericDaoBase implements } @Override - public List getHostTags(long hostId) { + public List getHostTags(long hostId) { SearchCriteria sc = HostSearch.create(); sc.setParameters("hostId", hostId); - List results = search(sc, null); - List hostTags = new ArrayList(results.size()); - for (HostTagVO result : results) { - hostTags.add(result.getTag()); - } - - return hostTags; + return search(sc, null); } @Override @@ -80,7 +75,15 @@ public class HostTagsDaoImpl extends GenericDaoBase implements } @Override - public void persist(long hostId, List hostTags) { + public List findHostRuleTags() { + SearchCriteria sc = HostSearch.create(); + sc.setParameters("isTagARule", true); + + return search(sc, null); + } + + @Override + public void persist(long hostId, List hostTags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); @@ -91,10 +94,20 @@ public class HostTagsDaoImpl extends GenericDaoBase implements for (String tag : hostTags) { tag = tag.trim(); if (tag.length() > 0) { - HostTagVO vo = new HostTagVO(hostId, tag); + HostTagVO vo = new HostTagVO(hostId, tag, isTagARule); persist(vo); } } txn.commit(); } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {hostTagRuleExecutionTimeout}; + } + + @Override + public String getConfigComponentName() { + return HostTagsDaoImpl.class.getSimpleName(); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java b/engine/schema/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java similarity index 97% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java rename to engine/schema/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java index e44087e6ada..b4bd56f5b4e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/hypervisor/vmware/dao/VmwareDatacenterDaoImpl.java @@ -20,10 +20,11 @@ package com.cloud.hypervisor.vmware.dao; import java.util.List; +import com.cloud.dc.dao.VmwareDatacenterDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDao.java index f4d0ad71b8c..43d4d25305e 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDao.java @@ -17,6 +17,7 @@ package com.cloud.network.dao; import java.util.List; +import java.util.Map; import com.cloud.utils.db.GenericDao; @@ -26,4 +27,6 @@ public interface NetworkDomainDao extends GenericDao { NetworkDomainVO getDomainNetworkMapByNetworkId(long networkId); List listNetworkIdsByDomain(long domainId); + + Map> listDomainsOfSharedNetworksUsedByDomainPath(String domainPath); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDaoImpl.java index 188f306cecc..ce86a8636a1 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDomainDaoImpl.java @@ -16,10 +16,17 @@ // under the License. package com.cloud.network.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; - +import com.cloud.utils.db.TransactionLegacy; +import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.DB; @@ -31,9 +38,23 @@ import com.cloud.utils.db.SearchCriteria.Op; @Component @DB() public class NetworkDomainDaoImpl extends GenericDaoBase implements NetworkDomainDao { + public static Logger logger = Logger.getLogger(NetworkDomainDaoImpl.class.getName()); final SearchBuilder AllFieldsSearch; final SearchBuilder DomainsSearch; + private static final String LIST_DOMAINS_OF_SHARED_NETWORKS_USED_BY_DOMAIN_PATH = "SELECT shared_nw.domain_id, \n" + + "GROUP_CONCAT('VM:', vm.uuid, ' | NW:' , network.uuid) \n" + + "FROM cloud.domain_network_ref AS shared_nw\n" + + "INNER JOIN cloud.nics AS nic ON (nic.network_id = shared_nw.network_id AND nic.removed IS NULL)\n" + + "INNER JOIN cloud.vm_instance AS vm ON (vm.id = nic.instance_id)\n" + + "INNER JOIN cloud.domain AS domain ON (domain.id = vm.domain_id)\n" + + "INNER JOIN cloud.domain AS domain_sn ON (domain_sn.id = shared_nw.domain_id)\n" + + "INNER JOIN cloud.networks AS network ON (shared_nw.network_id = network.id)\n" + + "WHERE shared_nw.subdomain_access = 1\n" + + "AND domain.path LIKE ?\n" + + "AND domain_sn.path NOT LIKE ?\n" + + "GROUP BY shared_nw.network_id"; + protected NetworkDomainDaoImpl() { super(); @@ -71,4 +92,37 @@ public class NetworkDomainDaoImpl extends GenericDaoBase } return networkIdsToReturn; } + + @Override + public Map> listDomainsOfSharedNetworksUsedByDomainPath(String domainPath) { + logger.debug(String.format("Retrieving the domains of the shared networks with subdomain access used by domain with path [%s].", domainPath)); + + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try (PreparedStatement pstmt = txn.prepareStatement(LIST_DOMAINS_OF_SHARED_NETWORKS_USED_BY_DOMAIN_PATH)) { + Map> domainsOfSharedNetworksUsedByDomainPath = new HashMap<>(); + + String domainSearch = domainPath.concat("%"); + pstmt.setString(1, domainSearch); + pstmt.setString(2, domainSearch); + + try (ResultSet rs = pstmt.executeQuery()) { + while (rs.next()) { + Long domainId = rs.getLong(1); + List vmUuidsAndNetworkUuids = Arrays.asList(rs.getString(2).split(",")); + + domainsOfSharedNetworksUsedByDomainPath.put(domainId, vmUuidsAndNetworkUuids); + } + } + + return domainsOfSharedNetworksUsedByDomainPath; + } catch (SQLException e) { + logger.error(String.format("Failed to retrieve the domains of the shared networks with subdomain access used by domain with path [%s] due to [%s]. Returning an empty " + + "list of domains.", domainPath, e.getMessage())); + + logger.debug(String.format("Failed to retrieve the domains of the shared networks with subdomain access used by domain with path [%s]. Returning an empty " + + "list of domains.", domainPath), e); + + return new HashMap<>(); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java index 4eaa2b575e0..280d5dfaf4b 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java @@ -17,6 +17,8 @@ package com.cloud.network.vpc; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + import java.util.UUID; import javax.persistence.Column; @@ -85,6 +87,11 @@ public class NetworkACLVO implements NetworkACL { return name; } + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "uuid", "name", "vpcId"); + } + public void setUuid(String uuid) { this.uuid = uuid; } diff --git a/engine/schema/src/main/java/com/cloud/storage/BucketVO.java b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java new file mode 100644 index 00000000000..181b02e5a1b --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage; + +import com.cloud.utils.db.GenericDao; +import com.google.gson.annotations.Expose; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "bucket") +public class BucketVO implements Bucket { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "account_id") + long accountId; + + @Column(name = "domain_id") + long domainId; + + @Column(name = "object_store_id") + long objectStoreId; + + @Expose + @Column(name = "name") + String name; + + @Expose + @Column(name = "state", updatable = true, nullable = false) + @Enumerated(value = EnumType.STRING) + private State state; + + @Column(name = "size") + Long size; + + @Column(name = "quota") + Integer quota; + + @Column(name = "versioning") + boolean versioning; + + @Column(name = "encryption") + boolean encryption; + + @Column(name = "object_lock") + boolean objectLock; + + @Column(name = "policy") + String policy; + + @Column(name = "bucket_url") + String bucketURL; + + @Column(name = "access_key") + String accessKey; + + @Column(name = "secret_key") + String secretKey; + + @Column(name = GenericDao.CREATED_COLUMN) + Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + Date removed; + + @Column(name = "uuid") + String uuid; + + public BucketVO() { + } + + public BucketVO(long accountId, long domainId, long objectStoreId, String name, Integer quota, boolean versioning, + boolean encryption, boolean objectLock, String policy) + { + this.accountId = accountId; + this.domainId = domainId; + this.objectStoreId = objectStoreId; + this.name = name; + state = State.Allocated; + uuid = UUID.randomUUID().toString(); + this.quota = quota; + this.versioning = versioning; + this.encryption = encryption; + this.objectLock = objectLock; + this.policy = policy; + this.size = 0L; + } + + @Override + public long getId() { + return id; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getObjectStoreId() { + return objectStoreId; + } + + @Override + public String getName() { + return name; + } + + public Long getSize() { + return size; + } + + @Override + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + @Override + public State getState() { + return state; + } + + @Override + public void setName(String name) { + this.name = name; + } + + public void setState(State state) { + this.state = state; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public Integer getQuota() { + return quota; + } + + public void setQuota(Integer quota) { + this.quota = quota; + } + + public boolean isVersioning() { + return versioning; + } + + public void setVersioning(boolean versioning) { + this.versioning = versioning; + } + + public boolean isEncryption() { + return encryption; + } + + public void setEncryption(boolean encryption) { + this.encryption = encryption; + } + + public boolean isObjectLock() { + return objectLock; + } + + public void setObjectLock(boolean objectLock) { + this.objectLock = objectLock; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(String policy) { + this.policy = policy; + } + + public String getBucketURL() { + return bucketURL; + } + public void setBucketURL(String bucketURL) { + this.bucketURL = bucketURL; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setSize(Long size) { + this.size = size; + } + + @Override + public Class getEntityType() { + return Bucket.class; + } + + @Override + public String toString() { + return String.format("Bucket %s", new ToStringBuilder(this, ToStringStyle.JSON_STYLE).append("uuid", getUuid()).append("name", getName()) + .append("ObjectStoreId", getObjectStoreId()).toString()); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java b/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java index 18c0dc3c326..2675c36f27f 100755 --- a/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java @@ -23,7 +23,9 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import com.cloud.utils.NumbersUtil; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.BooleanUtils; @Entity @Table(name = "storage_pool_tags") @@ -43,9 +45,19 @@ public class StoragePoolTagVO implements InternalIdentity { @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + public StoragePoolTagVO(long poolId, String tag) { this.poolId = poolId; this.tag = tag; + this.isTagARule = false; + } + + public StoragePoolTagVO(long poolId, String tag, Boolean isTagARule) { + this.poolId = poolId; + this.tag = tag; + this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false); } @Override @@ -61,4 +73,20 @@ public class StoragePoolTagVO implements InternalIdentity { return tag; } + public boolean isTagARule() { + return this.isTagARule; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof StoragePoolTagVO) { + return this.poolId == ((StoragePoolTagVO)obj).getPoolId(); + } + return false; + } + + @Override + public int hashCode() { + return NumbersUtil.hash(id); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java new file mode 100644 index 00000000000..f45f28b5c2c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.dao; + +import com.cloud.storage.BucketVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface BucketDao extends GenericDao { + List listByObjectStoreId(long objectStoreId); + + List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId); + + List searchByIds(Long[] ids); +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java new file mode 100644 index 00000000000..83b5f6bdb74 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.dao; + +import com.cloud.storage.BucketVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.naming.ConfigurationException; +import java.util.List; +import java.util.Map; + +@Component +public class BucketDaoImpl extends GenericDaoBase implements BucketDao { + public static final Logger s_logger = Logger.getLogger(BucketDaoImpl.class.getName()); + private SearchBuilder searchFilteringStoreId; + + private SearchBuilder bucketSearch; + + private static final String STORE_ID = "store_id"; + private static final String STATE = "state"; + private static final String ACCOUNT_ID = "account_id"; + + protected BucketDaoImpl() { + + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + searchFilteringStoreId = createSearchBuilder(); + searchFilteringStoreId.and(STORE_ID, searchFilteringStoreId.entity().getObjectStoreId(), SearchCriteria.Op.EQ); + searchFilteringStoreId.and(ACCOUNT_ID, searchFilteringStoreId.entity().getAccountId(), SearchCriteria.Op.EQ); + searchFilteringStoreId.and(STATE, searchFilteringStoreId.entity().getState(), SearchCriteria.Op.NEQ); + searchFilteringStoreId.done(); + + bucketSearch = createSearchBuilder(); + bucketSearch.and("idIN", bucketSearch.entity().getId(), SearchCriteria.Op.IN); + bucketSearch.done(); + + return true; + } + @Override + public List listByObjectStoreId(long objectStoreId) { + SearchCriteria sc = searchFilteringStoreId.create(); + sc.setParameters(STORE_ID, objectStoreId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return listBy(sc); + } + + @Override + public List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId) { + SearchCriteria sc = searchFilteringStoreId.create(); + sc.setParameters(STORE_ID, objectStoreId); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return listBy(sc); + } + + @Override + public List searchByIds(Long[] ids) { + SearchCriteria sc = bucketSearch.create(); + sc.setParameters("idIN", ids); + return search(sc, null, null, false); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java index 946b46b5cfe..9352ee21858 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java @@ -25,10 +25,14 @@ import com.cloud.utils.db.GenericDao; public interface StoragePoolTagsDao extends GenericDao { - void persist(long poolId, List storagePoolTags); + void persist(long poolId, List storagePoolTags, Boolean isTagARule); + + void persist(List storagePoolTags); List getStoragePoolTags(long poolId); void deleteTags(long poolId); List searchByIds(Long... stIds); StorageTagResponse newStorageTagResponse(StoragePoolTagVO tag); + List findStoragePoolTags(long poolId); + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java index f20e0c411de..c01c66763af 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java @@ -21,6 +21,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -50,7 +53,7 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase storagePoolTags) { + public void persist(long poolId, List storagePoolTags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); @@ -61,13 +64,23 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase 0) { - StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag); + StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag, isTagARule); persist(vo); } } txn.commit(); } + public void persist(List storagePoolTags) { + Transaction.execute(TransactionLegacy.CLOUD_DB, new TransactionCallbackNoReturn() { + @Override public void doInTransactionWithoutResult(TransactionStatus status) { + for (StoragePoolTagVO storagePoolTagVO : storagePoolTags) { + persist(storagePoolTagVO); + } + } + }); + } + @Override public List getStoragePoolTags(long poolId) { SearchCriteria sc = StoragePoolSearch.create(); @@ -157,4 +170,12 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase findStoragePoolTags(long poolId) { + SearchCriteria sc = StoragePoolSearch.create(); + sc.setParameters("poolId", poolId); + + return search(sc, null); + } + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 79899b7119e..be6588e3189 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -152,5 +152,7 @@ public interface VolumeDao extends GenericDao, StateDao listByPoolIdAndPaths(long id, List pathList); + VolumeVO findByPoolIdAndPath(long id, String path); + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 056b7206d72..bf556622463 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -71,6 +71,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol protected GenericSearchBuilder primaryStorageSearch; protected GenericSearchBuilder primaryStorageSearch2; protected GenericSearchBuilder secondaryStorageSearch; + private final SearchBuilder poolAndPathSearch; @Inject ResourceTagDao _tagsDao; @@ -487,6 +488,11 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol volumeIdSearch.and("idIN", volumeIdSearch.entity().getId(), Op.IN); volumeIdSearch.done(); + poolAndPathSearch = createSearchBuilder(); + poolAndPathSearch.and("poolId", poolAndPathSearch.entity().getPoolId(), Op.EQ); + poolAndPathSearch.and("path", poolAndPathSearch.entity().getPath(), Op.EQ); + poolAndPathSearch.done(); + } @Override @@ -802,6 +808,14 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol return listBy(sc); } + @Override + public VolumeVO findByPoolIdAndPath(long id, String path) { + SearchCriteria sc = poolAndPathSearch.create(); + sc.setParameters("poolId", id); + sc.setParameters("path", path); + return findOneBy(sc); + } + @Override public List listByIds(List ids) { if (CollectionUtils.isEmpty(ids)) { diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 0b38acb5c21..de161afea07 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -21,6 +21,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; public class DatabaseAccessObject { @@ -85,8 +86,8 @@ public class DatabaseAccessObject { return columnExists; } - public String generateIndexName(String tableName, String columnName) { - return String.format("i_%s__%s", tableName, columnName); + public String generateIndexName(String tableName, String... columnName) { + return String.format("i_%s__%s", tableName, StringUtils.join(columnName, "__")); } public boolean indexExists(Connection conn, String tableName, String indexName) { @@ -101,8 +102,8 @@ public class DatabaseAccessObject { return false; } - public void createIndex(Connection conn, String tableName, String columnName, String indexName) { - String stmt = String.format("CREATE INDEX %s on %s (%s)", indexName, tableName, columnName); + public void createIndex(Connection conn, String tableName, String indexName, String... columnNames) { + String stmt = String.format("CREATE INDEX %s ON %s (%s)", indexName, tableName, StringUtils.join(columnNames, ", ")); s_logger.debug("Statement: " + stmt); try (PreparedStatement pstmt = conn.prepareStatement(stmt)) { pstmt.execute(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index 6b4e1814de0..51e6ac7b9a1 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -23,11 +23,11 @@ public class DbUpgradeUtils { private static DatabaseAccessObject dao = new DatabaseAccessObject(); - public static void addIndexIfNeeded(Connection conn, String tableName, String columnName) { - String indexName = dao.generateIndexName(tableName, columnName); + public static void addIndexIfNeeded(Connection conn, String tableName, String... columnNames) { + String indexName = dao.generateIndexName(tableName, columnNames); if (!dao.indexExists(conn, tableName, indexName)) { - dao.createIndex(conn, tableName, columnName, indexName); + dao.createIndex(conn, tableName, indexName, columnNames); } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java index fd44e79e7cf..bdfe58cbf89 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java @@ -76,6 +76,7 @@ public class Upgrade41810to41900 implements DbUpgrade, DbUpgradeSystemVmTemplate public void performDataMigration(Connection conn) { decryptConfigurationValuesFromAccountAndDomainScopesNotInSecureHiddenCategories(conn); migrateBackupDates(conn); + addIndexes(conn); } @Override @@ -254,4 +255,11 @@ public class Upgrade41810to41900 implements DbUpgrade, DbUpgradeSystemVmTemplate } } + private void addIndexes(Connection conn) { + DbUpgradeUtils.addIndexIfNeeded(conn, "alert", "archived", "created"); + DbUpgradeUtils.addIndexIfNeeded(conn, "alert", "type", "data_center_id", "pod_id"); + + DbUpgradeUtils.addIndexIfNeeded(conn, "event", "resource_type", "resource_id"); + } + } diff --git a/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java b/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java new file mode 100644 index 00000000000..ab5fcfc493c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage; + +import org.apache.cloudstack.api.InternalIdentity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "bucket_statistics") +public class BucketStatisticsVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private Long id; + + @Column(name = "account_id", updatable = false) + private long accountId; + + @Column(name = "bucket_id", updatable = false) + private long bucketId; + + @Column(name = "size") + private long size; + + protected BucketStatisticsVO() { + } + + public BucketStatisticsVO(long accountId, long bucketId) { + this.accountId = accountId; + this.bucketId = bucketId; + } + + public long getAccountId() { + return accountId; + } + + public long getBucketId() { + return bucketId; + } + + @Override + public long getId() { + return id; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java new file mode 100644 index 00000000000..48388af9446 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface BucketStatisticsDao extends GenericDao { + BucketStatisticsVO findBy(long accountId, long bucketId); + + BucketStatisticsVO lock(long accountId, long bucketId); + + List listBy(long accountId); +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java new file mode 100644 index 00000000000..2261389eab6 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class BucketStatisticsDaoImpl extends GenericDaoBase implements BucketStatisticsDao { + private static final Logger s_logger = Logger.getLogger(BucketStatisticsDaoImpl.class); + private final SearchBuilder AllFieldsSearch; + private final SearchBuilder AccountSearch; + + public BucketStatisticsDaoImpl() { + AccountSearch = createSearchBuilder(); + AccountSearch.and("account", AccountSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AccountSearch.done(); + + AllFieldsSearch = createSearchBuilder(); + AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AllFieldsSearch.and("bucket", AllFieldsSearch.entity().getBucketId(), SearchCriteria.Op.EQ); + AllFieldsSearch.done(); + } + + @Override + public BucketStatisticsVO findBy(long accountId, long bucketId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("account", accountId); + sc.setParameters("bucket", bucketId); + return findOneBy(sc); + } + + @Override + public BucketStatisticsVO lock(long accountId, long bucketId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("account", accountId); + sc.setParameters("bucket", bucketId); + return lockOneRandomRow(sc, true); + } + + @Override + public List listBy(long accountId) { + SearchCriteria sc = AccountSearch.create(); + sc.setParameters("account", accountId); + return search(sc, null); + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java index 4099b3ada0d..ea490e60f9e 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.dao; +import com.cloud.usage.BucketStatisticsVO; import com.cloud.usage.UsageVO; import com.cloud.user.AccountVO; import com.cloud.user.UserStatisticsVO; @@ -45,6 +46,12 @@ public interface UsageDao extends GenericDao { Long getLastUserStatsId(); + Long getLastBucketStatsId(); + + void saveBucketStats(List userStats); + + void updateBucketStats(List userStats); + List listPublicTemplatesByAccount(long accountId); Long getLastVmDiskStatsId(); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java index 4553ed822b4..0d9e727abe2 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.dao; +import com.cloud.usage.BucketStatisticsVO; import com.cloud.usage.UsageVO; import com.cloud.user.AccountVO; import com.cloud.user.UserStatisticsVO; @@ -82,6 +83,13 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage + "WHERE cloud_usage.usage_type = ? AND cloud_usage.account_id = ? AND cloud_usage.start_date >= ? AND cloud_usage.end_date <= ? " + "GROUP BY cloud_usage.usage_id "; + private static final String GET_LAST_BUCKET_STATS_ID = "SELECT id FROM cloud_usage.bucket_statistics ORDER BY id DESC LIMIT 1"; + + private static final String INSERT_BUCKET_STATS = "INSERT INTO cloud_usage.bucket_statistics (id, account_id, bucket_id, size) VALUES (?,?,?,?)"; + + private static final String UPDATE_BUCKET_STATS = "UPDATE cloud_usage.bucket_statistics SET size=? WHERE id=?"; + + public UsageDaoImpl() { } @@ -285,6 +293,69 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage return null; } + @Override + public Long getLastBucketStatsId() { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + PreparedStatement pstmt = null; + String sql = GET_LAST_BUCKET_STATS_ID; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return Long.valueOf(rs.getLong(1)); + } + } catch (Exception ex) { + s_logger.error("error getting last bucket stats id", ex); + } + return null; + } + + @Override + public void saveBucketStats(List bucketStats) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + txn.start(); + String sql = INSERT_BUCKET_STATS; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); // in reality I just want CLOUD_USAGE dataSource connection + for (BucketStatisticsVO bucketStat : bucketStats) { + pstmt.setLong(1, bucketStat.getId()); + pstmt.setLong(2, bucketStat.getAccountId()); + pstmt.setLong(3, bucketStat.getBucketId()); + pstmt.setLong(4, bucketStat.getSize()); + pstmt.addBatch(); + } + pstmt.executeBatch(); + txn.commit(); + } catch (Exception ex) { + txn.rollback(); + s_logger.error("error saving bucket stats to cloud_usage db", ex); + throw new CloudRuntimeException(ex.getMessage()); + } + } + + @Override + public void updateBucketStats(List bucketStats) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + txn.start(); + String sql = UPDATE_BUCKET_STATS; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); // in reality I just want CLOUD_USAGE dataSource connection + for (BucketStatisticsVO bucketStat : bucketStats) { + pstmt.setLong(1, bucketStat.getSize()); + pstmt.setLong(2, bucketStat.getId()); + pstmt.addBatch(); + } + pstmt.executeBatch(); + txn.commit(); + } catch (Exception ex) { + txn.rollback(); + s_logger.error("error updating bucket stats to cloud_usage db", ex); + throw new CloudRuntimeException(ex.getMessage()); + } + } + @Override public List listPublicTemplatesByAccount(long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 22165641f7e..68f57329d77 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -79,7 +79,9 @@ public interface NicDao extends GenericDao { List listByNetworkIdTypeAndGatewayAndBroadcastUri(long networkId, VirtualMachine.Type vmType, String gateway, URI broadcastUri); - int countNicsForStartingVms(long networkId); + int countNicsForNonStoppedVms(long networkId); + + int countNicsForNonStoppedRunningVrs(long networkId); NicVO getControlNicForVM(long vmId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index 211e5480423..59d2417b073 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -44,7 +44,7 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { private GenericSearchBuilder IpSearch; private SearchBuilder NonReleasedSearch; private GenericSearchBuilder deviceIdSearch; - private GenericSearchBuilder CountByForStartingVms; + private GenericSearchBuilder CountByForNonStoppedVms; private SearchBuilder PeerRouterSearch; @Inject @@ -91,14 +91,16 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { deviceIdSearch.and("instance", deviceIdSearch.entity().getInstanceId(), Op.EQ); deviceIdSearch.done(); - CountByForStartingVms = createSearchBuilder(Integer.class); - CountByForStartingVms.select(null, Func.COUNT, CountByForStartingVms.entity().getId()); - CountByForStartingVms.and("networkId", CountByForStartingVms.entity().getNetworkId(), Op.EQ); - CountByForStartingVms.and("removed", CountByForStartingVms.entity().getRemoved(), Op.NULL); + CountByForNonStoppedVms = createSearchBuilder(Integer.class); + CountByForNonStoppedVms.select(null, Func.COUNT, CountByForNonStoppedVms.entity().getId()); + CountByForNonStoppedVms.and("vmType", CountByForNonStoppedVms.entity().getVmType(), Op.EQ); + CountByForNonStoppedVms.and("vmTypeNEQ", CountByForNonStoppedVms.entity().getVmType(), Op.NEQ); + CountByForNonStoppedVms.and("networkId", CountByForNonStoppedVms.entity().getNetworkId(), Op.EQ); + CountByForNonStoppedVms.and("removed", CountByForNonStoppedVms.entity().getRemoved(), Op.NULL); SearchBuilder join1 = _vmDao.createSearchBuilder(); - join1.and("state", join1.entity().getState(), Op.EQ); - CountByForStartingVms.join("vm", join1, CountByForStartingVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); - CountByForStartingVms.done(); + join1.and("state", join1.entity().getState(), Op.IN); + CountByForNonStoppedVms.join("vm", join1, CountByForNonStoppedVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); + CountByForNonStoppedVms.done(); PeerRouterSearch = createSearchBuilder(); PeerRouterSearch.and("instanceId", PeerRouterSearch.entity().getInstanceId(), Op.NEQ); @@ -346,10 +348,21 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { } @Override - public int countNicsForStartingVms(long networkId) { - SearchCriteria sc = CountByForStartingVms.create(); + public int countNicsForNonStoppedVms(long networkId) { + SearchCriteria sc = CountByForNonStoppedVms.create(); sc.setParameters("networkId", networkId); - sc.setJoinParameters("vm", "state", VirtualMachine.State.Starting); + sc.setParameters("vmType", VirtualMachine.Type.User); + sc.setJoinParameters("vm", "state", new Object[] {VirtualMachine.State.Starting, VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Migrating}); + List results = customSearch(sc, null); + return results.get(0); + } + + @Override + public int countNicsForNonStoppedRunningVrs(long networkId) { + SearchCriteria sc = CountByForNonStoppedVms.create(); + sc.setParameters("networkId", networkId); + sc.setParameters("vmTypeNEQ", VirtualMachine.Type.User); + sc.setJoinParameters("vm", "state", new Object[] {VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Migrating}); List results = customSearch(sc, null); return results.get(0); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDao.java b/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDao.java index 07be976f202..27040c2f54e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDao.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.affinity.dao; import java.util.List; +import java.util.Map; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; @@ -28,4 +29,6 @@ public interface AffinityGroupDomainMapDao extends GenericDao listByDomain(Object... domainId); + Map> listDomainsOfAffinityGroupsUsedByDomainPath(String domainPath); + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDaoImpl.java index 5ecf63d6a6c..1dd22df46d7 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/affinity/dao/AffinityGroupDomainMapDaoImpl.java @@ -16,23 +16,46 @@ // under the License. package org.apache.cloudstack.affinity.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.annotation.PostConstruct; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; +import org.apache.log4j.Logger; +import com.cloud.network.dao.NetworkDomainDaoImpl; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.TransactionLegacy; public class AffinityGroupDomainMapDaoImpl extends GenericDaoBase implements AffinityGroupDomainMapDao { + public static Logger logger = Logger.getLogger(NetworkDomainDaoImpl.class.getName()); private SearchBuilder ListByAffinityGroup; private SearchBuilder DomainsSearch; + private static final String LIST_DOMAINS_WITH_AFFINITY_GROUPS_WITH_SUBDOMAIN_ACCESS_USED_BY_DOMAIN_PATH = "SELECT affinity_group_domain_map.domain_id, \n" + + "GROUP_CONCAT('VM:', vm.uuid, ' | AG:' , affinity_group.uuid) \n" + + "FROM cloud.affinity_group_domain_map AS affinity_group_domain_map\n" + + "INNER JOIN cloud.affinity_group_vm_map AS affinity_group_vm_map ON (cloud.affinity_group_domain_map.affinity_group_id = affinity_group_vm_map.affinity_group_id)\n" + + "INNER JOIN cloud.vm_instance AS vm ON (vm.id = affinity_group_vm_map.instance_id)\n" + + "INNER JOIN cloud.domain AS domain ON (domain.id = vm.domain_id)\n" + + "INNER JOIN cloud.domain AS domain_sn ON (domain_sn.id = affinity_group_domain_map.domain_id)\n" + + "INNER JOIN cloud.affinity_group AS affinity_group ON (affinity_group.id = affinity_group_domain_map.affinity_group_id)\n" + + "WHERE affinity_group_domain_map.subdomain_access = 1\n" + + "AND domain.path LIKE ?\n" + + "AND domain_sn.path NOT LIKE ?\n" + + "GROUP BY affinity_group.id"; + public AffinityGroupDomainMapDaoImpl() { } @@ -62,4 +85,38 @@ public class AffinityGroupDomainMapDaoImpl extends GenericDaoBase> listDomainsOfAffinityGroupsUsedByDomainPath(String domainPath) { + logger.debug(String.format("Retrieving the domains of the affinity groups with subdomain access used by domain with path [%s].", domainPath)); + + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try (PreparedStatement pstmt = txn.prepareStatement(LIST_DOMAINS_WITH_AFFINITY_GROUPS_WITH_SUBDOMAIN_ACCESS_USED_BY_DOMAIN_PATH)) { + Map> domainsOfAffinityGroupsUsedByDomainPath = new HashMap<>(); + + String domainSearch = domainPath.concat("%"); + pstmt.setString(1, domainSearch); + pstmt.setString(2, domainSearch); + + + try (ResultSet rs = pstmt.executeQuery()) { + while (rs.next()) { + Long domainId = rs.getLong(1); + List vmUuidsAndAffinityGroupUuids = Arrays.asList(rs.getString(2).split(",")); + + domainsOfAffinityGroupsUsedByDomainPath.put(domainId, vmUuidsAndAffinityGroupUuids); + } + } + + return domainsOfAffinityGroupsUsedByDomainPath; + } catch (SQLException e) { + logger.error(String.format("Failed to retrieve the domains of the affinity groups with subdomain access used by domain with path [%s] due to [%s]. Returning an " + + "empty list of domains.", domainPath, e.getMessage())); + + logger.debug(String.format("Failed to retrieve the domains of the affinity groups with subdomain access used by domain with path [%s]. Returning an empty " + + "list of domains.", domainPath), e); + + return new HashMap<>(); + } + } + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java new file mode 100644 index 00000000000..b0da0c5e747 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "heuristics") +public class HeuristicVO implements Heuristic { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Long id; + + @Column(name = "uuid", nullable = false) + private String uuid = UUID.randomUUID().toString(); + + @Column(name = "name") + private String name; + + @Column(name = "description") + private String description; + + @Column(name = "zone_id", nullable = false) + private Long zoneId; + + @Column(name = "type", nullable = false) + private String type; + + @Column(name = "heuristic_rule", nullable = false, length = 65535) + private String heuristicRule; + + @Column(name = GenericDao.CREATED_COLUMN, nullable = false) + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed = null; + + public HeuristicVO() { + } + + public HeuristicVO(String name, String description, Long zoneId, String type, String heuristicRule) { + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + } + + @Override + public long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "heuristicRule", "type"); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java new file mode 100644 index 00000000000..1c4057a7beb --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; + +public interface SecondaryStorageHeuristicDao extends GenericDao { + + HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java new file mode 100644 index 00000000000..0b51b2aec6b --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class SecondaryStorageHeuristicDaoImpl extends GenericDaoBase implements SecondaryStorageHeuristicDao { + private SearchBuilder zoneAndTypeSearch; + + @PostConstruct + public void init() { + zoneAndTypeSearch = createSearchBuilder(); + zoneAndTypeSearch.and("zoneId", zoneAndTypeSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + zoneAndTypeSearch.and("type", zoneAndTypeSearch.entity().getType(), SearchCriteria.Op.IN); + zoneAndTypeSearch.done(); + } + + @Override + public HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type) { + SearchCriteria searchCriteria = zoneAndTypeSearch.create(); + searchCriteria.setParameters("zoneId", zoneId); + searchCriteria.setParameters("type", type.toString()); + final Filter filter = new Filter(HeuristicVO.class, "created", false); + + return findOneBy(searchCriteria, filter); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index ba9825c3c86..1c31b3e0cc4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -49,4 +49,6 @@ public interface ImageStoreDao extends GenericDao { List findByProtocol(String protocol); ImageStoreVO findOneByZoneAndProtocol(long zoneId, String protocol); + + List listImageStoresByZoneIds(Long... zoneIds); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java index 3468b6008d9..a4827a1beae 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java @@ -43,6 +43,8 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem private SearchBuilder protocolSearch; private SearchBuilder zoneProtocolSearch; + private SearchBuilder zonesInSearch; + public ImageStoreDaoImpl() { super(); protocolSearch = createSearchBuilder(); @@ -55,6 +57,11 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem zoneProtocolSearch.and("protocol", zoneProtocolSearch.entity().getProtocol(), SearchCriteria.Op.EQ); zoneProtocolSearch.and("role", zoneProtocolSearch.entity().getRole(), SearchCriteria.Op.EQ); zoneProtocolSearch.done(); + + zonesInSearch = createSearchBuilder(); + zonesInSearch.and("zonesIn", zonesInSearch.entity().getDcId(), SearchCriteria.Op.IN); + zonesInSearch.done(); + } @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -191,4 +198,12 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem List results = listBy(sc, filter); return results.size() == 0 ? null : results.get(0); } + + + @Override + public List listImageStoresByZoneIds(Long... zoneIds) { + SearchCriteria sc = zonesInSearch.create(); + sc.setParametersIfNotNull("zonesIn", zoneIds); + return listBy(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java new file mode 100644 index 00000000000..94f6b5ec372 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.api.response.ObjectStoreResponse; + +import java.util.List; + +public interface ObjectStoreDao extends GenericDao { + ObjectStoreVO findByName(String name); + + List findByProvider(String provider); + + ObjectStoreVO findByUrl(String url); + + List listObjectStores(); + + List searchByIds(Long[] osIds); + + ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store); + + ObjectStoreResponse setObjectStoreResponse(ObjectStoreResponse storeData, ObjectStoreVO store); + + Integer countAllObjectStores(); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java new file mode 100644 index 00000000000..51abde013b6 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreDaoImpl extends GenericDaoBase implements ObjectStoreDao { + private SearchBuilder nameSearch; + private SearchBuilder providerSearch; + @Inject + private ConfigurationDao _configDao; + private final SearchBuilder osSearch; + + private SearchBuilder urlSearch; + + protected ObjectStoreDaoImpl() { + osSearch = createSearchBuilder(); + osSearch.and("idIN", osSearch.entity().getId(), SearchCriteria.Op.IN); + osSearch.done(); + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + nameSearch = createSearchBuilder(); + nameSearch.and("name", nameSearch.entity().getName(), SearchCriteria.Op.EQ); + nameSearch.done(); + + providerSearch = createSearchBuilder(); + providerSearch.and("providerName", providerSearch.entity().getProviderName(), SearchCriteria.Op.EQ); + providerSearch.done(); + + urlSearch = createSearchBuilder(); + urlSearch.and("url", urlSearch.entity().getUrl(), SearchCriteria.Op.EQ); + urlSearch.done(); + + return true; + } + + @Override + public ObjectStoreVO findByName(String name) { + SearchCriteria sc = nameSearch.create(); + sc.setParameters("name", name); + return findOneBy(sc); + } + + @Override + public List findByProvider(String provider) { + SearchCriteria sc = providerSearch.create(); + sc.setParameters("providerName", provider); + return listBy(sc); + } + + @Override + public ObjectStoreVO findByUrl(String url) { + SearchCriteria sc = urlSearch.create(); + sc.setParameters("url", url); + return findOneBy(sc); + } + + @Override + public List listObjectStores() { + SearchCriteria sc = createSearchCriteria(); + return listBy(sc); + } + + @Override + public List searchByIds(Long[] osIds) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = _configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + // query details by batches + List osList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (osIds.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= osIds.length) { + Long[] ids = new Long[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + ids[k] = osIds[j]; + } + SearchCriteria sc = osSearch.create(); + sc.setParameters("idIN", ids); + List stores = searchIncludingRemoved(sc, null, null, false); + if (stores != null) { + osList.addAll(stores); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < osIds.length) { + int batch_size = (osIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = osIds[j]; + } + SearchCriteria sc = osSearch.create(); + sc.setParameters("idIN", ids); + List stores = searchIncludingRemoved(sc, null, null, false); + if (stores != null) { + osList.addAll(stores); + } + } + return osList; + } + + @Override + public ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store) { + ObjectStoreResponse osResponse = new ObjectStoreResponse(); + osResponse.setId(store.getUuid()); + osResponse.setName(store.getName()); + osResponse.setProviderName(store.getProviderName()); + String url = store.getUrl(); + osResponse.setUrl(url); + osResponse.setObjectName("objectstore"); + return osResponse; + } + + @Override + public ObjectStoreResponse setObjectStoreResponse(ObjectStoreResponse storeData, ObjectStoreVO store) { + return storeData; + } + + @Override + public Integer countAllObjectStores() { + SearchCriteria sc = createSearchCriteria(); + return getCount(sc); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java new file mode 100644 index 00000000000..1f4047f8f90 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import org.apache.cloudstack.api.ResourceDetail; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "object_store_details") +public class ObjectStoreDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "store_id") + long resourceId; + + @Column(name = "name") + String name; + + @Column(name = "value") + String value; + + public ObjectStoreDetailVO() { + } + public ObjectStoreDetailVO(long storeId, String name, String value) { + this.resourceId = storeId; + this.name = name; + this.value = value; + } + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java new file mode 100644 index 00000000000..170a28af502 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import java.util.Map; + +public interface ObjectStoreDetailsDao extends GenericDao, ResourceDetailsDao { + + void update(long storeId, Map details); + + Map getDetails(long storeId); + + void deleteDetails(long storeId); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java new file mode 100644 index 00000000000..e1000e5d045 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreDetailsDaoImpl extends ResourceDetailsDaoBase implements ObjectStoreDetailsDao { + + protected final SearchBuilder storeSearch; + + public ObjectStoreDetailsDaoImpl() { + super(); + storeSearch = createSearchBuilder(); + storeSearch.and("store", storeSearch.entity().getResourceId(), Op.EQ); + storeSearch.done(); + } + + @Override + public void update(long storeId, Map details) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + txn.start(); + expunge(sc); + for (Map.Entry entry : details.entrySet()) { + ObjectStoreDetailVO detail = new ObjectStoreDetailVO(storeId, entry.getKey(), entry.getValue()); + persist(detail); + } + txn.commit(); + } + + @Override + public Map getDetails(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + List details = listBy(sc); + Map detailsMap = new HashMap(); + for (ObjectStoreDetailVO detail : details) { + String name = detail.getName(); + String value = detail.getValue(); + if (name.equals(ApiConstants.KEY)) { + value = DBEncryptionUtil.decrypt(value); + } + detailsMap.put(name, value); + } + + return detailsMap; + } + + @Override + public void deleteDetails(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + List results = search(sc, null); + for (ObjectStoreDetailVO result : results) { + remove(result.getId()); + } + } + + @Override + public ObjectStoreDetailVO findDetail(long storeId, String name) { + QueryBuilder sc = QueryBuilder.create(ObjectStoreDetailVO.class); + sc.and(sc.entity().getResourceId(), Op.EQ, storeId); + sc.and(sc.entity().getName(), Op.EQ, name); + return sc.find(); + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + // ToDo: Add Display + super.addDetail(new ObjectStoreDetailVO(resourceId, key, value)); + } + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java new file mode 100644 index 00000000000..885cbfd98ab --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import org.apache.cloudstack.storage.object.ObjectStore; +import com.cloud.utils.db.GenericDao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.TableGenerator; +import javax.persistence.Transient; +import java.util.Date; +import java.util.Map; + +@Entity +@Table(name = "object_store") +public class ObjectStoreVO implements ObjectStore { + @Id + @TableGenerator(name = "object_store_sq", table = "sequence", pkColumnName = "name", valueColumnName = "value", pkColumnValue = "object_store_seq", allocationSize = 1) + @Column(name = "id", nullable = false) + private long id; + + @Column(name = "name", nullable = false) + private String name; + + @Column(name = "uuid", nullable = false) + private String uuid; + + @Column(name = "url", nullable = false, length = 2048) + private String url; + + @Column(name = "object_provider_name", nullable = false) + private String providerName; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + @Column(name = "total_size") + private Long totalSize; + + @Column(name = "used_bytes") + private Long usedBytes; + + @Transient + Map details; + + @Override + public long getId() { + return this.id; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public String getProviderName() { + return this.providerName; + } + + public void setName(String name) { + this.name = name; + } + + public void setProviderName(String provider) { + this.providerName = provider; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String getUuid() { + return this.uuid; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Long getTotalSize() { + return totalSize; + } + + public void setTotalSize(Long totalSize) { + this.totalSize = totalSize; + } + + public Long getUsedBytes() { + return usedBytes; + } + + public void setUsedBytes(Long usedBytes) { + this.usedBytes = usedBytes; + } + + public void setDetails(Map details) { + this.details = details; + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 80fddc9bd94..8f77b4ba63e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -53,7 +53,7 @@ public interface PrimaryDataStoreDao extends GenericDao { */ void updateCapacityIops(long id, long capacityIops); - StoragePoolVO persist(StoragePoolVO pool, Map details, List tags); + StoragePoolVO persist(StoragePoolVO pool, Map details, List tags, Boolean isTagARule); /** * Find pool by name. @@ -77,7 +77,7 @@ public interface PrimaryDataStoreDao extends GenericDao { */ List findPoolsByDetails(long dcId, long podId, Long clusterId, Map details, ScopeType scope); - List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags); + List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout); List findDisabledPoolsByScope(long dcId, Long podId, Long clusterId, ScopeType scope); @@ -112,9 +112,9 @@ public interface PrimaryDataStoreDao extends GenericDao { List listPoolsByCluster(long clusterId); - List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags); + List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule); - List findZoneWideStoragePoolsByTags(long dcId, String[] tags); + List findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule); List findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 2fc52b3fb28..af7dbdc0225 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -67,11 +67,11 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase protected final String DetailsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_details ON storage_pool.id = storage_pool_details.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; protected final String DetailsSqlSuffix = ") GROUP BY storage_pool_details.pool_id HAVING COUNT(storage_pool_details.name) >= ?"; - private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and storage_pool.scope = ? and ("; + private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and storage_pool.scope = ? and ("; private final String ZoneWideTagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?"; // Storage tags are now separate from storage_pool_details, leaving only details on that table - protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; + protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; protected final String TagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?"; private static final String GET_STORAGE_POOLS_OF_VOLUMES_WITHOUT_OR_NOT_HAVING_TAGS = "SELECT s.* " + @@ -278,7 +278,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase @Override @DB - public StoragePoolVO persist(StoragePoolVO pool, Map details, List tags) { + public StoragePoolVO persist(StoragePoolVO pool, Map details, List tags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); pool = super.persist(pool); @@ -289,7 +289,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } } if (CollectionUtils.isNotEmpty(tags)) { - _tagsDao.persist(pool.getId(), tags); + _tagsDao.persist(pool.getId(), tags, isTagARule); } txn.commit(); return pool; @@ -404,10 +404,15 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags) { + public List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout) { List storagePools = null; if (tags == null || tags.length == 0) { storagePools = listBy(dcId, podId, clusterId, ScopeType.CLUSTER); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } + } else { String sqlValues = getSqlValuesFromStorageTags(tags); storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.CLUSTER, sqlValues, ValueType.TAGS, tags.length); @@ -437,10 +442,14 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags) { + public List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule) { List storagePools = null; if (tags == null || tags.length == 0) { storagePools = listBy(dcId, podId, clusterId, ScopeType.HOST); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } } else { String sqlValues = getSqlValuesFromStorageTags(tags); storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.HOST, sqlValues, ValueType.TAGS, tags.length); @@ -483,13 +492,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findZoneWideStoragePoolsByTags(long dcId, String[] tags) { + public List findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule) { if (tags == null || tags.length == 0) { QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); sc.and(sc.entity().getDataCenterId(), Op.EQ, dcId); sc.and(sc.entity().getStatus(), Op.EQ, Status.Up); sc.and(sc.entity().getScope(), Op.EQ, ScopeType.ZONE); - return sc.list(); + + List storagePools = sc.list(); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } + + return storagePools; } else { String sqlValues = getSqlValuesFromStorageTags(tags); String sql = getSqlPreparedStatement(ZoneWideTagsSqlPrefix, ZoneWideTagsSqlSuffix, sqlValues, null); @@ -497,6 +513,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } } + protected List getPoolsWithoutTagRule(List storagePools) { + List storagePoolsToReturn = new ArrayList<>(); + for (StoragePoolVO storagePool : storagePools) { + + List poolTags = _tagsDao.findStoragePoolTags(storagePool.getId()); + + if (CollectionUtils.isEmpty(poolTags) || !poolTags.get(0).isTagARule()) { + storagePoolsToReturn.add(storagePool); + } + } + + return storagePoolsToReturn; + } + @Override public List searchForStoragePoolTags(long poolId) { return _tagsDao.getStoragePoolTags(poolId); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index f0c5e7630e1..5d958383161 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -276,11 +276,16 @@ + + + + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 61f03be1713..27170fcac14 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -44,6 +44,21 @@ CREATE TABLE IF NOT EXISTS `cloud`.`quarantined_ips` ( -- create_public_parameter_on_roles. #6960 ALTER TABLE `cloud`.`roles` ADD COLUMN `public_role` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Indicates whether the role will be visible to all users (public) or only to root admins (private). If this parameter is not specified during the creation of the role its value will be defaulted to true (public).'; +-- Create heuristic table for dynamic allocating resources to the secondary storage +CREATE TABLE IF NOT EXISTS `cloud`.`heuristics` ( + `id` bigint(20) unsigned NOT NULL auto_increment, + `uuid` varchar(255) UNIQUE NOT NULL, + `name` text NOT NULL, + `description` text DEFAULT NULL, + `zone_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the zone to apply the heuristic, foreign key to `data_center` table', + `type` varchar(255) NOT NULL, + `heuristic_rule` text NOT NULL COMMENT 'JS script that defines to which secondary storage the resource will be allocated.', + `created` datetime NOT NULL, + `removed` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `fk_heuristics__zone_id` FOREIGN KEY(`zone_id`) REFERENCES `cloud`.`data_center`(`id`) +); + -- Add tables for VM Scheduler DROP TABLE IF EXISTS `cloud`.`vm_schedule`; CREATE TABLE `cloud`.`vm_schedule` ( @@ -224,3 +239,78 @@ CREATE TABLE `cloud`.`oauth_provider` ( `removed` datetime COMMENT 'date removed if not null', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Flexible tags +ALTER TABLE `cloud`.`storage_pool_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0; + +ALTER TABLE `cloud`.`storage_pool_tags` MODIFY tag text NOT NULL; + +ALTER TABLE `cloud`.`host_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0; + +ALTER TABLE `cloud`.`host_tags` MODIFY tag text NOT NULL; + +DROP TABLE IF EXISTS `cloud`.`object_store`; +CREATE TABLE `cloud`.`object_store` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `name` varchar(255) NOT NULL COMMENT 'name of object store', + `object_provider_name` varchar(255) NOT NULL COMMENT 'id of object_store_provider', + `url` varchar(255) NOT NULL COMMENT 'url of the object store', + `uuid` varchar(255) COMMENT 'uuid of object store', + `created` datetime COMMENT 'date the object store first signed on', + `removed` datetime COMMENT 'date removed if not null', + `total_size` bigint unsigned COMMENT 'storage total size statistics', + `used_bytes` bigint unsigned COMMENT 'storage available bytes statistics', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`object_store_details`; +CREATE TABLE `cloud`.`object_store_details` ( + `id` bigint unsigned UNIQUE NOT NULL AUTO_INCREMENT COMMENT 'id', + `store_id` bigint unsigned NOT NULL COMMENT 'store the detail is related to', + `name` varchar(255) NOT NULL COMMENT 'name of the detail', + `value` varchar(255) NOT NULL COMMENT 'value of the detail', + PRIMARY KEY (`id`), + CONSTRAINT `fk_object_store_details__store_id` FOREIGN KEY `fk_object_store__store_id`(`store_id`) REFERENCES `object_store`(`id`) ON DELETE CASCADE, + INDEX `i_object_store__name__value`(`name`(128), `value`(128)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`bucket`; +CREATE TABLE `cloud`.`bucket` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `name` varchar(255) NOT NULL COMMENT 'name of bucket', + `object_store_id` varchar(255) NOT NULL COMMENT 'id of object_store', + `state` varchar(255) NOT NULL COMMENT 'state of the bucket', + `uuid` varchar(255) COMMENT 'uuid of bucket', + `domain_id` bigint unsigned NOT NULL COMMENT 'domain the bucket belongs to', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + `quota` bigint unsigned COMMENT 'Allocated bucket quota in GB', + `versioning` boolean COMMENT 'versioning enable/disable', + `encryption` boolean COMMENT 'encryption enable/disbale', + `object_lock` boolean COMMENT 'Lock objects in bucket', + `policy` varchar(255) COMMENT 'Bucket Access Policy', + `access_key` varchar(255) COMMENT 'Bucket Access Key', + `secret_key` varchar(255) COMMENT 'Bucket Secret Key', + `bucket_url` varchar(255) COMMENT 'URL to access bucket', + `created` datetime COMMENT 'date the bucket was created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`bucket_statistics`; +CREATE TABLE `cloud`.`bucket_statistics` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `bucket_id` bigint unsigned NOT NULL COMMENT 'id of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud_usage`.`bucket_statistics`; +CREATE TABLE `cloud_usage`.`bucket_statistics` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `bucket_id` bigint unsigned NOT NULL COMMENT 'id of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql new file mode 100644 index 00000000000..5c6d4fd772b --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql @@ -0,0 +1,111 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`host_view`; + +DROP VIEW IF EXISTS `cloud`.`host_view`; + +CREATE VIEW `cloud`.`host_view` AS +SELECT + host.id, + host.uuid, + host.name, + host.status, + host.disconnected, + host.type, + host.private_ip_address, + host.version, + host.hypervisor_type, + host.hypervisor_version, + host.capabilities, + host.last_ping, + host.created, + host.removed, + host.resource_state, + host.mgmt_server_id, + host.cpu_sockets, + host.cpus, + host.speed, + host.ram, + cluster.id cluster_id, + cluster.uuid cluster_uuid, + cluster.name cluster_name, + cluster.cluster_type, + data_center.id data_center_id, + data_center.uuid data_center_uuid, + data_center.name data_center_name, + data_center.networktype data_center_type, + host_pod_ref.id pod_id, + host_pod_ref.uuid pod_uuid, + host_pod_ref.name pod_name, + GROUP_CONCAT(DISTINCT(host_tags.tag)) AS tag, + `host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, + guest_os_category.id guest_os_category_id, + guest_os_category.uuid guest_os_category_uuid, + guest_os_category.name guest_os_category_name, + mem_caps.used_capacity memory_used_capacity, + mem_caps.reserved_capacity memory_reserved_capacity, + cpu_caps.used_capacity cpu_used_capacity, + cpu_caps.reserved_capacity cpu_reserved_capacity, + async_job.id job_id, + async_job.uuid job_uuid, + async_job.job_status job_status, + async_job.account_id job_account_id, + oobm.enabled AS `oobm_enabled`, + oobm.power_state AS `oobm_power_state`, + ha_config.enabled AS `ha_enabled`, + ha_config.ha_state AS `ha_state`, + ha_config.provider AS `ha_provider`, + `last_annotation_view`.`annotation` AS `annotation`, + `last_annotation_view`.`created` AS `last_annotated`, + `user`.`username` AS `username` +FROM + `cloud`.`host` + LEFT JOIN + `cloud`.`cluster` ON host.cluster_id = cluster.id + LEFT JOIN + `cloud`.`data_center` ON host.data_center_id = data_center.id + LEFT JOIN + `cloud`.`host_pod_ref` ON host.pod_id = host_pod_ref.id + LEFT JOIN + `cloud`.`host_details` ON host.id = host_details.host_id + AND host_details.name = 'guest.os.category.id' + LEFT JOIN + `cloud`.`guest_os_category` ON guest_os_category.id = CONVERT ( host_details.value, UNSIGNED ) + LEFT JOIN + `cloud`.`host_tags` ON host_tags.host_id = host.id + LEFT JOIN + `cloud`.`op_host_capacity` mem_caps ON host.id = mem_caps.host_id + AND mem_caps.capacity_type = 0 + LEFT JOIN + `cloud`.`op_host_capacity` cpu_caps ON host.id = cpu_caps.host_id + AND cpu_caps.capacity_type = 1 + LEFT JOIN + `cloud`.`async_job` ON async_job.instance_id = host.id + AND async_job.instance_type = 'Host' + AND async_job.job_status = 0 + LEFT JOIN + `cloud`.`oobm` ON oobm.host_id = host.id + left join + `cloud`.`ha_config` ON ha_config.resource_id=host.id + and ha_config.resource_type='Host' + LEFT JOIN + `cloud`.`last_annotation_view` ON `last_annotation_view`.`entity_uuid` = `host`.`uuid` + LEFT JOIN + `cloud`.`user` ON `user`.`uuid` = `last_annotation_view`.`user_uuid` +GROUP BY + `host`.`id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql new file mode 100644 index 00000000000..e6cc9458208 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql @@ -0,0 +1,68 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`storage_pool_view`; + +DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; + +CREATE VIEW `cloud`.`storage_pool_view` AS +SELECT + `storage_pool`.`id` AS `id`, + `storage_pool`.`uuid` AS `uuid`, + `storage_pool`.`name` AS `name`, + `storage_pool`.`status` AS `status`, + `storage_pool`.`path` AS `path`, + `storage_pool`.`pool_type` AS `pool_type`, + `storage_pool`.`host_address` AS `host_address`, + `storage_pool`.`created` AS `created`, + `storage_pool`.`removed` AS `removed`, + `storage_pool`.`capacity_bytes` AS `capacity_bytes`, + `storage_pool`.`capacity_iops` AS `capacity_iops`, + `storage_pool`.`scope` AS `scope`, + `storage_pool`.`hypervisor` AS `hypervisor`, + `storage_pool`.`storage_provider_name` AS `storage_provider_name`, + `storage_pool`.`parent` AS `parent`, + `cluster`.`id` AS `cluster_id`, + `cluster`.`uuid` AS `cluster_uuid`, + `cluster`.`name` AS `cluster_name`, + `cluster`.`cluster_type` AS `cluster_type`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `data_center`.`networktype` AS `data_center_type`, + `host_pod_ref`.`id` AS `pod_id`, + `host_pod_ref`.`uuid` AS `pod_uuid`, + `host_pod_ref`.`name` AS `pod_name`, + `storage_pool_tags`.`tag` AS `tag`, + `storage_pool_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, + `op_host_capacity`.`used_capacity` AS `disk_used_capacity`, + `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id` +FROM + ((((((`cloud`.`storage_pool` + LEFT JOIN `cloud`.`cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`))) + LEFT JOIN `cloud`.`data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`))) + LEFT JOIN `cloud`.`host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`))) + LEFT JOIN `cloud`.`storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`)))) + LEFT JOIN `cloud`.`op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`) + AND (`op_host_capacity`.`capacity_type` IN (3 , 9))))) + LEFT JOIN `cloud`.`async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) + AND (`async_job`.`instance_type` = 'StoragePool') + AND (`async_job`.`job_status` = 0)))); diff --git a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java index 9ab010dd6a7..76bc5270b4f 100755 --- a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java +++ b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java @@ -1,61 +1,84 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.host; - -import com.cloud.service.ServiceOfferingVO; -import com.cloud.vm.VirtualMachine; -import java.util.Arrays; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import org.junit.Test; -import org.junit.Before; - -public class HostVOTest { - HostVO host; - ServiceOfferingVO offering; - - @Before - public void setUp() throws Exception { - host = new HostVO(); - offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, - false, "TestSO", false,VirtualMachine.Type.User,false); - } - - @Test - public void testNoSO() { - assertFalse(host.checkHostServiceOfferingTags(null)); - } - - @Test - public void testNoTag() { - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testRightTag() { - host.setHostTags(Arrays.asList("tag1","tag2")); - offering.setHostTag("tag2,tag1"); - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testWrongTag() { - host.setHostTags(Arrays.asList("tag1","tag2")); - offering.setHostTag("tag2,tag4"); - assertFalse(host.checkHostServiceOfferingTags(offering)); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import com.cloud.service.ServiceOfferingVO; +import com.cloud.vm.VirtualMachine; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import org.junit.Test; +import org.junit.Before; + +public class HostVOTest { + HostVO host; + ServiceOfferingVO offering; + + @Before + public void setUp() throws Exception { + host = new HostVO(); + offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, + false, "TestSO", false,VirtualMachine.Type.User,false); + } + + @Test + public void testNoSO() { + assertFalse(host.checkHostServiceOfferingTags(null)); + } + + @Test + public void testNoTag() { + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testRightTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag1"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testWrongTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag4"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatMatches() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("A"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatDoesNotMatch() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("B"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithNullServiceTag() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag(null); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } +} diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java index 8f1ee3e0450..bd05fbe3c4c 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java @@ -93,8 +93,8 @@ public class DatabaseAccessObjectTest { @Test public void generateIndexNameTest() { - String indexName = dao.generateIndexName("mytable","mycolumn"); - Assert.assertEquals( "i_mytable__mycolumn", indexName); + String indexName = dao.generateIndexName("mytable","mycolumn1", "mycolumn2"); + Assert.assertEquals( "i_mytable__mycolumn1__mycolumn2", indexName); } @Test @@ -136,10 +136,11 @@ public class DatabaseAccessObjectTest { Connection conn = connectionMock; String tableName = "mytable"; - String columnName = "mycolumn"; + String columnName1 = "mycolumn1"; + String columnName2 = "mycolumn2"; String indexName = "myindex"; - dao.createIndex(conn, tableName, columnName, indexName); + dao.createIndex(conn, tableName, indexName, columnName1, columnName2); verify(connectionMock, times(1)).prepareStatement(anyString()); verify(preparedStatementMock, times(1)).execute(); verify(preparedStatementMock, times(1)).close(); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index e450addb261..370753ed923 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -193,7 +193,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { destData.getType() == DataObjectType.TEMPLATE)) { // volume transfer from primary to secondary. Volume transfer between primary pools are already handled by copyVolumeBetweenPools // Delete cache in order to certainly transfer a latest image. - s_logger.debug("Delete " + cacheType + " cache(id: " + cacheId + + if (s_logger.isDebugEnabled()) s_logger.debug("Delete " + cacheType + " cache(id: " + cacheId + ", uuid: " + cacheUuid + ")"); cacheMgr.deleteCacheObject(srcForCopy); } else { @@ -205,7 +205,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { ", uuid: " + cacheUuid + ")"); cacheMgr.deleteCacheObject(srcForCopy); } else { - s_logger.debug("Decrease reference count of " + cacheType + + if (s_logger.isDebugEnabled()) s_logger.debug("Decrease reference count of " + cacheType + " cache(id: " + cacheId + ", uuid: " + cacheUuid + ")"); cacheMgr.releaseCacheObject(srcForCopy); } @@ -213,7 +213,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } return answer; } catch (Exception e) { - s_logger.debug("copy object failed: ", e); + if (s_logger.isDebugEnabled()) s_logger.debug("copy object failed: ", e); if (cacheData != null) { cacheMgr.deleteCacheObject(cacheData); } @@ -331,7 +331,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } return answer; } catch (Exception e) { - s_logger.debug("Failed to send to storage pool", e); + if (s_logger.isDebugEnabled()) s_logger.debug("Failed to send to storage pool", e); throw new CloudRuntimeException("Failed to send to storage pool", e); } } @@ -388,7 +388,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { if (answer == null || !answer.getResult()) { if (answer != null) { - s_logger.debug("copy to image store failed: " + answer.getDetails()); + if (s_logger.isDebugEnabled()) s_logger.debug("copy to image store failed: " + answer.getDetails()); } objOnImageStore.processEvent(Event.OperationFailed); imageStore.delete(objOnImageStore); @@ -411,7 +411,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { if (answer == null || !answer.getResult()) { if (answer != null) { - s_logger.debug("copy to primary store failed: " + answer.getDetails()); + if (s_logger.isDebugEnabled()) s_logger.debug("copy to primary store failed: " + answer.getDetails()); } objOnImageStore.processEvent(Event.OperationFailed); imageStore.delete(objOnImageStore); @@ -471,13 +471,17 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { s_logger.error(errMsg); answer = new Answer(command, false, errMsg); } else { + if (s_logger.isDebugEnabled()) s_logger.debug("Sending MIGRATE_COPY request to node " + ep); answer = ep.sendMessage(command); + if (s_logger.isDebugEnabled()) s_logger.debug("Received MIGRATE_COPY response from node with answer: " + answer); } if (answer == null || !answer.getResult()) { throw new CloudRuntimeException("Failed to migrate volume " + volume + " to storage pool " + destPool); } else { // Update the volume details after migration. + if (s_logger.isDebugEnabled()) s_logger.debug("MIGRATE_COPY updating volume"); + VolumeVO volumeVo = volDao.findById(volume.getId()); Long oldPoolId = volume.getPoolId(); volumeVo.setPath(((MigrateVolumeAnswer)answer).getVolumePath()); @@ -496,6 +500,8 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } volumeVo.setFolder(folder); volDao.update(volume.getId(), volumeVo); + if (s_logger.isDebugEnabled()) s_logger.debug("MIGRATE_COPY update volume data complete"); + } return answer; @@ -507,7 +513,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { Answer answer = null; String errMsg = null; try { - s_logger.debug("copyAsync inspecting src type " + srcData.getType().toString() + " copyAsync inspecting dest type " + destData.getType().toString()); + if (s_logger.isDebugEnabled()) s_logger.debug("copyAsync inspecting src type " + srcData.getType().toString() + " copyAsync inspecting dest type " + destData.getType().toString()); if (srcData.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.VOLUME) { answer = copyVolumeFromSnapshot(srcData, destData); } else if (srcData.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.TEMPLATE) { @@ -516,11 +522,16 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { answer = cloneVolume(srcData, destData); } else if (destData.getType() == DataObjectType.VOLUME && srcData.getType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary && destData.getDataStore().getRole() == DataStoreRole.Primary) { + if (s_logger.isDebugEnabled()) s_logger.debug("About to MIGRATE copy between datasources"); if (srcData.getId() == destData.getId()) { // The volume has to be migrated across storage pools. + if (s_logger.isDebugEnabled()) s_logger.debug("MIGRATE copy using migrateVolumeToPool STARTING"); answer = migrateVolumeToPool(srcData, destData); + if (s_logger.isDebugEnabled()) s_logger.debug("MIGRATE copy using migrateVolumeToPool DONE: " + answer.getResult()); } else { + if (s_logger.isDebugEnabled()) s_logger.debug("MIGRATE copy using copyVolumeBetweenPools STARTING"); answer = copyVolumeBetweenPools(srcData, destData); + if (s_logger.isDebugEnabled()) s_logger.debug("MIGRATE copy using copyVolumeBetweenPools DONE: " + answer.getResult()); } } else if (srcData.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.SNAPSHOT) { answer = copySnapshot(srcData, destData); @@ -532,7 +543,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { errMsg = answer.getDetails(); } } catch (Exception e) { - s_logger.debug("copy failed", e); + if (s_logger.isDebugEnabled()) s_logger.debug("copy failed", e); errMsg = e.toString(); } CopyCommandResult result = new CopyCommandResult(null, answer); @@ -627,7 +638,7 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } return answer; } catch (Exception e) { - s_logger.debug("copy snasphot failed: ", e); + if (s_logger.isDebugEnabled()) s_logger.debug("copy snasphot failed: ", e); if (cacheData != null) { cacheMgr.deleteCacheObject(cacheData); } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 1419ae36d25..a93f624aa53 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import com.cloud.agent.api.PrepareForMigrationAnswer; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -106,6 +107,7 @@ import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; @@ -186,6 +188,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { private EndPointSelector selector; @Inject VMTemplatePoolDao templatePoolDao; + @Inject + private VolumeDataFactory _volFactory; @Override public StrategyPriority canHandle(DataObject srcData, DataObject destData) { @@ -400,15 +404,15 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } else if (!isVolumeOnManagedStorage(destVolumeInfo)) { handleVolumeMigrationFromManagedStorageToNonManagedStorage(srcVolumeInfo, destVolumeInfo, callback); } else { - String errMsg = "The source volume to migrate and the destination volume are both on managed storage. " + - "Migration in this case is not yet supported."; - - handleError(errMsg, callback); + handleVolumeMigrationFromManagedStorageToManagedStorage(srcVolumeInfo, destVolumeInfo, callback); } } else if (!isVolumeOnManagedStorage(destVolumeInfo)) { - String errMsg = "The 'StorageSystemDataMotionStrategy' does not support this migration use case."; - - handleError(errMsg, callback); + if (!HypervisorType.KVM.equals(srcVolumeInfo.getHypervisorType())) { + String errMsg = String.format("Currently migrating volumes between managed storage providers is not supported on %s hypervisor", srcVolumeInfo.getHypervisorType().toString()); + handleError(errMsg, callback); + } else { + handleVolumeMigrationForKVM(srcVolumeInfo, destVolumeInfo, callback); + } } else { handleVolumeMigrationFromNonManagedStorageToManagedStorage(srcVolumeInfo, destVolumeInfo, callback); } @@ -453,7 +457,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { String volumePath = null; try { - if (!ImageFormat.QCOW2.equals(srcVolumeInfo.getFormat())) { + if (!HypervisorType.KVM.equals(srcVolumeInfo.getHypervisorType())) { throw new CloudRuntimeException("Currently, only the KVM hypervisor type is supported for the migration of a volume " + "from managed storage to non-managed storage."); } @@ -485,7 +489,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { errMsg = "Migration operation failed in 'StorageSystemDataMotionStrategy.handleVolumeCopyFromManagedStorageToSecondaryStorage': " + ex.getMessage(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { CopyCmdAnswer copyCmdAnswer; @@ -512,12 +516,22 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } } + private void handleVolumeMigrationFromManagedStorageToManagedStorage(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, + AsyncCompletionCallback callback) { + if (!HypervisorType.KVM.equals(srcVolumeInfo.getHypervisorType())) { + String errMsg = String.format("Currently migrating volumes between managed storage providers is not supported on %s hypervisor", srcVolumeInfo.getHypervisorType().toString()); + handleError(errMsg, callback); + } else { + handleVolumeMigrationForKVM(srcVolumeInfo, destVolumeInfo, callback); + } + } + private void handleVolumeMigrationFromManagedStorageToNonManagedStorage(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, AsyncCompletionCallback callback) { String errMsg = null; try { - if (!ImageFormat.QCOW2.equals(srcVolumeInfo.getFormat())) { + if (!HypervisorType.KVM.equals(srcVolumeInfo.getHypervisorType())) { throw new CloudRuntimeException("Currently, only the KVM hypervisor type is supported for the migration of a volume " + "from managed storage to non-managed storage."); } @@ -525,10 +539,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { HypervisorType hypervisorType = HypervisorType.KVM; VirtualMachine vm = srcVolumeInfo.getAttachedVM(); - if (vm != null && vm.getState() != VirtualMachine.State.Stopped) { - throw new CloudRuntimeException("Currently, if a volume to migrate from managed storage to non-managed storage is attached to " + - "a VM, the VM must be in the Stopped state."); - } + checkAvailableForMigration(vm); long destStoragePoolId = destVolumeInfo.getPoolId(); StoragePoolVO destStoragePoolVO = _storagePoolDao.findById(destStoragePoolId); @@ -553,7 +564,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { errMsg = "Migration operation failed in 'StorageSystemDataMotionStrategy.handleVolumeMigrationFromManagedStorageToNonManagedStorage': " + ex.getMessage(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { CopyCmdAnswer copyCmdAnswer; @@ -579,9 +590,10 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { private void verifyFormatWithPoolType(ImageFormat imageFormat, StoragePoolType poolType) { if (imageFormat != ImageFormat.VHD && imageFormat != ImageFormat.OVA && imageFormat != ImageFormat.QCOW2 && - !(imageFormat == ImageFormat.RAW && StoragePoolType.PowerFlex == poolType)) { - throw new CloudRuntimeException("Only the following image types are currently supported: " + - ImageFormat.VHD.toString() + ", " + ImageFormat.OVA.toString() + ", " + ImageFormat.QCOW2.toString() + ", and " + ImageFormat.RAW.toString() + "(for PowerFlex)"); + !(imageFormat == ImageFormat.RAW && (StoragePoolType.PowerFlex == poolType || + StoragePoolType.FiberChannel == poolType))) { + throw new CloudRuntimeException(String.format("Only the following image types are currently supported: %s, %s, %s, %s (for PowerFlex and FiberChannel)", + ImageFormat.VHD.toString(), ImageFormat.OVA.toString(), ImageFormat.QCOW2.toString(), ImageFormat.RAW.toString())); } } @@ -685,14 +697,14 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { handleVolumeMigrationForXenServer(srcVolumeInfo, destVolumeInfo); } else { - handleVolumeMigrationForKVM(srcVolumeInfo, destVolumeInfo); + handleVolumeMigrationForKVM(srcVolumeInfo, destVolumeInfo, callback); } } catch (Exception ex) { errMsg = "Migration operation failed in 'StorageSystemDataMotionStrategy.handleVolumeMigrationFromNonManagedStorageToManagedStorage': " + ex.getMessage(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { CopyCmdAnswer copyCmdAnswer; @@ -826,24 +838,73 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { _volumeDao.update(srcVolumeInfo.getId(), volumeVO); } - private void handleVolumeMigrationForKVM(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { + private void handleVolumeMigrationForKVM(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, AsyncCompletionCallback callback) { VirtualMachine vm = srcVolumeInfo.getAttachedVM(); - if (vm != null && vm.getState() != VirtualMachine.State.Stopped) { - throw new CloudRuntimeException("Currently, if a volume to migrate from non-managed storage to managed storage on KVM is attached to " + - "a VM, the VM must be in the Stopped state."); + checkAvailableForMigration(vm); + + String errMsg = null; + try { + destVolumeInfo.getDataStore().getDriver().createAsync(destVolumeInfo.getDataStore(), destVolumeInfo, null); + VolumeVO volumeVO = _volumeDao.findById(destVolumeInfo.getId()); + updatePathFromScsiName(volumeVO); + destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); + HostVO hostVO = getHostOnWhichToExecuteMigrationCommand(srcVolumeInfo, destVolumeInfo); + + // migrate the volume via the hypervisor + String path = migrateVolumeForKVM(srcVolumeInfo, destVolumeInfo, hostVO, "Unable to migrate the volume from non-managed storage to managed storage"); + + updateVolumePath(destVolumeInfo.getId(), path); + volumeVO = _volumeDao.findById(destVolumeInfo.getId()); + // only set this if it was not set. default to QCOW2 for KVM + if (volumeVO.getFormat() == null) { + volumeVO.setFormat(ImageFormat.QCOW2); + _volumeDao.update(volumeVO.getId(), volumeVO); + } + } catch (Exception ex) { + errMsg = "Primary storage migration failed due to an unexpected error: " + + ex.getMessage(); + if (ex instanceof CloudRuntimeException) { + throw ex; + } else { + throw new CloudRuntimeException(errMsg, ex); + } + } finally { + CopyCmdAnswer copyCmdAnswer; + if (errMsg != null) { + copyCmdAnswer = new CopyCmdAnswer(errMsg); + } + else { + destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); + DataTO dataTO = destVolumeInfo.getTO(); + copyCmdAnswer = new CopyCmdAnswer(dataTO); + } + + CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); + result.setResult(errMsg); + callback.complete(result); } + } - destVolumeInfo.getDataStore().getDriver().createAsync(destVolumeInfo.getDataStore(), destVolumeInfo, null); + private void checkAvailableForMigration(VirtualMachine vm) { + if (vm != null && (vm.getState() != VirtualMachine.State.Stopped && vm.getState() != VirtualMachine.State.Migrating)) { + throw new CloudRuntimeException("Currently, if a volume to migrate from non-managed storage to managed storage on KVM is attached to " + + "a VM, the VM must be in the Stopped or Migrating state."); + } + } - VolumeVO volumeVO = _volumeDao.findById(destVolumeInfo.getId()); - - volumeVO.setPath(volumeVO.get_iScsiName()); - - _volumeDao.update(volumeVO.getId(), volumeVO); - - destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId(), destVolumeInfo.getDataStore()); + /** + * Only update the path from the iscsiName if the iscsiName is set. Otherwise take no action to avoid nullifying the path + * with a previously set path value. + */ + private void updatePathFromScsiName(VolumeVO volumeVO) { + if (volumeVO.get_iScsiName() != null) { + volumeVO.setPath(volumeVO.get_iScsiName()); + _volumeDao.update(volumeVO.getId(), volumeVO); + } + } + private HostVO getHostOnWhichToExecuteMigrationCommand(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { long srcStoragePoolId = srcVolumeInfo.getPoolId(); StoragePoolVO srcStoragePoolVO = _storagePoolDao.findById(srcStoragePoolId); @@ -856,14 +917,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { hostVO = getHost(destVolumeInfo.getDataCenterId(), HypervisorType.KVM, false); } - // migrate the volume via the hypervisor - migrateVolumeForKVM(srcVolumeInfo, destVolumeInfo, hostVO, "Unable to migrate the volume from non-managed storage to managed storage"); - - volumeVO = _volumeDao.findById(destVolumeInfo.getId()); - - volumeVO.setFormat(ImageFormat.QCOW2); - - _volumeDao.update(volumeVO.getId(), volumeVO); + return hostVO; } /** @@ -1075,7 +1129,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { catch (Exception ex) { errMsg = ex.getMessage(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { if (usingBackendSnapshot) { @@ -1293,7 +1347,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { catch (Exception ex) { errMsg = "Copy operation failed in 'StorageSystemDataMotionStrategy.handleCreateManagedVolumeFromNonManagedSnapshot': " + ex.getMessage(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { handleQualityOfServiceForVolumeMigration(volumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.NO_MIGRATION); @@ -1674,6 +1728,42 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { return copyCmdAnswer; } + /** + * Use normal volume semantics (create a volume known to cloudstack, ask the storage driver to create it as a copy of the snapshot) + + * @param volumeVO + * @param snapshotInfo + */ + public void prepTempVolumeForCopyFromSnapshot(SnapshotInfo snapshotInfo) { + VolumeVO volumeVO = null; + try { + volumeVO = new VolumeVO(Volume.Type.DATADISK, snapshotInfo.getName() + "_" + System.currentTimeMillis() + ".TMP", + snapshotInfo.getDataCenterId(), snapshotInfo.getDomainId(), snapshotInfo.getAccountId(), 0, ProvisioningType.THIN, snapshotInfo.getSize(), 0L, 0L, ""); + volumeVO.setPoolId(snapshotInfo.getDataStore().getId()); + _volumeDao.persist(volumeVO); + VolumeInfo tempVolumeInfo = this._volFactory.getVolume(volumeVO.getId()); + + if (snapshotInfo.getDataStore().getDriver().canCopy(snapshotInfo, tempVolumeInfo)) { + snapshotInfo.getDataStore().getDriver().copyAsync(snapshotInfo, tempVolumeInfo, null, null); + // refresh volume info as data could have changed + tempVolumeInfo = this._volFactory.getVolume(volumeVO.getId()); + // save the "temp" volume info into the snapshot details (we need this to clean up at the end) + _snapshotDetailsDao.addDetail(snapshotInfo.getId(), "TemporaryVolumeCopyUUID", tempVolumeInfo.getUuid(), true); + _snapshotDetailsDao.addDetail(snapshotInfo.getId(), "TemporaryVolumeCopyPath", tempVolumeInfo.getPath(), true); + // NOTE: for this to work, the Driver must return a custom SnapshotObjectTO object from getTO() + // whenever the TemporaryVolumeCopyPath is set. + } else { + throw new CloudRuntimeException("Storage driver indicated it could create a volume from the snapshot but rejected the subsequent request to do so"); + } + } catch (Throwable e) { + // cleanup temporary volume + if (volumeVO != null) { + _volumeDao.remove(volumeVO.getId()); + } + throw e; + } + } + /** * If the underlying storage system is making use of read-only snapshots, this gives the storage system the opportunity to * create a volume from the snapshot so that we can copy the VHD file that should be inside of the snapshot to secondary storage. @@ -1685,8 +1775,13 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { * resign the SR and the VDI that should be inside of the snapshot before copying the VHD file to secondary storage. */ private void createVolumeFromSnapshot(SnapshotInfo snapshotInfo) { - SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "create"); + if ("true".equalsIgnoreCase(snapshotInfo.getDataStore().getDriver().getCapabilities().get("CAN_CREATE_TEMP_VOLUME_FROM_SNAPSHOT"))) { + prepTempVolumeForCopyFromSnapshot(snapshotInfo); + return; + } + + SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "create"); try { snapshotInfo.getDataStore().getDriver().createAsync(snapshotInfo.getDataStore(), snapshotInfo, null); } @@ -1701,6 +1796,24 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { * invocation of createVolumeFromSnapshot(SnapshotInfo). */ private void deleteVolumeFromSnapshot(SnapshotInfo snapshotInfo) { + VolumeVO volumeVO = null; + // cleanup any temporary volume previously created for copy from a snapshot + if ("true".equalsIgnoreCase(snapshotInfo.getDataStore().getDriver().getCapabilities().get("CAN_CREATE_TEMP_VOLUME_FROM_SNAPSHOT"))) { + SnapshotDetailsVO tempUuid = null; + tempUuid = _snapshotDetailsDao.findDetail(snapshotInfo.getId(), "TemporaryVolumeCopyUUID"); + if (tempUuid == null || tempUuid.getValue() == null) { + return; + } + + volumeVO = _volumeDao.findByUuid(tempUuid.getValue()); + if (volumeVO != null) { + _volumeDao.remove(volumeVO.getId()); + } + _snapshotDetailsDao.remove(tempUuid.getId()); + _snapshotDetailsDao.removeDetail(snapshotInfo.getId(), "TemporaryVolumeCopyUUID"); + return; + } + SnapshotDetailsVO snapshotDetails = handleSnapshotDetails(snapshotInfo.getId(), "delete"); try { @@ -1884,9 +1997,10 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + Answer pfma; try { - Answer pfma = agentManager.send(destHost.getId(), pfmc); + pfma = agentManager.send(destHost.getId(), pfmc); if (pfma == null || !pfma.getResult()) { String details = pfma != null ? pfma.getDetails() : "null answer returned"; @@ -1894,8 +2008,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { throw new AgentUnavailableException(msg, destHost.getId()); } - } - catch (final OperationTimedoutException e) { + } catch (final OperationTimedoutException e) { throw new AgentUnavailableException("Operation timed out", destHost.getId()); } @@ -1911,6 +2024,12 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { migrateCommand.setMigrateStorageManaged(managedStorageDestination); migrateCommand.setMigrateNonSharedInc(migrateNonSharedInc); + Integer newVmCpuShares = ((PrepareForMigrationAnswer) pfma).getNewVmCpuShares(); + if (newVmCpuShares != null) { + LOGGER.debug(String.format("Setting CPU shares to [%d] as part of migrate VM with volumes command for VM [%s].", newVmCpuShares, vmTO)); + migrateCommand.setNewVmCpuShares(newVmCpuShares); + } + boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); migrateCommand.setAutoConvergence(kvmAutoConvergence); @@ -2363,7 +2482,10 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { try { StoragePoolVO storagePoolVO = _storagePoolDao.findById(volumeInfo.getPoolId()); - if (!ImageFormat.QCOW2.equals(volumeInfo.getFormat()) && !(ImageFormat.RAW.equals(volumeInfo.getFormat()) && StoragePoolType.PowerFlex == storagePoolVO.getPoolType())) { + if (!ImageFormat.QCOW2.equals(volumeInfo.getFormat()) && + !(ImageFormat.RAW.equals(volumeInfo.getFormat()) && ( + StoragePoolType.PowerFlex == storagePoolVO.getPoolType() || + StoragePoolType.FiberChannel == storagePoolVO.getPoolType()))) { throw new CloudRuntimeException("When using managed storage, you can only create a template from a volume on KVM currently."); } @@ -2506,7 +2628,13 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { long snapshotId = snapshotInfo.getId(); - if (storagePoolVO.getPoolType() == StoragePoolType.PowerFlex) { + // if the snapshot required a temporary volume be created check if the UUID is set so we can + // retrieve the temporary volume's path to use during remote copy + List storedDetails = _snapshotDetailsDao.findDetails(snapshotInfo.getId(), "TemporaryVolumeCopyPath"); + if (storedDetails != null && storedDetails.size() > 0) { + String value = storedDetails.get(0).getValue(); + snapshotDetails.put(DiskTO.PATH, value); + } else if (storagePoolVO.getPoolType() == StoragePoolType.PowerFlex || storagePoolVO.getPoolType() == StoragePoolType.FiberChannel) { snapshotDetails.put(DiskTO.IQN, snapshotInfo.getPath()); } else { snapshotDetails.put(DiskTO.IQN, getSnapshotProperty(snapshotId, DiskTO.IQN)); @@ -2718,8 +2846,6 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { } private String migrateVolumeForKVM(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, HostVO hostVO, String errMsg) { - boolean srcVolumeDetached = srcVolumeInfo.getAttachedVM() == null; - try { Map srcDetails = getVolumeDetails(srcVolumeInfo); Map destDetails = getVolumeDetails(destVolumeInfo); @@ -2727,16 +2853,11 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { MigrateVolumeCommand migrateVolumeCommand = new MigrateVolumeCommand(srcVolumeInfo.getTO(), destVolumeInfo.getTO(), srcDetails, destDetails, StorageManager.KvmStorageOfflineMigrationWait.value()); - if (srcVolumeDetached) { - _volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); - } - + _volumeService.grantAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); handleQualityOfServiceForVolumeMigration(destVolumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.MIGRATION); - _volumeService.grantAccess(destVolumeInfo, hostVO, destVolumeInfo.getDataStore()); MigrateVolumeAnswer migrateVolumeAnswer = (MigrateVolumeAnswer)agentManager.send(hostVO.getId(), migrateVolumeCommand); - if (migrateVolumeAnswer == null || !migrateVolumeAnswer.getResult()) { if (migrateVolumeAnswer != null && StringUtils.isNotEmpty(migrateVolumeAnswer.getDetails())) { throw new CloudRuntimeException(migrateVolumeAnswer.getDetails()); @@ -2745,42 +2866,22 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException(errMsg); } } - - if (srcVolumeDetached) { - _volumeService.revokeAccess(destVolumeInfo, hostVO, destVolumeInfo.getDataStore()); - } - - try { - _volumeService.revokeAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); - } - catch (Exception e) { - // This volume should be deleted soon, so just log a warning here. - LOGGER.warn(e.getMessage(), e); - } - return migrateVolumeAnswer.getVolumePath(); - } - catch (Exception ex) { + } catch (CloudRuntimeException ex) { + throw ex; + } catch (Exception ex) { + throw new CloudRuntimeException("Unexpected error during volume migration: " + ex.getMessage(), ex); + } finally { try { - _volumeService.revokeAccess(destVolumeInfo, hostVO, destVolumeInfo.getDataStore()); - } - catch (Exception e) { - // This volume should be deleted soon, so just log a warning here. - LOGGER.warn(e.getMessage(), e); - } - - if (srcVolumeDetached) { _volumeService.revokeAccess(srcVolumeInfo, hostVO, srcVolumeInfo.getDataStore()); + _volumeService.revokeAccess(destVolumeInfo, hostVO, destVolumeInfo.getDataStore()); + handleQualityOfServiceForVolumeMigration(destVolumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.NO_MIGRATION); + } catch (Throwable e) { + LOGGER.warn("During cleanup post-migration and exception occured: " + e); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Exception during post-migration cleanup.", e); + } } - - String msg = "Failed to perform volume migration : "; - - LOGGER.warn(msg, ex); - - throw new CloudRuntimeException(msg + ex.getMessage(), ex); - } - finally { - handleQualityOfServiceForVolumeMigration(destVolumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.NO_MIGRATION); } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index 6d25c481537..5bb0d19be74 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -94,7 +94,7 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, @Override public ImageStoreEntity getImageStore(String uuid) { ImageStoreVO dataStore = dataStoreDao.findByUuid(uuid); - return getImageStore(dataStore.getId()); + return dataStore == null ? null : getImageStore(dataStore.getId()); } @Override @@ -248,6 +248,16 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, return stores; } + @Override + public List listImageStoresFilteringByZoneIds(Long... zoneIds) { + List stores = dataStoreDao.listImageStoresByZoneIds(zoneIds); + List imageStores = new ArrayList<>(); + for (ImageStoreVO store : stores) { + imageStores.add(getImageStore(store.getId())); + } + return imageStores; + } + @Override public String getConfigComponentName() { return ImageStoreProviderManager.class.getSimpleName(); diff --git a/engine/storage/integration-test/src/test/resources/storageContext.xml b/engine/storage/integration-test/src/test/resources/storageContext.xml index fc24753127e..7c95345f673 100644 --- a/engine/storage/integration-test/src/test/resources/storageContext.xml +++ b/engine/storage/integration-test/src/test/resources/storageContext.xml @@ -87,4 +87,8 @@ + + + + diff --git a/engine/storage/object/pom.xml b/engine/storage/object/pom.xml new file mode 100644 index 00000000000..5bad4e8a3d6 --- /dev/null +++ b/engine/storage/object/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + cloud-engine-storage-object + Apache CloudStack Engine Storage Object Component + + org.apache.cloudstack + cloud-engine + 4.19.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java new file mode 100644 index 00000000000..418121503c1 --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataTO; +import org.apache.cloudstack.engine.subsystem.api.storage.BucketInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; + +import java.util.Date; + +public class BucketObject implements BucketInfo { + + private String name; + + @Override + public long getDomainId() { + return 0; + } + + @Override + public long getAccountId() { + return 0; + } + + @Override + public Class getEntityType() { + return null; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public void addPayload(Object data) { + + } + + @Override + public Object getPayload() { + return null; + } + + @Override + public Bucket getBucket() { + return null; + } + + @Override + public long getId() { + return 0; + } + + @Override + public String getUri() { + return null; + } + + @Override + public DataTO getTO() { + return null; + } + + @Override + public DataStore getDataStore() { + return null; + } + + @Override + public Long getSize() { + return null; + } + + @Override + public Integer getQuota() { + return null; + } + + @Override + public boolean isVersioning() { + return false; + } + + @Override + public boolean isEncryption() { + return false; + } + + @Override + public boolean isObjectLock() { + return false; + } + + @Override + public String getPolicy() { + return null; + } + + @Override + public String getBucketURL() { + return null; + } + + @Override + public String getAccessKey() { + return null; + } + + @Override + public String getSecretKey() { + return null; + } + + @Override + public long getPhysicalSize() { + return 0; + } + + @Override + public DataObjectType getType() { + return null; + } + + @Override + public String getUuid() { + return null; + } + + @Override + public boolean delete() { + return false; + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event) { + + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answer) { + + } + + @Override + public void incRefCount() { + + } + + @Override + public void decRefCount() { + + } + + @Override + public Long getRefCount() { + return null; + } + + @Override + public long getObjectStoreId() { + return 0; + } + + @Override + public Date getCreated() { + return null; + } + + @Override + public State getState() { + return null; + } +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java new file mode 100644 index 00000000000..a0db89bad4e --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStorageService; +import org.apache.log4j.Logger; + +public class ObjectStorageServiceImpl implements ObjectStorageService { + + private static final Logger s_logger = Logger.getLogger(ObjectStorageServiceImpl.class); + + +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java new file mode 100644 index 00000000000..40f503692e1 --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.manager; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.ObjectStoreImpl; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreProviderManagerImpl implements ObjectStoreProviderManager, Configurable { + private static final Logger s_logger = Logger.getLogger(ObjectStoreProviderManagerImpl.class); + @Inject + ObjectStoreDao objectStoreDao; + + @Inject + DataStoreProviderManager providerManager; + + Map driverMaps; + + @PostConstruct + public void config() { + driverMaps = new HashMap(); + } + + @Override + public ObjectStoreEntity getObjectStore(long objectStoreId) { + ObjectStoreVO objectStore = objectStoreDao.findById(objectStoreId); + String providerName = objectStore.getProviderName(); + ObjectStoreProvider provider = (ObjectStoreProvider)providerManager.getDataStoreProvider(providerName); + ObjectStoreEntity objStore = ObjectStoreImpl.getDataStore(objectStore, driverMaps.get(provider.getName()), provider); + return objStore; + } + + @Override + public boolean registerDriver(String providerName, ObjectStoreDriver driver) { + if (driverMaps.containsKey(providerName)) { + return false; + } + driverMaps.put(providerName, driver); + return true; + } + + @Override + public ObjectStoreEntity getObjectStore(String uuid) { + ObjectStoreVO objectStore = objectStoreDao.findByUuid(uuid); + return getObjectStore(objectStore.getId()); + } + + @Override + public List listObjectStores() { + List stores = objectStoreDao.listObjectStores(); + List ObjectStores = new ArrayList(); + for (ObjectStoreVO store : stores) { + ObjectStores.add(getObjectStore(store.getId())); + } + return ObjectStores; + } + + @Override + public List listObjectStoreByProvider(String provider) { + List stores = objectStoreDao.findByProvider(provider); + List ObjectStores = new ArrayList(); + for (ObjectStoreVO store : stores) { + ObjectStores.add(getObjectStore(store.getId())); + } + return ObjectStores; + } + + @Override + public String getConfigComponentName() { + return ObjectStoreProviderManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { }; + } +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java new file mode 100644 index 00000000000..825b349bdfc --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.store; + +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.DataStoreRole; +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.log4j.Logger; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class ObjectStoreImpl implements ObjectStoreEntity { + private static final Logger s_logger = Logger.getLogger(ObjectStoreImpl.class); + + protected ObjectStoreDriver driver; + protected ObjectStoreVO objectStoreVO; + protected ObjectStoreProvider provider; + + public ObjectStoreImpl() { + super(); + } + + protected void configure(ObjectStoreVO objectStoreVO, ObjectStoreDriver objectStoreDriver, ObjectStoreProvider provider) { + this.driver = objectStoreDriver; + this.objectStoreVO = objectStoreVO; + this.provider = provider; + } + + public static ObjectStoreEntity getDataStore(ObjectStoreVO objectStoreVO, ObjectStoreDriver objectStoreDriver, ObjectStoreProvider provider) { + ObjectStoreImpl instance = ComponentContext.inject(ObjectStoreImpl.class); + instance.configure(objectStoreVO, objectStoreDriver, provider); + return instance; + } + + @Override + public DataStoreDriver getDriver() { + return this.driver; + } + + @Override + public DataStoreRole getRole() { + return null; + } + + @Override + public long getId() { + return this.objectStoreVO.getId(); + } + + @Override + public String getUri() { + return this.objectStoreVO.getUrl(); + } + + @Override + public Scope getScope() { + return null; + } + + + @Override + public String getUuid() { + return this.objectStoreVO.getUuid(); + } + + public Date getCreated() { + return this.objectStoreVO.getCreated(); + } + + @Override + public String getName() { + return objectStoreVO.getName(); + } + + @Override + public DataObject create(DataObject obj) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + return driver.createBucket(bucket, objectLock); + } + + @Override + public boolean deleteBucket(String bucketName) { + return driver.deleteBucket(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean setBucketEncryption(String bucketName) { + return driver.setBucketEncryption(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean deleteBucketEncryption(String bucketName) { + return driver.deleteBucketEncryption(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean setBucketVersioning(String bucketName) { + return driver.setBucketVersioning(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean deleteBucketVersioning(String bucketName) { + return driver.deleteBucketVersioning(bucketName, objectStoreVO.getId()); + } + + @Override + public void setBucketPolicy(String bucketName, String policy) { + driver.setBucketPolicy(bucketName, policy, objectStoreVO.getId()); + } + + @Override + public void setQuota(String bucketName, int quota) { + driver.setBucketQuota(bucketName, objectStoreVO.getId(), quota); + } + + @Override + public Map getAllBucketsUsage() { + return driver.getAllBucketsUsage(objectStoreVO.getId()); + } + + @Override + public List listBuckets() { + return driver.listBuckets(objectStoreVO.getId()); + } + + /* + Create user if not exists + */ + @Override + public boolean createUser(long accountId) { + return driver.createUser(accountId, objectStoreVO.getId()); + } + + @Override + public boolean delete(DataObject obj) { + return false; + } + + @Override + public DataStoreTO getTO() { + return null; + } + + @Override + public String getProviderName() { + return objectStoreVO.getProviderName(); + } + + @Override + public String getUrl() { + return objectStoreVO.getUrl(); + } + +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java new file mode 100644 index 00000000000..ff88262c4ba --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.store.lifecycle; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; + +public interface ObjectStoreLifeCycle extends DataStoreLifeCycle { +} diff --git a/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml b/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml new file mode 100644 index 00000000000..57bd9f87749 --- /dev/null +++ b/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml @@ -0,0 +1,36 @@ + + + + + + + + diff --git a/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java b/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java new file mode 100644 index 00000000000..392388ca8b8 --- /dev/null +++ b/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.object.manager; + +import junit.framework.TestCase; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.cloudstack.storage.object.store.ObjectStoreImpl; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ObjectStoreProviderManagerImplTest extends TestCase{ + + ObjectStoreProviderManagerImpl objectStoreProviderManagerImplSpy; + + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + ObjectStoreVO objectStoreVO; + + @Mock + DataStoreProviderManager providerManager; + + @Mock + ObjectStoreProvider provider; + + @Mock + Map driverMaps; + + @Mock + ObjectStoreDriver objectStoreDriver; + + @Mock + ObjectStoreEntity objectStoreEntity; + + MockedStatic mockObjectStoreImpl; + + @Before + public void setup(){ + objectStoreProviderManagerImplSpy = Mockito.spy(new ObjectStoreProviderManagerImpl()); + objectStoreProviderManagerImplSpy.objectStoreDao = objectStoreDao; + objectStoreProviderManagerImplSpy.providerManager = providerManager; + objectStoreProviderManagerImplSpy.driverMaps = driverMaps; + mockObjectStoreImpl = mockStatic(ObjectStoreImpl.class); + } + + @Test + public void getObjectStoreTest() { + Mockito.doReturn(objectStoreVO).when(objectStoreDao).findById(Mockito.anyLong()); + Mockito.doReturn(provider).when(providerManager).getDataStoreProvider(Mockito.anyString()); + Mockito.doReturn(objectStoreDriver).when(driverMaps).get(Mockito.anyString()); + Mockito.doReturn("Simulator").when(provider).getName(); + Mockito.doReturn("Simulator").when(objectStoreVO).getProviderName(); + + when(ObjectStoreImpl.getDataStore(Mockito.any(ObjectStoreVO.class), Mockito.any(ObjectStoreDriver.class), + Mockito.any(ObjectStoreProvider.class))).thenReturn(objectStoreEntity); + assertNotNull(objectStoreProviderManagerImplSpy.getObjectStore(1L)); + } + + @Test + public void listObjectStoresTest() { + List stores = new ArrayList<>(); + stores.add(objectStoreVO); + Mockito.doReturn(objectStoreVO).when(objectStoreDao).findById(Mockito.anyLong()); + Mockito.doReturn(provider).when(providerManager).getDataStoreProvider(Mockito.anyString()); + Mockito.doReturn(objectStoreDriver).when(driverMaps).get(Mockito.anyString()); + Mockito.doReturn("Simulator").when(provider).getName(); + Mockito.doReturn("Simulator").when(objectStoreVO).getProviderName(); + when(ObjectStoreImpl.getDataStore(Mockito.any(ObjectStoreVO.class), Mockito.any(ObjectStoreDriver.class), + Mockito.any(ObjectStoreProvider.class))).thenReturn(objectStoreEntity); + Mockito.doReturn(stores).when(objectStoreDao).listObjectStores(); + assertEquals(1, objectStoreProviderManagerImplSpy.listObjectStores().size()); + } + + @Override + @After + public void tearDown() throws Exception { + mockObjectStoreImpl.close(); + super.tearDown(); + } +} diff --git a/engine/storage/object/src/test/resource/testContext.xml b/engine/storage/object/src/test/resource/testContext.xml new file mode 100644 index 00000000000..7352b1148f7 --- /dev/null +++ b/engine/storage/object/src/test/resource/testContext.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.cloudstack.framework + + + + + + + + + + + + + + + + + + + + + + diff --git a/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 4268cf6446f..9c7ee983474 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -47,12 +47,14 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.log4j.Logger; @@ -99,6 +101,9 @@ public class SnapshotServiceImpl implements SnapshotService { @Inject ConfigurationDao _configDao; + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + static private class CreateSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; final AsyncCallFuture future; @@ -297,7 +302,7 @@ public class SnapshotServiceImpl implements SnapshotService { fullSnapshot = snapshotFullBackup; } if (fullSnapshot) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } else { SnapshotInfo parentSnapshot = snapshot.getParent(); // Note that DataStore information in parentSnapshot is for primary @@ -314,12 +319,25 @@ public class SnapshotServiceImpl implements SnapshotService { } } if (parentSnapshotOnBackupStore == null) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } return dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole()); } } + /** + * Verify if the data center has heuristic rules for allocating snapshots; if there is then returns the {@link DataStore} returned by the JS script. + * Otherwise, returns {@link DataStore}s with free capacity. + */ + protected DataStore getImageStoreForSnapshot(Long dataCenterId, SnapshotInfo snapshot) { + DataStore imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(dataCenterId, HeuristicType.SNAPSHOT, snapshot); + + if (imageStore == null) { + imageStore = dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + } + return imageStore; + } + @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotObject snapObj = (SnapshotObject)snapshot; diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java index 917fb2d9c75..6d59b6f36e0 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java @@ -18,6 +18,9 @@ */ package org.apache.cloudstack.storage.snapshot; +import com.cloud.storage.DataStoreRole; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -26,8 +29,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallFuture; -import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; -import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,8 +43,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import com.cloud.storage.DataStoreRole; - @RunWith(MockitoJUnitRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) public class SnapshotServiceImplTest { @@ -57,20 +58,28 @@ public class SnapshotServiceImplTest { SnapshotDataFactory _snapshotFactory; @Mock - AsyncCallbackDispatcher caller; + HeuristicRuleHelper heuristicRuleHelperMock; + + @Mock + SnapshotInfo snapshotMock; + + @Mock + VolumeInfo volumeInfoMock; + + @Mock + DataStoreManager dataStoreManagerMock; + + private static final long DUMMY_ID = 1L; @Test public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { - SnapshotInfo snapshot = Mockito.mock(SnapshotInfo.class); - VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class); - - Mockito.when(snapshot.getId()).thenReturn(1L); - Mockito.when(snapshot.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotMock.getId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotMock.getVolumeId()).thenReturn(DUMMY_ID); Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null); - Mockito.when(volFactory.getVolume(1L, DataStoreRole.Primary)).thenReturn(volumeInfo); + Mockito.when(volFactory.getVolume(DUMMY_ID, DataStoreRole.Primary)).thenReturn(volumeInfoMock); PrimaryDataStore store = Mockito.mock(PrimaryDataStore.class); - Mockito.when(volumeInfo.getDataStore()).thenReturn(store); + Mockito.when(volumeInfoMock.getDataStore()).thenReturn(store); PrimaryDataStoreDriver driver = Mockito.mock(PrimaryDataStoreDriver.class); Mockito.when(store.getDriver()).thenReturn(driver); @@ -80,7 +89,29 @@ public class SnapshotServiceImplTest { Mockito.when(mock.get()).thenReturn(result); Mockito.when(result.isFailed()).thenReturn(false); })) { - Assert.assertTrue(snapshotService.revertSnapshot(snapshot)); + Assert.assertTrue(snapshotService.revertSnapshot(snapshotMock)); } } + + @Test + public void getImageStoreForSnapshotTestShouldListFreeImageStoresWithNoHeuristicRule() { + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(null); + Mockito.when(snapshotMock.getDataCenterId()).thenReturn(DUMMY_ID); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + + @Test + public void getImageStoreForSnapshotTestShouldReturnImageStoreReturnedByTheHeuristicRule() { + DataStore dataStore = Mockito.mock(DataStore.class); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(dataStore); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index f2b0f17232b..89a7b577ae7 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -28,6 +28,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.ScopeType; import com.cloud.storage.StoragePoolStatus; @@ -85,6 +86,9 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement */ private SecureRandom secureRandom = new SecureRandom(); + @Inject + protected StoragePoolJoinDao storagePoolJoinDao; + @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java index fe49504fda1..9c0f84ab14a 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.VolumeApiServiceImpl; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -78,11 +79,12 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat logDisabledStoragePools(dcId, podId, clusterId, ScopeType.CLUSTER); } - List pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags()); + List pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags(), true, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value()); + pools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(dcId, podId, clusterId, ScopeType.CLUSTER, List.of(dskCh.getTags()))); s_logger.debug(String.format("Found pools [%s] that match with tags [%s].", pools, Arrays.toString(dskCh.getTags()))); // add remaining pools in cluster, that did not match tags, to avoid set - List allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null); + List allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null, false, 0); allPools.removeAll(pools); for (StoragePoolVO pool : allPools) { s_logger.trace(String.format("Adding pool [%s] to the 'avoid' set since it did not match any tags.", pool)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java index 774c2229a09..4ec15b9e43f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java @@ -101,7 +101,8 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { return null; } List availablePools = - storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags()); + storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags(), true); + availablePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), ScopeType.HOST, List.of(dskCh.getTags()))); for (StoragePoolVO pool : availablePools) { if (suitablePools.size() == returnUpTo) { break; @@ -117,7 +118,7 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { } // add remaining pools in cluster to the 'avoid' set which did not match tags - List allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null); + List allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null, false); allPools.removeAll(availablePools); for (StoragePoolVO pool : allPools) { avoid.addPool(pool.getId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java index 1b3835560df..ba130b4e2e5 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java @@ -63,9 +63,9 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { } List suitablePools = new ArrayList<>(); - - List storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags()); - if (storagePools == null) { + List storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags(), true); + storagePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), null, null, ScopeType.ZONE, List.of(dskCh.getTags()))); + if (storagePools.isEmpty()) { LOGGER.debug(String.format("Could not find any zone wide storage pool that matched with any of the following tags [%s].", Arrays.toString(dskCh.getTags()))); storagePools = new ArrayList<>(); } @@ -82,7 +82,7 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { storagePools.addAll(anyHypervisorStoragePools); // add remaining pools in zone, that did not match tags, to avoid set - List allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null); + List allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null, false); allPools.removeAll(storagePools); for (StoragePoolVO pool : allPools) { avoid.addPool(pool.getId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index cd525ae0ef7..757623e3d04 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.springframework.stereotype.Component; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; @@ -40,6 +41,8 @@ public class DataStoreManagerImpl implements DataStoreManager { PrimaryDataStoreProviderManager primaryStoreMgr; @Inject ImageStoreProviderManager imageDataStoreMgr; + @Inject + ObjectStoreProviderManager objectStoreProviderMgr; @Override public DataStore getDataStore(long storeId, DataStoreRole role) { @@ -50,6 +53,8 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.getImageStore(storeId); } else if (role == DataStoreRole.ImageCache) { return imageDataStoreMgr.getImageStore(storeId); + } else if (role == DataStoreRole.Object) { + return objectStoreProviderMgr.getObjectStore(storeId); } } catch (CloudRuntimeException e) { throw e; @@ -63,6 +68,8 @@ public class DataStoreManagerImpl implements DataStoreManager { return primaryStoreMgr.getPrimaryDataStore(uuid); } else if (role == DataStoreRole.Image) { return imageDataStoreMgr.getImageStore(uuid); + } else if (role == DataStoreRole.Object) { + return objectStoreProviderMgr.getObjectStore(uuid); } throw new CloudRuntimeException("un recognized type" + role); } @@ -77,6 +84,16 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.listImageStoresByScopeExcludingReadOnly(scope); } + @Override + public List getImageStoresByZoneIds(Long... zoneIds) { + return imageDataStoreMgr.listImageStoresFilteringByZoneIds(zoneIds); + } + + @Override + public DataStore getImageStoreByUuid(String uuid) { + return imageDataStoreMgr.getImageStore(uuid); + } + @Override public DataStore getRandomImageStore(long zoneId) { List stores = getImageStoresByScope(new ZoneScope(zoneId)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java index 98eeb6b4405..35e758accca 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java @@ -30,6 +30,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -56,6 +58,8 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto PrimaryDataStoreProviderManager primaryDataStoreProviderMgr; @Inject ImageStoreProviderManager imageStoreProviderMgr; + @Inject + ObjectStoreProviderManager objectStoreProviderMgr; @Override public DataStoreProvider getDataStoreProvider(String name) { @@ -144,6 +148,8 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto primaryDataStoreProviderMgr.registerHostListener(provider.getName(), provider.getHostListener()); } else if (types.contains(DataStoreProviderType.IMAGE)) { imageStoreProviderMgr.registerDriver(provider.getName(), (ImageStoreDriver)provider.getDataStoreDriver()); + } else if (types.contains(DataStoreProviderType.OBJECT)) { + objectStoreProviderMgr.registerDriver(provider.getName(), (ObjectStoreDriver)provider.getDataStoreDriver()); } } catch (Exception e) { s_logger.debug("configure provider failed", e); @@ -169,6 +175,11 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return this.getDataStoreProvider(DataStoreProvider.NFS_IMAGE); } + @Override + public DataStoreProvider getDefaultObjectStoreProvider() { + return this.getDataStoreProvider(DataStoreProvider.S3_IMAGE); + } + @Override public List getDataStoreProviders(String type) { if (type == null) { @@ -180,7 +191,9 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return this.getImageDataStoreProviders(); } else if (type.equalsIgnoreCase(DataStoreProvider.DataStoreProviderType.ImageCache.toString())) { return this.getCacheDataStoreProviders(); - } else { + } else if (type.equalsIgnoreCase(DataStoreProviderType.OBJECT.toString())) { + return this.getObjectStoreProviders(); + }else { throw new InvalidParameterValueException("Invalid parameter: " + type); } } @@ -223,4 +236,16 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return providers; } + public List getObjectStoreProviders() { + List providers = new ArrayList(); + for (DataStoreProvider provider : providerMap.values()) { + if (provider.getTypes().contains(DataStoreProviderType.OBJECT)) { + StorageProviderResponse response = new StorageProviderResponse(); + response.setName(provider.getName()); + response.setType(DataStoreProviderType.OBJECT.toString()); + providers.add(response); + } + } + return providers; + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 47e2ee38307..39f42e842c1 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -80,5 +80,8 @@ public interface ImageStoreProviderManager { List listImageStoresWithFreeCapacity(List imageStores); List orderImageStoresOnFreeCapacity(List imageStores); + + List listImageStoresFilteringByZoneIds(Long... zoneIds); + long getImageStoreZoneId(long dataStoreId); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java new file mode 100644 index 00000000000..e6027a1959f --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.cloud.agent.api.to.DataTO; +import com.cloud.host.Host; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.log4j.Logger; + +import java.util.Map; + +public abstract class BaseObjectStoreDriverImpl implements ObjectStoreDriver { + private static final Logger LOGGER = Logger.getLogger(BaseObjectStoreDriverImpl.class); + + @Override + public Map getCapabilities() { + return null; + } + + @Override + public DataTO getTO(DataObject data) { + return null; + } + + protected class CreateContext extends AsyncRpcContext { + final DataObject data; + + public CreateContext(AsyncCompletionCallback callback, DataObject data) { + super(callback); + this.data = data; + } + } + + @Override + public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + } + + @Override + public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + } + + @Override + public void copyAsync(DataObject srcdata, DataObject destData, AsyncCompletionCallback callback) { + } + + @Override + public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback callback) { + copyAsync(srcData, destData, callback); + } + + @Override + public boolean canCopy(DataObject srcData, DataObject destData) { + return false; + } + + @Override + public void resize(DataObject data, AsyncCompletionCallback callback) { + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java new file mode 100644 index 00000000000..4953b9b0cdf --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; + +import java.util.List; +import java.util.Map; + +public interface ObjectStoreDriver extends DataStoreDriver { + Bucket createBucket(Bucket bucket, boolean objectLock); + + List listBuckets(long storeId); + + boolean deleteBucket(String bucketName, long storeId); + + AccessControlList getBucketAcl(String bucketName, long storeId); + + void setBucketAcl(String bucketName, AccessControlList acl, long storeId); + + void setBucketPolicy(String bucketName, String policyType, long storeId); + + BucketPolicy getBucketPolicy(String bucketName, long storeId); + + void deleteBucketPolicy(String bucketName, long storeId); + + boolean createUser(long accountId, long storeId); + + boolean setBucketEncryption(String bucketName, long storeId); + + boolean deleteBucketEncryption(String bucketName, long storeId); + + + boolean setBucketVersioning(String bucketName, long storeId); + + boolean deleteBucketVersioning(String bucketName, long storeId); + + void setBucketQuota(String bucketName, long storeId, long size); + + Map getAllBucketsUsage(long storeId); +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java new file mode 100644 index 00000000000..c58d801e40e --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.datastore; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailVO; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; + +@Component +public class ObjectStoreHelper { + @Inject + ObjectStoreDao ObjectStoreDao; + @Inject + ObjectStoreDetailsDao ObjectStoreDetailsDao; + + public ObjectStoreVO createObjectStore(Map params, Map details) { + ObjectStoreVO store = new ObjectStoreVO(); + + store.setProviderName((String)params.get("providerName")); + store.setUuid(UUID.randomUUID().toString()); + store.setUrl((String)params.get("url")); + store.setName((String)params.get("name")); + + store = ObjectStoreDao.persist(store); + + // persist details + if (details != null) { + Iterator keyIter = details.keySet().iterator(); + while (keyIter.hasNext()) { + String key = keyIter.next().toString(); + String value = details.get(key); + ObjectStoreDetailVO detail = new ObjectStoreDetailVO(store.getId(), key, value); + ObjectStoreDetailsDao.persist(detail); + } + } + return store; + } + + public boolean deleteObjectStore(long id) { + ObjectStoreVO store = ObjectStoreDao.findById(id); + if (store == null) { + throw new CloudRuntimeException("can't find Object store:" + id); + } + + ObjectStoreDao.remove(id); + return true; + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java new file mode 100644 index 00000000000..b23f3194139 --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.datastore; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; + +import java.util.List; + +public interface ObjectStoreProviderManager { + ObjectStoreEntity getObjectStore(String uuid); + + List listObjectStores(); + + List listObjectStoreByProvider(String provider); + + ObjectStoreEntity getObjectStore(long objectStoreId); + + boolean registerDriver(String uuid, ObjectStoreDriver driver); + +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index c3379ad316b..fbb4a6e1618 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -136,7 +136,7 @@ public class PrimaryDataStoreHelper { storageTags.add(tag); } } - dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags); + dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags, params.isTagARule()); return dataStoreMgr.getDataStore(dataStoreVO.getId(), DataStoreRole.Primary); } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index ffc12b98c84..c0ef227251c 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -882,9 +882,7 @@ public class VolumeServiceImpl implements VolumeService { */ private TemplateInfo createManagedTemplateVolume(TemplateInfo srcTemplateInfo, PrimaryDataStore destPrimaryDataStore) { // create a template volume on primary storage - AsyncCallFuture createTemplateFuture = new AsyncCallFuture<>(); TemplateInfo templateOnPrimary = (TemplateInfo)destPrimaryDataStore.create(srcTemplateInfo, srcTemplateInfo.getDeployAsIsConfiguration()); - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId(), srcTemplateInfo.getDeployAsIsConfiguration()); if (templatePoolRef == null) { @@ -897,7 +895,6 @@ public class VolumeServiceImpl implements VolumeService { // At this point, we have an entry in the DB that points to our cached template. // We need to lock it as there may be other VMs that may get started using the same template. // We want to avoid having to create multiple cache copies of the same template. - int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600); long templatePoolRefId = templatePoolRef.getId(); @@ -909,28 +906,27 @@ public class VolumeServiceImpl implements VolumeService { try { // create a cache volume on the back-end - templateOnPrimary.processEvent(Event.CreateOnlyRequested); + CreateAsyncCompleteCallback callback = new CreateAsyncCompleteCallback(); - CreateVolumeContext createContext = new CreateVolumeContext<>(null, templateOnPrimary, createTemplateFuture); - AsyncCallbackDispatcher createCaller = AsyncCallbackDispatcher.create(this); - - createCaller.setCallback(createCaller.getTarget().createManagedTemplateImageCallback(null, null)).setContext(createContext); - - destPrimaryDataStore.getDriver().createAsync(destPrimaryDataStore, templateOnPrimary, createCaller); - - VolumeApiResult result = createTemplateFuture.get(); - - if (result.isFailed()) { - String errMesg = result.getResult(); - + destPrimaryDataStore.getDriver().createAsync(destPrimaryDataStore, templateOnPrimary, callback); + // validate we got a good result back + if (callback.result == null || callback.result.isFailed()) { + String errMesg; + if (callback.result == null) { + errMesg = "Unknown/unable to determine result"; + } else { + errMesg = callback.result.getResult(); + } + templateOnPrimary.processEvent(Event.OperationFailed); throw new CloudRuntimeException("Unable to create template " + templateOnPrimary.getId() + " on primary storage " + destPrimaryDataStore.getId() + ":" + errMesg); } + + templateOnPrimary.processEvent(Event.OperationSuccessed); + } catch (Throwable e) { s_logger.debug("Failed to create template volume on storage", e); - templateOnPrimary.processEvent(Event.OperationFailed); - throw new CloudRuntimeException(e.getMessage()); } finally { _tmpltPoolDao.releaseFromLockTable(templatePoolRefId); @@ -939,6 +935,17 @@ public class VolumeServiceImpl implements VolumeService { return templateOnPrimary; } + private static class CreateAsyncCompleteCallback implements AsyncCompletionCallback { + + public CreateCmdResult result; + + @Override + public void complete(CreateCmdResult result) { + this.result = result; + } + + } + /** * This function copies a template from secondary storage to a template volume * created on managed storage. This template volume will be used as a cache. @@ -1464,6 +1471,16 @@ public class VolumeServiceImpl implements VolumeService { if (templatePoolRef.getDownloadState() == Status.NOT_DOWNLOADED) { copyTemplateToManagedTemplateVolume(srcTemplateInfo, templateOnPrimary, templatePoolRef, destPrimaryDataStore, destHost); } + } catch (Exception e) { + if (templateOnPrimary != null) { + templateOnPrimary.processEvent(Event.OperationFailed); + } + VolumeApiResult result = new VolumeApiResult(volumeInfo); + result.setResult(e.getLocalizedMessage()); + result.setSuccess(false); + future.complete(result); + s_logger.warn("Failed to create template on primary storage", e); + return future; } finally { if (lock != null) { lock.unlock(); @@ -1478,8 +1495,8 @@ public class VolumeServiceImpl implements VolumeService { createManagedVolumeCloneTemplateAsync(volumeInfo, templateOnPrimary, destPrimaryDataStore, future); } else { // We have a template on PowerFlex primary storage. Create new volume and copy to it. - s_logger.debug("Copying the template to the volume on primary storage"); - createManagedVolumeCopyManagedTemplateAsync(volumeInfo, destPrimaryDataStore, templateOnPrimary, destHost, future); + createManagedVolumeCopyManagedTemplateAsyncWithLock(volumeInfo, destPrimaryDataStore, templateOnPrimary, + destHost, future, destDataStoreId, srcTemplateInfo.getId()); } } else { s_logger.debug("Primary storage does not support cloning or no support for UUID resigning on the host side; copying the template normally"); @@ -1490,6 +1507,32 @@ public class VolumeServiceImpl implements VolumeService { return future; } + private void createManagedVolumeCopyManagedTemplateAsyncWithLock(VolumeInfo volumeInfo, PrimaryDataStore destPrimaryDataStore, TemplateInfo templateOnPrimary, + Host destHost, AsyncCallFuture future, long destDataStoreId, long srcTemplateId) { + GlobalLock lock = null; + try { + String tmplIdManagedPoolIdDestinationHostLockString = "tmplId:" + srcTemplateId + "managedPoolId:" + destDataStoreId + "destinationHostId:" + destHost.getId(); + lock = GlobalLock.getInternLock(tmplIdManagedPoolIdDestinationHostLockString); + if (lock == null) { + throw new CloudRuntimeException("Unable to create volume from template, couldn't get global lock on " + tmplIdManagedPoolIdDestinationHostLockString); + } + + int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600); + if (!lock.lock(storagePoolMaxWaitSeconds)) { + s_logger.debug("Unable to create volume from template, couldn't lock on " + tmplIdManagedPoolIdDestinationHostLockString); + throw new CloudRuntimeException("Unable to create volume from template, couldn't lock on " + tmplIdManagedPoolIdDestinationHostLockString); + } + + s_logger.debug("Copying the template to the volume on primary storage"); + createManagedVolumeCopyManagedTemplateAsync(volumeInfo, destPrimaryDataStore, templateOnPrimary, destHost, future); + } finally { + if (lock != null) { + lock.unlock(); + lock.releaseRef(); + } + } + } + private boolean computeSupportsVolumeClone(long zoneId, HypervisorType hypervisorType) { if (HypervisorType.VMware.equals(hypervisorType) || HypervisorType.KVM.equals(hypervisorType)) { return true; diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java index 337131038e8..fef3e4376dc 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java @@ -22,6 +22,8 @@ import java.util.List; public class Host extends GenericPresetVariable { private List tags; + private Boolean isTagARule; + public List getTags() { return tags; } @@ -31,4 +33,12 @@ public class Host extends GenericPresetVariable { fieldNamesToIncludeInToString.add("tags"); } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + fieldNamesToIncludeInToString.add("isTagARule"); + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 6ce8bd889a6..9723d3e5899 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -25,9 +25,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.cloud.host.HostTagVO; import javax.inject.Inject; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -351,7 +353,17 @@ public class PresetVariableHelper { Host host = new Host(); host.setId(hostVo.getUuid()); host.setName(hostVo.getName()); - host.setTags(hostTagsDao.getHostTags(hostId)); + List hostTagVOList = hostTagsDao.getHostTags(hostId); + List hostTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(hostTagVOList)) { + isTagARule = hostTagVOList.get(0).getIsTagARule(); + if (!isTagARule) { + hostTags = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); + } + } + host.setTags(hostTags); + host.setIsTagARule(isTagARule); return host; } @@ -508,7 +520,17 @@ public class PresetVariableHelper { storage.setId(storagePoolVo.getUuid()); storage.setName(storagePoolVo.getName()); storage.setScope(storagePoolVo.getScope()); - storage.setTags(storagePoolTagsDao.getStoragePoolTags(storageId)); + List storagePoolTagVOList = storagePoolTagsDao.findStoragePoolTags(storageId); + List storageTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(storagePoolTagVOList)) { + isTagARule = storagePoolTagVOList.get(0).isTagARule(); + if (!isTagARule) { + storageTags = storagePoolTagVOList.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()); + } + } + storage.setTags(storageTags); + storage.setIsTagARule(isTagARule); return storage; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java index 3533c5d45c1..6be1dfb025a 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java @@ -23,6 +23,8 @@ import com.cloud.storage.ScopeType; public class Storage extends GenericPresetVariable { private List tags; + + private Boolean isTagARule; private ScopeType scope; public List getTags() { @@ -34,6 +36,15 @@ public class Storage extends GenericPresetVariable { fieldNamesToIncludeInToString.add("tags"); } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + fieldNamesToIncludeInToString.add("isTagARule"); + } + public ScopeType getScope() { return scope; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 553dc840c0d..6f19fa95f1b 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -55,6 +55,7 @@ public class QuotaTypes extends UsageTypes { quotaTypeList.put(VOLUME_SECONDARY, new QuotaTypes(VOLUME_SECONDARY, "VOLUME_SECONDARY", UsageUnitTypes.GB_MONTH.toString(), "Volume secondary storage usage")); quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot primary storage usage")); quotaTypeList.put(BACKUP, new QuotaTypes(BACKUP, "BACKUP", UsageUnitTypes.GB_MONTH.toString(), "Backup storage usage")); + quotaTypeList.put(BUCKET, new QuotaTypes(BUCKET, "BUCKET", UsageUnitTypes.GB_MONTH.toString(), "Object Store bucket usage")); quotaTypeMap = Collections.unmodifiableMap(quotaTypeList); } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index ae15e573fa8..b973d1145c3 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -27,7 +27,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -241,6 +243,14 @@ public class PresetVariableHelperTest { return host; } + private List getHostTagsForTests() { + return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); + } + + private List getHostRuleTagsForTests() { + return List.of(new HostTagVO(1, "tagrule", true)); + } + private Storage getStorageForTests() { Storage storage = new Storage(); storage.setId("storage_id"); @@ -250,6 +260,14 @@ public class PresetVariableHelperTest { return storage; } + private List getStorageTagsForTests() { + return Arrays.asList(new StoragePoolTagVO(1, "tag1", false), new StoragePoolTagVO(1, "tag2", false)); + } + + private List getStorageRuleTagsForTests() { + return List.of(new StoragePoolTagVO(1, "tagrule", true)); + } + private Set> getQuotaTypesForTests(Integer... typesToRemove) { Map quotaTypesMap = new LinkedHashMap<>(QuotaTypes.listQuotaTypes()); @@ -539,15 +557,16 @@ public class PresetVariableHelperTest { @Test public void setPresetVariableHostInValueIfUsageTypeIsRunningVmTestQuotaTypeIsRunningVmSetHost() { Value result = new Value(); - Host expected = getHostForTests(); + Host expectedHost = getHostForTests(); + List expectedHostTags = getHostTagsForTests(); - Mockito.doReturn(expected).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong()); + Mockito.doReturn(expectedHost).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong()); presetVariableHelperSpy.setPresetVariableHostInValueIfUsageTypeIsRunningVm(result, UsageTypes.RUNNING_VM, vmInstanceVoMock); Assert.assertNotNull(result.getHost()); - assertPresetVariableIdAndName(expected, result.getHost()); - Assert.assertEquals(expected.getTags(), result.getHost().getTags()); + assertPresetVariableIdAndName(expectedHost, result.getHost()); + Assert.assertEquals(expectedHost.getTags(), result.getHost().getTags()); validateFieldNamesToIncludeInToString(Arrays.asList("host"), result); } @@ -555,18 +574,39 @@ public class PresetVariableHelperTest { public void getPresetVariableValueHostTestSetFieldsAndReturnObject() { Host expected = getHostForTests(); HostVO hostVoMock = Mockito.mock(HostVO.class); + List hostTagVOListMock = getHostTagsForTests(); Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); mockMethodValidateIfObjectIsNull(); Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(hostVoMock).getName(); - Mockito.doReturn(expected.getTags()).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); + Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l); assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getTags(), result.getTags()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "tags"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result); + } + + @Test + public void getPresetVariableValueHostTestSetFieldsWithRuleTagAndReturnObject() { + Host expected = getHostForTests(); + HostVO hostVoMock = Mockito.mock(HostVO.class); + List hostTagVOListMock = getHostRuleTagsForTests(); + + Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(hostVoMock).getName(); + Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); + + Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(new ArrayList<>(), result.getTags()); + Assert.assertTrue(result.getIsTagARule()); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result); } @Test @@ -755,13 +795,15 @@ public class PresetVariableHelperTest { Storage expected = getStorageForTests(); Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt()); + List storageTagVOListMock = getStorageTagsForTests(); + StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class); Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName(); Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope(); - Mockito.doReturn(expected.getTags()).when(storagePoolTagsDaoMock).getStoragePoolTags(Mockito.anyLong()); + Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong()); Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2); @@ -769,7 +811,32 @@ public class PresetVariableHelperTest { Assert.assertEquals(expected.getScope(), result.getScope()); Assert.assertEquals(expected.getTags(), result.getTags()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "scope", "tags"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "scope", "tags"), result); + } + + @Test + public void getPresetVariableValueStorageTestGetSecondaryStorageForSnapshotReturnsNullWithRuleTag() { + Storage expected = getStorageForTests(); + Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt()); + + List storageTagVOListMock = getStorageRuleTagsForTests(); + + StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class); + Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName(); + Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope(); + Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong()); + + Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getScope(), result.getScope()); + Assert.assertEquals(new ArrayList<>(), result.getTags()); + Assert.assertTrue(result.getIsTagARule()); + + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "scope", "tags"), result); } @Test diff --git a/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java b/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java index ec9bbc0cd97..83d2feaa845 100644 --- a/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java +++ b/framework/spring/module/src/main/java/org/apache/cloudstack/spring/module/model/impl/DefaultModuleDefinitionSet.java @@ -105,10 +105,16 @@ public class DefaultModuleDefinitionSet implements ModuleDefinitionSet { log.info(String.format("Starting module [%s].", moduleDefinitionName)); runnable.run(); } catch (BeansException e) { - log.error(String.format("Failed to start module [%s] due to: [%s].", moduleDefinitionName, e.getMessage()), e); + log.warn(String.format("Failed to start module [%s] due to: [%s].", moduleDefinitionName, e.getMessage())); + if (log.isDebugEnabled()) { + log.debug(String.format("module start failure of module [%s] was due to: ", moduleDefinitionName), e); + } } } catch (EmptyStackException e) { - log.error(String.format("Failed to obtain module context due to [%s]. Using root context instead.", e.getMessage()), e); + log.warn(String.format("Failed to obtain module context due to [%s]. Using root context instead.", e.getMessage())); + if (log.isDebugEnabled()) { + log.debug("Failed to obtain module context: ", e); + } } } }); @@ -125,9 +131,15 @@ public class DefaultModuleDefinitionSet implements ModuleDefinitionSet { log.debug(String.format("Trying to load module [%s] context.", moduleDefinitionName)); loadContext(def, parent); } catch (EmptyStackException e) { - log.error(String.format("Failed to obtain module context due to [%s]. Using root context instead.", e.getMessage()), e); + log.warn(String.format("Failed to obtain module context due to [%s]. Using root context instead.", e.getMessage())); + if (log.isDebugEnabled()) { + log.debug("Failed to obtain module context: ", e); + } } catch (BeansException e) { - log.error(String.format("Failed to load module [%s] due to: [%s].", def.getName(), e.getMessage()), e); + log.warn(String.format("Failed to start module [%s] due to: [%s].", def.getName(), e.getMessage())); + if (log.isDebugEnabled()) { + log.debug(String.format("module start failure of module [%s] was due to: ", def.getName()), e); + } } } }); diff --git a/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java b/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java index 9363ebd2379..0306a062df9 100644 --- a/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java +++ b/plugins/acl/project-role-based/src/main/java/org/apache/cloudstack/acl/ProjectRoleBasedApiAccessChecker.java @@ -61,7 +61,9 @@ public class ProjectRoleBasedApiAccessChecker extends AdapterBase implements AP @Override public boolean isEnabled() { if (!roleService.isEnabled()) { - LOGGER.trace("RoleService is disabled. We will not use ProjectRoleBasedApiAccessChecker."); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("RoleService is disabled. We will not use ProjectRoleBasedApiAccessChecker."); + } } return roleService.isEnabled(); } @@ -119,7 +121,9 @@ public class ProjectRoleBasedApiAccessChecker extends AdapterBase implements AP Account userAccount = accountService.getAccount(user.getAccountId()); if (accountService.isRootAdmin(userAccount.getId()) || accountService.isDomainAdmin(userAccount.getAccountId())) { - LOGGER.info(String.format("Account [%s] is Root Admin or Domain Admin, all APIs are allowed.", userAccount.getAccountName())); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Account [%s] is Root Admin or Domain Admin, all APIs are allowed.", userAccount.getAccountName())); + } return true; } diff --git a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java index 966c83722ec..f012dbfa972 100644 --- a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java +++ b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java @@ -35,9 +35,9 @@ import org.springframework.stereotype.Component; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.storage.Storage.StoragePoolType; diff --git a/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java b/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java index 82284f9a1e0..96f8a7a7977 100644 --- a/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java +++ b/plugins/backup/networker/src/test/java/org/apache/backup/networker/NetworkerClientTest.java @@ -46,7 +46,7 @@ import com.github.tomakehurst.wiremock.junit.WireMockRule; public class NetworkerClientTest { private final String adminUsername = "administrator"; private final String adminPassword = "password"; - private final int port = 9399; + private final int port = 9400; private final String url = "http://localhost:" + port + "/nwrestapi/v3"; private NetworkerClient client; @Rule @@ -87,7 +87,7 @@ public class NetworkerClientTest { " \"comment\": \"-CSBKP-\",\n" + " \"links\": [\n" + " {\n" + - " \"href\": \"http://localhost:9399/nwrestapi/v3/global/protectionpolicies/CSBRONZE\",\n" + + " \"href\": \"http://localhost:9400/nwrestapi/v3/global/protectionpolicies/CSBRONZE\",\n" + " \"rel\": \"item\"\n" + " }\n" + " ],\n" + @@ -108,7 +108,7 @@ public class NetworkerClientTest { " \"comment\": \"-CSBKP-\",\n" + " \"links\": [\n" + " {\n" + - " \"href\": \"http://localhost:9399/nwrestapi/v3/global/protectionpolicies/CSGOLD\",\n" + + " \"href\": \"http://localhost:9400/nwrestapi/v3/global/protectionpolicies/CSGOLD\",\n" + " \"rel\": \"item\"\n" + " }\n" + " ],\n" + @@ -129,7 +129,7 @@ public class NetworkerClientTest { " \"comment\": \"-CSBKP-\",\n" + " \"links\": [\n" + " {\n" + - " \"href\": \"http://localhost:9399/nwrestapi/v3/global/protectionpolicies/CSSILVER\",\n" + + " \"href\": \"http://localhost:9400/nwrestapi/v3/global/protectionpolicies/CSSILVER\",\n" + " \"rel\": \"item\"\n" + " }\n" + " ],\n" + @@ -210,7 +210,7 @@ public class NetworkerClientTest { " \"level\": \"Manual\",\n" + " \"links\": [\n" + " {\n" + - " \"href\": \"http://localhost:9399/nwrestapi/v3/global/backups/6034732f-00000006-7acd14e3-62cd14e3-00871500-5a80015d\",\n" + + " \"href\": \"http://localhost:9400/nwrestapi/v3/global/backups/6034732f-00000006-7acd14e3-62cd14e3-00871500-5a80015d\",\n" + " \"rel\": \"item\"\n" + " }\n" + " ],\n" + @@ -271,7 +271,7 @@ public class NetworkerClientTest { " \"level\": \"Manual\",\n" + " \"links\": [\n" + " {\n" + - " \"href\": \"http://localhost:9399/nwrestapi/v3/global/backups/98d29c5e-00000006-81ccda87-62ccda87-00801500-5a80015d\",\n" + + " \"href\": \"http://localhost:9400/nwrestapi/v3/global/backups/98d29c5e-00000006-81ccda87-62ccda87-00801500-5a80015d\",\n" + " \"rel\": \"item\"\n" + " }\n" + " ],\n" + @@ -332,7 +332,7 @@ public class NetworkerClientTest { " \"level\": \"Manual\",\n" + " \"links\": [\n" + " {\n" + - " \"href\": \"http://localhost:9399/nwrestapi/v3/global/backups/d371d629-00000006-84ccd61b-62ccd61b-007d1500-5a80015d\",\n" + + " \"href\": \"http://localhost:9400/nwrestapi/v3/global/backups/d371d629-00000006-84ccd61b-62ccd61b-007d1500-5a80015d\",\n" + " \"rel\": \"item\"\n" + " }\n" + " ],\n" + diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index 02f08d602bb..c0091e47061 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -41,9 +41,9 @@ import org.apache.commons.lang3.BooleanUtils; import org.apache.log4j.Logger; import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.vmware.VmwareDatacenter; +import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index 8a46d10a7b5..70920df5eb5 100644 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -22,7 +22,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.ListUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -72,7 +74,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { } String hostTag = offering.getHostTag(); if (hostTag != null) { - s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId + " having host tag:" + hostTag); + s_logger.debug(String.format("Looking for hosts in dc [%s], pod [%s], cluster [%s] and complying with host tag [%s].", dcId, podId, clusterId, hostTag)); } else { s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); } @@ -82,7 +84,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { if (hostTag != null) { hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag)); } else { - hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId)); + hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); } } else { // list all computing hosts, regardless of whether they support routing...it's random after all @@ -90,9 +92,16 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { if (hostTag != null) { hostsCopy = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag); } else { - hostsCopy = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); } } + hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTag)); + + if (hostsCopy.isEmpty()) { + s_logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTag)); + throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile)); + } + s_logger.debug("Random Allocator found " + hostsCopy.size() + " hosts"); if (hostsCopy.size() == 0) { return suitableHosts; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 8242d244efa..60e6bcffeb6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -73,6 +73,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.apache.xerces.impl.xpath.regex.Match; @@ -485,6 +486,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv */ private static final String COMMAND_SET_MEM_BALLOON_STATS_PERIOD = "virsh dommemstat %s --period %s --live"; + private static int hostCpuMaxCapacity = 0; + + private static final int CGROUP_V2_UPPER_LIMIT = 10000; + + private static final String COMMAND_GET_CGROUP_HOST_VERSION = "stat -fc %T /sys/fs/cgroup/"; + + public static final String CGROUP_V2 = "cgroup2fs"; + protected long getHypervisorLibvirtVersion() { return hypervisorLibvirtVersion; } @@ -565,6 +574,18 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return new ExecutionResult(true, null); } + /** + * @return the host CPU max capacity according to the method {@link LibvirtComputingResource#calculateHostCpuMaxCapacity(int, Long)}; if the host utilizes cgroup v1, this + * value is 0. + */ + public int getHostCpuMaxCapacity() { + return hostCpuMaxCapacity; + } + + public void setHostCpuMaxCapacity(int hostCpuMaxCapacity) { + LibvirtComputingResource.hostCpuMaxCapacity = hostCpuMaxCapacity; + } + public LibvirtKvmAgentHook getTransformer() throws IOException { return new LibvirtKvmAgentHook(agentHooksBasedir, agentHooksLibvirtXmlScript, agentHooksLibvirtXmlMethod); } @@ -743,6 +764,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected StorageSubsystemCommandHandler storageHandler; + private boolean convertInstanceVerboseMode = false; protected boolean dpdkSupport = false; protected String dpdkOvsPath; protected String directDownloadTemporaryDownloadPath; @@ -803,6 +825,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return networkDirectDevice; } + public boolean isConvertInstanceVerboseModeEnabled() { + return convertInstanceVerboseMode; + } + /** * Defines resource's public and private network interface according to what is configured in agent.properties. */ @@ -991,6 +1017,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv params.putAll(getDeveloperProperties()); } + convertInstanceVerboseMode = BooleanUtils.isTrue(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VIRTV2V_VERBOSE_ENABLED)); + pool = (String)params.get("pool"); if (pool == null) { pool = "/root"; @@ -1004,10 +1032,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv rollingMaintenanceExecutor = BooleanUtils.isTrue(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.ROLLING_MAINTENANCE_SERVICE_EXECUTOR_DISABLED)) ? new RollingMaintenanceAgentExecutor(hooksDir) : new RollingMaintenanceServiceExecutor(hooksDir); - hypervisorURI = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HYPERVISOR_URI); - if (hypervisorURI == null) { - hypervisorURI = LibvirtConnection.getHypervisorURI(hypervisorType.toString()); - } + hypervisorURI = LibvirtConnection.getHypervisorURI(hypervisorType.toString()); networkDirectSourceMode = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.NETWORK_DIRECT_SOURCE_MODE); networkDirectDevice = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.NETWORK_DIRECT_DEVICE); @@ -1040,7 +1065,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } - enableSSLForKvmAgent(params); + enableSSLForKvmAgent(); configureLocalStorage(); /* Directory to use for Qemu sockets like for the Qemu Guest Agent */ @@ -1349,13 +1374,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } - private void enableSSLForKvmAgent(final Map params) { + private void enableSSLForKvmAgent() { final File keyStoreFile = PropertiesUtil.findConfigFile(KeyStoreUtils.KS_FILENAME); if (keyStoreFile == null) { s_logger.info("Failed to find keystore file: " + KeyStoreUtils.KS_FILENAME); return; } - String keystorePass = (String)params.get(KeyStoreUtils.KS_PASSPHRASE_PROPERTY); + String keystorePass = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KEYSTORE_PASSPHRASE); if (StringUtils.isBlank(keystorePass)) { s_logger.info("Failed to find passphrase for keystore: " + KeyStoreUtils.KS_FILENAME); return; @@ -2270,7 +2295,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return new Pair, Integer>(macAddressToNicNum, devNum); } - protected PowerState convertToPowerState(final DomainState ps) { + public PowerState convertToPowerState(final DomainState ps) { final PowerState state = POWER_STATES_TABLE.get(ps); return state == null ? PowerState.PowerUnknown : state; } @@ -2703,12 +2728,41 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv */ protected CpuTuneDef createCpuTuneDef(VirtualMachineTO vmTO) { CpuTuneDef ctd = new CpuTuneDef(); - int shares = vmTO.getCpus() * (vmTO.getMinSpeed() != null ? vmTO.getMinSpeed() : vmTO.getSpeed()); - ctd.setShares(shares); + ctd.setShares(calculateCpuShares(vmTO)); setQuotaAndPeriod(vmTO, ctd); return ctd; } + /** + * Calculates the VM CPU shares considering the cgroup version of the host. + *
    + *
  • + * If the host utilize cgroup v1, then, the CPU shares is calculated as VM CPU shares = CPU cores * CPU frequency. + *
    • + *
  • + * If the host utilize cgroup v2, the CPU shares calculation considers the cgroup v2 upper limit of 10,000, and a linear scale conversion is applied + * considering the maximum host CPU shares (i.e. using the number of CPU cores and CPU nominal frequency of the host). Therefore, the VM CPU shares is calculated as + * VM CPU shares = (VM requested shares * cgroup upper limit) / host max shares. + *
    • + *
+ */ + public int calculateCpuShares(VirtualMachineTO vmTO) { + int vCpus = vmTO.getCpus(); + int cpuSpeed = ObjectUtils.defaultIfNull(vmTO.getMinSpeed(), vmTO.getSpeed()); + int requestedCpuShares = vCpus * cpuSpeed; + int hostCpuMaxCapacity = getHostCpuMaxCapacity(); + + if (hostCpuMaxCapacity > 0) { + int updatedCpuShares = (int) Math.ceil((requestedCpuShares * CGROUP_V2_UPPER_LIMIT) / (double) hostCpuMaxCapacity); + s_logger.debug(String.format("This host utilizes cgroupv2 (as the max shares value is [%s]), thus, the VM requested shares of [%s] will be converted to " + + "consider the host limits; the new CPU shares value is [%s].", hostCpuMaxCapacity, requestedCpuShares, updatedCpuShares)); + return updatedCpuShares; + } + s_logger.debug(String.format("This host does not have a maximum CPU shares set; therefore, this host utilizes cgroupv1 and the VM requested CPU shares [%s] will not be " + + "converted.", requestedCpuShares)); + return requestedCpuShares; + } + private CpuModeDef createCpuModeDef(VirtualMachineTO vmTO, int vcpus) { final CpuModeDef cmd = new CpuModeDef(); cmd.setMode(guestCpuMode); @@ -3544,8 +3598,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv @Override public StartupCommand[] initialize() { - final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed, dom0MinCpuCores); + calculateHostCpuMaxCapacity(info.getAllocatableCpus(), info.getCpuSpeed()); String capabilities = String.join(",", info.getCapabilities()); if (dpdkSupport) { @@ -3593,6 +3647,32 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return startupCommandsArray; } + /** + * Calculates and sets the host CPU max capacity according to the cgroup version of the host. + *
    + *
  • + * cgroup v1: the max CPU capacity for the host is set to 0. + *
    • + *
  • + * cgroup v2: the max CPU capacity for the host is the value of cpuCores * cpuSpeed. + *
    • + *
+ */ + protected void calculateHostCpuMaxCapacity(int cpuCores, Long cpuSpeed) { + String output = Script.runSimpleBashScript(COMMAND_GET_CGROUP_HOST_VERSION); + s_logger.info(String.format("Host uses control group [%s].", output)); + + if (!CGROUP_V2.equals(output)) { + s_logger.info(String.format("Setting host CPU max capacity to 0, as it uses cgroup v1.", getHostCpuMaxCapacity())); + setHostCpuMaxCapacity(0); + return; + } + + s_logger.info(String.format("Calculating the max shares of the host.")); + setHostCpuMaxCapacity(cpuCores * cpuSpeed.intValue()); + s_logger.info(String.format("The max shares of the host is [%d].", getHostCpuMaxCapacity())); + } + private StartupStorageCommand createLocalStoragePool(String localStoragePath, String localStorageUUID, StartupRoutingCommand cmd) { StartupStorageCommand sscmd = null; try { @@ -3697,7 +3777,39 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } - protected List getAllVmNames(final Connect conn) { + /** + * Given a disk path on KVM host, attempts to find source host and path using mount command + * @param diskPath KVM host path for virtual disk + * @return Pair with IP of host and path + */ + public Pair getSourceHostPath(String diskPath) { + String sourceHostIp = null; + String sourcePath = null; + try { + String mountResult = Script.runSimpleBashScript("mount | grep \"" + diskPath + "\""); + s_logger.debug("Got mount result for " + diskPath + "\n\n" + mountResult); + if (StringUtils.isNotEmpty(mountResult)) { + String[] res = mountResult.strip().split(" "); + if (res[0].contains(":")) { + res = res[0].split(":"); + sourceHostIp = res[0].strip(); + sourcePath = res[1].strip(); + } else { + // Assume local storage + sourceHostIp = getPrivateIp(); + sourcePath = diskPath; + } + } + if (StringUtils.isNotEmpty(sourceHostIp) && StringUtils.isNotEmpty(sourcePath)) { + return new Pair<>(sourceHostIp, sourcePath); + } + } catch (Exception ex) { + s_logger.warn("Failed to list source host and IP for " + diskPath + ex.toString()); + } + return null; + } + + public List getAllVmNames(final Connect conn) { final ArrayList la = new ArrayList(); try { final String names[] = conn.listDefinedDomains(); @@ -4112,7 +4224,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return DiskDef.DiskBus.SCSI; } else if (platformEmulator.contains("Ubuntu") || StringUtils.startsWithAny(platformEmulator, - "Fedora", "CentOS", "Red Hat Enterprise Linux", "Debian GNU/Linux", "FreeBSD", "Oracle", "Other PV")) { + "Fedora", "CentOS", "Red Hat Enterprise Linux", "Debian GNU/Linux", "FreeBSD", "Oracle", + "Rocky Linux", "AlmaLinux", "Other PV")) { return DiskDef.DiskBus.VIRTIO; } else if (isUefiEnabled && StringUtils.startsWithAny(platformEmulator, "Windows", "Other")) { return DiskDef.DiskBus.SATA; @@ -5258,4 +5371,25 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } } + + /* + Scp volume from remote host to local directory + */ + public String copyVolume(String srcIp, String username, String password, String localDir, String remoteFile, String tmpPath) { + try { + String outputFile = UUID.randomUUID().toString(); + StringBuilder command = new StringBuilder("qemu-img convert -O qcow2 "); + command.append(remoteFile); + command.append(" "+tmpPath); + command.append(outputFile); + s_logger.debug("Converting remoteFile: "+remoteFile); + SshHelper.sshExecute(srcIp, 22, username, null, password, command.toString()); + s_logger.debug("Copying remoteFile to: "+localDir); + SshHelper.scpFrom(srcIp, 22, username, null, password, localDir, tmpPath+outputFile); + s_logger.debug("Successfully copyied remoteFile to: "+localDir+"/"+outputFile); + return outputFile; + } catch (Exception e) { + throw new RuntimeException(e); + } + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java index c70a72f399c..7563f964759 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java @@ -19,6 +19,8 @@ package com.cloud.hypervisor.kvm.resource; import java.util.HashMap; import java.util.Map; +import com.cloud.agent.properties.AgentProperties; +import com.cloud.agent.properties.AgentPropertiesFileHandler; import org.apache.log4j.Logger; import org.libvirt.Connect; import org.libvirt.LibvirtException; @@ -88,10 +90,15 @@ public class LibvirtConnection { } static String getHypervisorURI(String hypervisorType) { + String uri = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HYPERVISOR_URI); + if (uri != null) { + return uri; + } + if ("LXC".equalsIgnoreCase(hypervisorType)) { return "lxc:///"; - } else { - return "qemu:///system"; } + + return "qemu:///system"; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index a5565c2de34..f165796adef 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -57,8 +57,10 @@ public class LibvirtDomainXMLParser { private final List channels = new ArrayList(); private final List watchDogDefs = new ArrayList(); private Integer vncPort; + private String vncPasswd; private String desc; - + private LibvirtVMDef.CpuTuneDef cpuTuneDef; + private LibvirtVMDef.CpuModeDef cpuModeDef; private String name; public boolean parseDomainXML(String domXML) { @@ -278,6 +280,14 @@ public class LibvirtDomainXMLParser { String name = getAttrValue("target", "name", channel); String state = getAttrValue("target", "state", channel); + if (ChannelDef.ChannelType.valueOf(type.toUpperCase()).equals(ChannelDef.ChannelType.SPICEVMC)) { + continue; + } + + if (path == null) { + path = ""; + } + ChannelDef def = null; if (StringUtils.isBlank(state)) { def = new ChannelDef(name, ChannelDef.ChannelType.valueOf(type.toUpperCase()), new File(path)); @@ -305,6 +315,12 @@ public class LibvirtDomainXMLParser { vncPort = null; } } + + String passwd = graphic.getAttribute("passwd"); + if (passwd != null) { + vncPasswd = passwd; + } + } NodeList rngs = devices.getElementsByTagName("rng"); @@ -317,6 +333,26 @@ public class LibvirtDomainXMLParser { String period = getAttrValue("rate", "period", rng); if (StringUtils.isAnyEmpty(bytes, period)) { s_logger.debug(String.format("Bytes and period in the rng section should not be null, please check the VM %s", name)); + } + + if (bytes == null) { + bytes = "0"; + } + + if (period == null) { + period = "0"; + } + + if (bytes == null) { + bytes = "0"; + } + + if (period == null) { + period = "0"; + } + + if (StringUtils.isEmpty(backendModel)) { + def = new RngDef(path, Integer.parseInt(bytes), Integer.parseInt(period)); } else { if (StringUtils.isEmpty(backendModel)) { def = new RngDef(path, Integer.parseInt(bytes), Integer.parseInt(period)); @@ -350,7 +386,8 @@ public class LibvirtDomainXMLParser { watchDogDefs.add(def); } - + extractCpuTuneDef(rootElement); + extractCpuModeDef(rootElement); return true; } catch (ParserConfigurationException e) { s_logger.debug(e.toString()); @@ -411,6 +448,10 @@ public class LibvirtDomainXMLParser { return interfaces; } + public String getVncPasswd() { + return vncPasswd; + } + public MemBalloonDef getMemBalloon() { return memBalloonDef; } @@ -438,4 +479,65 @@ public class LibvirtDomainXMLParser { public String getName() { return name; } + + public LibvirtVMDef.CpuTuneDef getCpuTuneDef() { + return cpuTuneDef; + } + + public LibvirtVMDef.CpuModeDef getCpuModeDef() { + return cpuModeDef; + } + + private void extractCpuTuneDef(final Element rootElement) { + NodeList cpuTunesList = rootElement.getElementsByTagName("cputune"); + if (cpuTunesList.getLength() > 0) { + cpuTuneDef = new LibvirtVMDef.CpuTuneDef(); + final Element cpuTuneDefElement = (Element) cpuTunesList.item(0); + final String cpuShares = getTagValue("shares", cpuTuneDefElement); + if (StringUtils.isNotBlank(cpuShares)) { + cpuTuneDef.setShares((Integer.parseInt(cpuShares))); + } + + final String quota = getTagValue("quota", cpuTuneDefElement); + if (StringUtils.isNotBlank(quota)) { + cpuTuneDef.setQuota((Integer.parseInt(quota))); + } + + final String period = getTagValue("period", cpuTuneDefElement); + if (StringUtils.isNotBlank(period)) { + cpuTuneDef.setPeriod((Integer.parseInt(period))); + } + } + } + + private void extractCpuModeDef(final Element rootElement){ + NodeList cpuModeList = rootElement.getElementsByTagName("cpu"); + if (cpuModeList.getLength() > 0){ + cpuModeDef = new LibvirtVMDef.CpuModeDef(); + final Element cpuModeDefElement = (Element) cpuModeList.item(0); + final String cpuModel = getTagValue("model", cpuModeDefElement); + if (StringUtils.isNotBlank(cpuModel)){ + cpuModeDef.setModel(cpuModel); + } + NodeList cpuFeatures = cpuModeDefElement.getElementsByTagName("features"); + if (cpuFeatures.getLength() > 0) { + final ArrayList features = new ArrayList<>(cpuFeatures.getLength()); + for (int i = 0; i < cpuFeatures.getLength(); i++) { + final Element feature = (Element)cpuFeatures.item(i); + final String policy = feature.getAttribute("policy"); + String featureName = feature.getAttribute("name"); + if ("disable".equals(policy)) { + featureName = "-" + featureName; + } + features.add(featureName); + } + cpuModeDef.setFeatures(features); + } + final String sockets = getAttrValue("topology", "sockets", cpuModeDefElement); + final String cores = getAttrValue("topology", "cores", cpuModeDefElement); + if (StringUtils.isNotBlank(sockets) && StringUtils.isNotBlank(cores)) { + cpuModeDef.setTopology(Integer.parseInt(cores), Integer.parseInt(sockets)); + } + } + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index aac44fc1419..6b5fac0e942 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -609,7 +609,7 @@ public class LibvirtVMDef { } } - enum DiskType { + public enum DiskType { FILE("file"), BLOCK("block"), DIRECTROY("dir"), NETWORK("network"); String _diskType; @@ -1072,6 +1072,18 @@ public class LibvirtVMDef { public LibvirtDiskEncryptDetails getLibvirtDiskEncryptDetails() { return this.encryptDetails; } + public String getSourceHost() { + return _sourceHost; + } + + public int getSourceHostPort() { + return _sourcePort; + } + + public String getSourcePath() { + return _sourcePath; + } + @Override public String toString() { StringBuilder diskBuilder = new StringBuilder(); @@ -1737,6 +1749,10 @@ public class LibvirtVMDef { modeBuilder.append(""); return modeBuilder.toString(); } + + public int getCoresPerSocket() { + return _coresPerSocket; + } } public static class SerialDef { @@ -1793,7 +1809,7 @@ public class LibvirtVMDef { public final static class ChannelDef { enum ChannelType { - UNIX("unix"), SERIAL("serial"); + UNIX("unix"), SERIAL("serial"), SPICEVMC("spicevmc"); String type; ChannelType(String type) { @@ -2113,14 +2129,14 @@ public class LibvirtVMDef { private String path = "/dev/random"; private RngModel rngModel = RngModel.VIRTIO; private RngBackendModel rngBackendModel = RngBackendModel.RANDOM; - private int rngRateBytes = 2048; - private int rngRatePeriod = 1000; + private Integer rngRateBytes = 2048; + private Integer rngRatePeriod = 1000; public RngDef(String path) { this.path = path; } - public RngDef(String path, int rngRateBytes, int rngRatePeriod) { + public RngDef(String path, Integer rngRateBytes, Integer rngRatePeriod) { this.path = path; this.rngRateBytes = rngRateBytes; this.rngRatePeriod = rngRatePeriod; @@ -2139,7 +2155,7 @@ public class LibvirtVMDef { this.rngBackendModel = rngBackendModel; } - public RngDef(String path, RngBackendModel rngBackendModel, int rngRateBytes, int rngRatePeriod) { + public RngDef(String path, RngBackendModel rngBackendModel, Integer rngRateBytes, Integer rngRatePeriod) { this.path = path; this.rngBackendModel = rngBackendModel; this.rngRateBytes = rngRateBytes; @@ -2164,11 +2180,11 @@ public class LibvirtVMDef { } public int getRngRateBytes() { - return rngRateBytes; + return rngRateBytes != null ? rngRateBytes : 0; } public int getRngRatePeriod() { - return rngRatePeriod; + return rngRatePeriod != null ? rngRatePeriod : 0; } @Override diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java new file mode 100644 index 00000000000..8b0a5aab461 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -0,0 +1,86 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckVolumeAnswer; +import com.cloud.agent.api.CheckVolumeCommand; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.log4j.Logger; +import org.libvirt.LibvirtException; + +import java.util.Map; + +@ResourceWrapper(handles = CheckVolumeCommand.class) +public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtCheckVolumeCommandWrapper.class); + + @Override + public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + String result = null; + String srcFile = command.getSrcFile(); + StorageFilerTO storageFilerTO = command.getStorageFilerTO(); + KVMStoragePoolManager poolMgr = libvirtComputingResource.getStoragePoolMgr(); + KVMStoragePool pool = poolMgr.getStoragePool(storageFilerTO.getType(), storageFilerTO.getUuid()); + + try { + if (storageFilerTO.getType() == Storage.StoragePoolType.Filesystem || + storageFilerTO.getType() == Storage.StoragePoolType.NetworkFilesystem) { + final KVMPhysicalDisk vol = pool.getPhysicalDisk(srcFile); + final String path = vol.getPath(); + long size = getVirtualSizeFromFile(path); + return new CheckVolumeAnswer(command, "", size); + } else { + return new Answer(command, false, "Unsupported Storage Pool"); + } + + } catch (final Exception e) { + s_logger.error("Error while locating disk: "+ e.getMessage()); + return new Answer(command, false, result); + } + } + + private long getVirtualSizeFromFile(String path) { + try { + QemuImg qemu = new QemuImg(0); + QemuImgFile qemuFile = new QemuImgFile(path); + Map info = qemu.info(qemuFile); + if (info.containsKey(QemuImg.VIRTUAL_SIZE)) { + return Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); + } else { + throw new CloudRuntimeException("Unable to determine virtual size of volume at path " + path); + } + } catch (QemuImgException | LibvirtException ex) { + throw new CloudRuntimeException("Error when inspecting volume at path " + path, ex); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java new file mode 100644 index 00000000000..a26311891c8 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -0,0 +1,400 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.ConvertInstanceAnswer; +import com.cloud.agent.api.ConvertInstanceCommand; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.RemoteInstanceTO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +@ResourceWrapper(handles = ConvertInstanceCommand.class) +public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtConvertInstanceCommandWrapper.class); + + private static final List supportedInstanceConvertSourceHypervisors = + List.of(Hypervisor.HypervisorType.VMware); + + protected static final String checkIfConversionIsSupportedCommand = "which virt-v2v"; + + @Override + public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { + RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); + Hypervisor.HypervisorType sourceHypervisorType = sourceInstance.getHypervisorType(); + String sourceInstanceName = sourceInstance.getInstanceName(); + Hypervisor.HypervisorType destinationHypervisorType = cmd.getDestinationHypervisorType(); + List destinationStoragePools = cmd.getDestinationStoragePools(); + DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation(); + long timeout = (long) cmd.getWait() * 1000; + + if (!isInstanceConversionSupportedOnHost()) { + String msg = String.format("Cannot convert the instance %s from VMware as the virt-v2v binary is not found. " + + "Please install virt-v2v on the host before attempting the instance conversion", sourceInstanceName); + s_logger.info(msg); + return new ConvertInstanceAnswer(cmd, false, msg); + } + + if (!areSourceAndDestinationHypervisorsSupported(sourceHypervisorType, destinationHypervisorType)) { + String err = destinationHypervisorType != Hypervisor.HypervisorType.KVM ? + String.format("The destination hypervisor type is %s, KVM was expected, cannot handle it", destinationHypervisorType) : + String.format("The source hypervisor type %s is not supported for KVM conversion", sourceHypervisorType); + s_logger.error(err); + return new ConvertInstanceAnswer(cmd, false, err); + } + + final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + KVMStoragePool temporaryStoragePool = getTemporaryStoragePool(conversionTemporaryLocation, storagePoolMgr); + + s_logger.info(String.format("Attempting to convert the instance %s from %s to KVM", + sourceInstanceName, sourceHypervisorType)); + final String convertInstanceUrl = getConvertInstanceUrl(sourceInstance); + final String temporaryConvertUuid = UUID.randomUUID().toString(); + final String temporaryPasswordFilePath = createTemporaryPasswordFileAndRetrievePath(sourceInstance); + final String temporaryConvertPath = temporaryStoragePool.getLocalPath(); + boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); + + try { + boolean result = performInstanceConversion(convertInstanceUrl, sourceInstanceName, temporaryPasswordFilePath, + temporaryConvertPath, temporaryConvertUuid, timeout, verboseModeEnabled); + if (!result) { + String err = String.format("The virt-v2v conversion of the instance %s failed. " + + "Please check the agent logs for the virt-v2v output", sourceInstanceName); + s_logger.error(err); + return new ConvertInstanceAnswer(cmd, false, err); + } + String convertedBasePath = String.format("%s/%s", temporaryConvertPath, temporaryConvertUuid); + LibvirtDomainXMLParser xmlParser = parseMigratedVMXmlDomain(convertedBasePath); + + List temporaryDisks = xmlParser == null ? + getTemporaryDisksWithPrefixFromTemporaryPool(temporaryStoragePool, temporaryConvertPath, temporaryConvertUuid) : + getTemporaryDisksFromParsedXml(temporaryStoragePool, xmlParser, convertedBasePath); + + List destinationDisks = moveTemporaryDisksToDestination(temporaryDisks, + destinationStoragePools, storagePoolMgr); + + cleanupDisksAndDomainFromTemporaryLocation(temporaryDisks, temporaryStoragePool, temporaryConvertUuid); + + UnmanagedInstanceTO convertedInstanceTO = getConvertedUnmanagedInstance(temporaryConvertUuid, + destinationDisks, xmlParser); + return new ConvertInstanceAnswer(cmd, convertedInstanceTO); + } catch (Exception e) { + String error = String.format("Error converting instance %s from %s, due to: %s", + sourceInstanceName, sourceHypervisorType, e.getMessage()); + s_logger.error(error, e); + return new ConvertInstanceAnswer(cmd, false, error); + } finally { + s_logger.debug("Cleaning up instance conversion temporary password file"); + Script.runSimpleBashScript(String.format("rm -rf %s", temporaryPasswordFilePath)); + if (conversionTemporaryLocation instanceof NfsTO) { + s_logger.debug("Cleaning up secondary storage temporary location"); + storagePoolMgr.deleteStoragePool(temporaryStoragePool.getType(), temporaryStoragePool.getUuid()); + } + } + } + + protected KVMStoragePool getTemporaryStoragePool(DataStoreTO conversionTemporaryLocation, KVMStoragePoolManager storagePoolMgr) { + if (conversionTemporaryLocation instanceof NfsTO) { + NfsTO nfsTO = (NfsTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePoolByURI(nfsTO.getUrl()); + } else { + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + } + } + + protected boolean areSourceAndDestinationHypervisorsSupported(Hypervisor.HypervisorType sourceHypervisorType, + Hypervisor.HypervisorType destinationHypervisorType) { + return destinationHypervisorType == Hypervisor.HypervisorType.KVM && + supportedInstanceConvertSourceHypervisors.contains(sourceHypervisorType); + } + + protected List getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { + List disksDefs = xmlParser.getDisks(); + disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && + x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksDefs)) { + String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + sanitizeDisksPath(disksDefs); + return getPhysicalDisksFromDefPaths(disksDefs, pool); + } + + private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { + List disks = new ArrayList<>(); + for (LibvirtVMDef.DiskDef diskDef : disksDefs) { + KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); + disks.add(physicalDisk); + } + return disks; + } + + protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { + String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); + s_logger.info(msg); + pool.refresh(); + List disksWithPrefix = pool.listPhysicalDisks() + .stream() + .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksWithPrefix)) { + msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + return disksWithPrefix; + } + + private void cleanupDisksAndDomainFromTemporaryLocation(List disks, + KVMStoragePool temporaryStoragePool, + String temporaryConvertUuid) { + for (KVMPhysicalDisk disk : disks) { + s_logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); + temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); + } + s_logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); + Script.runSimpleBashScript(String.format("rm -f %s/%s*.xml", temporaryStoragePool.getLocalPath(), temporaryConvertUuid)); + } + + protected boolean isInstanceConversionSupportedOnHost() { + int exitValue = Script.runSimpleBashScriptForExitValue(checkIfConversionIsSupportedCommand); + return exitValue == 0; + } + + protected void sanitizeDisksPath(List disks) { + for (LibvirtVMDef.DiskDef disk : disks) { + String[] diskPathParts = disk.getDiskPath().split("/"); + String relativePath = diskPathParts[diskPathParts.length - 1]; + disk.setDiskPath(relativePath); + } + } + + protected List moveTemporaryDisksToDestination(List temporaryDisks, + List destinationStoragePools, + KVMStoragePoolManager storagePoolMgr) { + List targetDisks = new ArrayList<>(); + if (temporaryDisks.size() != destinationStoragePools.size()) { + String warn = String.format("Discrepancy between the converted instance disks (%s) " + + "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); + s_logger.warn(warn); + } + for (int i = 0; i < temporaryDisks.size(); i++) { + String poolPath = destinationStoragePools.get(i); + KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); + if (destinationPool == null) { + String err = String.format("Could not find a storage pool by URI: %s", poolPath); + s_logger.error(err); + continue; + } + KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); + if (s_logger.isDebugEnabled()) { + String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + + " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); + s_logger.debug(msg); + } + + String destinationName = UUID.randomUUID().toString(); + + KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); + targetDisks.add(destinationDisk); + } + return targetDisks; + } + + private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, + List vmDisks, + LibvirtDomainXMLParser xmlParser) { + UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); + instanceTO.setName(baseName); + instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); + instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); + return instanceTO; + } + + private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { + List nics = new ArrayList<>(); + if (xmlParser != null) { + List interfaces = xmlParser.getInterfaces(); + for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { + UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); + nic.setMacAddress(interfaceDef.getMacAddress()); + nic.setNicId(interfaceDef.getBrName()); + nic.setAdapterType(interfaceDef.getModel().toString()); + nics.add(nic); + } + } + return nics; + } + + protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { + List instanceDisks = new ArrayList<>(); + List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; + for (int i = 0; i< vmDisks.size(); i++) { + KVMPhysicalDisk physicalDisk = vmDisks.get(i); + KVMStoragePool storagePool = physicalDisk.getPool(); + UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); + disk.setPosition(i); + Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); + disk.setDatastoreHost(storagePoolHostAndPath.first()); + disk.setDatastorePath(storagePoolHostAndPath.second()); + disk.setDatastoreName(storagePool.getUuid()); + disk.setDatastoreType(storagePool.getType().name()); + disk.setCapacity(physicalDisk.getVirtualSize()); + disk.setFileBaseName(physicalDisk.getName()); + if (CollectionUtils.isNotEmpty(diskDefs)) { + LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); + disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } else { + // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver + disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } + instanceDisks.add(disk); + } + return instanceDisks; + } + + protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { + String sourceHostIp = null; + String sourcePath = null; + String storagePoolMountPoint = Script.runSimpleBashScript(String.format("mount | grep %s", storagePool.getLocalPath())); + if (StringUtils.isNotEmpty(storagePoolMountPoint)) { + String[] res = storagePoolMountPoint.strip().split(" "); + res = res[0].split(":"); + sourceHostIp = res[0].strip(); + sourcePath = res[1].strip(); + } + return new Pair<>(sourceHostIp, sourcePath); + } + + protected boolean performInstanceConversion(String convertInstanceUrl, String sourceInstanceName, + String temporaryPasswordFilePath, + String temporaryConvertFolder, + String temporaryConvertUuid, + long timeout, boolean verboseModeEnabled) { + Script script = new Script("virt-v2v", timeout, s_logger); + script.add("--root", "first"); + script.add("-ic", convertInstanceUrl); + script.add(sourceInstanceName); + script.add("--password-file", temporaryPasswordFilePath); + script.add("-o", "local"); + script.add("-os", temporaryConvertFolder); + script.add("-of", "qcow2"); + script.add("-on", temporaryConvertUuid); + if (verboseModeEnabled) { + script.add("-v"); + } + + String logPrefix = String.format("virt-v2v source: %s %s progress", convertInstanceUrl, sourceInstanceName); + OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(s_logger, logPrefix); + script.execute(outputLogger); + int exitValue = script.getExitValue(); + return exitValue == 0; + } + + private String createTemporaryPasswordFileAndRetrievePath(RemoteInstanceTO sourceInstance) { + String password = null; + if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + password = sourceInstance.getVcenterPassword(); + } + String passwordFile = String.format("/tmp/vmw-%s", UUID.randomUUID()); + String msg = String.format("Creating a temporary password file for VMware instance %s conversion on: %s", sourceInstance.getInstanceName(), passwordFile); + s_logger.debug(msg); + Script.runSimpleBashScriptForExitValueAvoidLogging(String.format("echo \"%s\" > %s", password, passwordFile)); + return passwordFile; + } + + private String getConvertInstanceUrl(RemoteInstanceTO sourceInstance) { + String url = null; + if (sourceInstance.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + url = getConvertInstanceUrlFromVmware(sourceInstance); + } + return url; + } + + private String getConvertInstanceUrlFromVmware(RemoteInstanceTO vmwareInstance) { + String vcenter = vmwareInstance.getVcenterHost(); + String datacenter = vmwareInstance.getDatacenterName(); + String username = vmwareInstance.getVcenterUsername(); + String host = vmwareInstance.getHostName(); + String cluster = vmwareInstance.getClusterName(); + + String encodedUsername = encodeUsername(username); + return String.format("vpx://%s@%s/%s/%s/%s?no_verify=1", + encodedUsername, vcenter, datacenter, cluster, host); + } + protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException { + String xmlPath = String.format("%s.xml", installPath); + if (!new File(xmlPath).exists()) { + String err = String.format("Conversion failed. Unable to find the converted XML domain, expected %s", xmlPath); + s_logger.error(err); + throw new CloudRuntimeException(err); + } + InputStream is = new BufferedInputStream(new FileInputStream(xmlPath)); + String xml = IOUtils.toString(is, Charset.defaultCharset()); + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + try { + parser.parseDomainXML(xml); + return parser; + } catch (RuntimeException e) { + String err = String.format("Error parsing the converted instance XML domain at %s: %s", xmlPath, e.getMessage()); + s_logger.error(err, e); + s_logger.debug(xml); + return null; + } + } + + protected String encodeUsername(String username) { + return URLEncoder.encode(username, Charset.defaultCharset()); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyRemoteVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyRemoteVolumeCommandWrapper.java new file mode 100644 index 00000000000..e48edd8eec0 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCopyRemoteVolumeCommandWrapper.java @@ -0,0 +1,93 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CopyRemoteVolumeAnswer; +import com.cloud.agent.api.CopyRemoteVolumeCommand; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.log4j.Logger; +import org.libvirt.LibvirtException; + +import java.util.Map; + +@ResourceWrapper(handles = CopyRemoteVolumeCommand.class) +public final class LibvirtCopyRemoteVolumeCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtCopyRemoteVolumeCommandWrapper.class); + + @Override + public Answer execute(final CopyRemoteVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { + String result = null; + String srcIp = command.getRemoteIp(); + String username = command.getUsername(); + String password = command.getPassword(); + String srcFile = command.getSrcFile(); + StorageFilerTO storageFilerTO = command.getStorageFilerTO(); + String tmpPath = command.getTmpPath(); + KVMStoragePoolManager poolMgr = libvirtComputingResource.getStoragePoolMgr(); + KVMStoragePool pool = poolMgr.getStoragePool(storageFilerTO.getType(), storageFilerTO.getUuid()); + String dstPath = pool.getLocalPath(); + + try { + if (storageFilerTO.getType() == Storage.StoragePoolType.Filesystem || + storageFilerTO.getType() == Storage.StoragePoolType.NetworkFilesystem) { + String filename = libvirtComputingResource.copyVolume(srcIp, username, password, dstPath, srcFile, tmpPath); + s_logger.debug("Volume Copy Successful"); + final KVMPhysicalDisk vol = pool.getPhysicalDisk(filename); + final String path = vol.getPath(); + long size = getVirtualSizeFromFile(path); + return new CopyRemoteVolumeAnswer(command, "", filename, size); + } else { + return new Answer(command, false, "Unsupported Storage Pool"); + } + + } catch (final Exception e) { + s_logger.error("Error while copying file from remote host: "+ e.getMessage()); + return new Answer(command, false, result); + } + } + + private long getVirtualSizeFromFile(String path) { + try { + QemuImg qemu = new QemuImg(0); + QemuImgFile qemuFile = new QemuImgFile(path); + Map info = qemu.info(qemuFile); + if (info.containsKey(QemuImg.VIRTUAL_SIZE)) { + return Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); + } else { + throw new CloudRuntimeException("Unable to determine virtual size of volume at path " + path); + } + } catch (QemuImgException | LibvirtException ex) { + throw new CloudRuntimeException("Error when inspecting volume at path " + path, ex); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java new file mode 100644 index 00000000000..700f058b59b --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java @@ -0,0 +1,194 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.GetRemoteVmsAnswer; +import com.cloud.agent.api.GetRemoteVmsCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtConnection; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.log4j.Logger; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainBlockInfo; +import org.libvirt.DomainInfo; +import org.libvirt.LibvirtException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +@ResourceWrapper(handles = GetRemoteVmsCommand.class) +public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtGetRemoteVmsCommandWrapper.class); + + @Override + public Answer execute(final GetRemoteVmsCommand command, final LibvirtComputingResource libvirtComputingResource) { + String result = null; + String hypervisorURI = "qemu+tcp://" + command.getRemoteIp() + + "/system"; + HashMap unmanagedInstances = new HashMap<>(); + try { + Connect conn = LibvirtConnection.getConnection(hypervisorURI); + final List allVmNames = libvirtComputingResource.getAllVmNames(conn); + for (String name : allVmNames) { + final Domain domain = libvirtComputingResource.getDomain(conn, name); + + final DomainInfo.DomainState ps = domain.getInfo().state; + + final VirtualMachine.PowerState state = libvirtComputingResource.convertToPowerState(ps); + + s_logger.debug("VM " + domain.getName() + ": powerstate = " + ps + "; vm state=" + state.toString()); + + if (state == VirtualMachine.PowerState.PowerOff) { + try { + UnmanagedInstanceTO instance = getUnmanagedInstance(libvirtComputingResource, domain, conn); + unmanagedInstances.put(instance.getName(), instance); + } catch (Exception e) { + s_logger.error("Error while fetching instance details", e); + } + } + domain.free(); + } + s_logger.debug("Found Vms: "+ unmanagedInstances.size()); + return new GetRemoteVmsAnswer(command, "", unmanagedInstances); + } catch (final LibvirtException e) { + s_logger.error("Error while listing stopped Vms on remote host: "+ e.getMessage()); + return new Answer(command, false, result); + } + } + + private UnmanagedInstanceTO getUnmanagedInstance(LibvirtComputingResource libvirtComputingResource, Domain domain, Connect conn) { + try { + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + parser.parseDomainXML(domain.getXMLDesc(1)); + + final UnmanagedInstanceTO instance = new UnmanagedInstanceTO(); + instance.setName(domain.getName()); + if (parser.getCpuModeDef() != null) { + instance.setCpuCoresPerSocket(parser.getCpuModeDef().getCoresPerSocket()); + } + Long memory = domain.getMaxMemory(); + instance.setMemory(memory.intValue()/1024); + if (parser.getCpuTuneDef() !=null) { + instance.setCpuSpeed(parser.getCpuTuneDef().getShares()); + } + instance.setPowerState(getPowerState(libvirtComputingResource.getVmState(conn,domain.getName()))); + instance.setNics(getUnmanagedInstanceNics(parser.getInterfaces())); + instance.setDisks(getUnmanagedInstanceDisks(parser.getDisks(),libvirtComputingResource, domain)); + instance.setVncPassword(parser.getVncPasswd() + "aaaaaaaaaaaaaa"); // Suffix back extra characters for DB compatibility + + return instance; + } catch (Exception e) { + s_logger.debug("Unable to retrieve unmanaged instance info. ", e); + throw new CloudRuntimeException("Unable to retrieve unmanaged instance info. " + e.getMessage()); + } + } + + private UnmanagedInstanceTO.PowerState getPowerState(VirtualMachine.PowerState vmPowerState) { + switch (vmPowerState) { + case PowerOn: + return UnmanagedInstanceTO.PowerState.PowerOn; + case PowerOff: + return UnmanagedInstanceTO.PowerState.PowerOff; + default: + return UnmanagedInstanceTO.PowerState.PowerUnknown; + + } + } + + private List getUnmanagedInstanceNics(List interfaces) { + final ArrayList nics = new ArrayList<>(interfaces.size()); + int counter = 0; + for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { + final UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); + nic.setNicId(String.valueOf(counter++)); + nic.setMacAddress(interfaceDef.getMacAddress()); + nic.setAdapterType(interfaceDef.getModel().toString()); + nic.setNetwork(interfaceDef.getDevName()); + nic.setPciSlot(interfaceDef.getSlot().toString()); + nic.setVlan(interfaceDef.getVlanTag()); + nics.add(nic); + } + return nics; + } + + private List getUnmanagedInstanceDisks(List disksInfo, + LibvirtComputingResource libvirtComputingResource, + Domain dm){ + final ArrayList disks = new ArrayList<>(disksInfo.size()); + int counter = 0; + for (LibvirtVMDef.DiskDef diskDef : disksInfo) { + if (diskDef.getDeviceType() != LibvirtVMDef.DiskDef.DeviceType.DISK) { + continue; + } + + final UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); + + disk.setPosition(counter); + + Long size; + try { + DomainBlockInfo blockInfo = dm.blockInfo(diskDef.getSourcePath()); + size = blockInfo.getCapacity(); + } catch (LibvirtException e) { + throw new RuntimeException(e); + } + + disk.setCapacity(size); + disk.setDiskId(String.valueOf(counter++)); + disk.setLabel(diskDef.getDiskLabel()); + disk.setController(diskDef.getBusType().toString()); + + + Pair sourceHostPath = getSourceHostPath(libvirtComputingResource, diskDef.getSourcePath()); + if (sourceHostPath != null) { + disk.setDatastoreHost(sourceHostPath.first()); + disk.setDatastorePath(sourceHostPath.second()); + } else { + disk.setDatastorePath(diskDef.getSourcePath()); + disk.setDatastoreHost(diskDef.getSourceHost()); + } + + disk.setDatastoreType(diskDef.getDiskType().toString()); + disk.setDatastorePort(diskDef.getSourceHostPort()); + disks.add(disk); + } + return disks; + } + + private Pair getSourceHostPath(LibvirtComputingResource libvirtComputingResource, String diskPath) { + int pathEnd = diskPath.lastIndexOf("/"); + if (pathEnd >= 0) { + diskPath = diskPath.substring(0, pathEnd); + return libvirtComputingResource.getSourceHostPath(diskPath); + } + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java new file mode 100644 index 00000000000..a2d84063d74 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.GetUnmanagedInstancesAnswer; +import com.cloud.agent.api.GetUnmanagedInstancesCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@ResourceWrapper(handles=GetUnmanagedInstancesCommand.class) +public final class LibvirtGetUnmanagedInstancesCommandWrapper extends CommandWrapper { + private static final Logger LOGGER = Logger.getLogger(LibvirtGetUnmanagedInstancesCommandWrapper.class); + + @Override + public GetUnmanagedInstancesAnswer execute(GetUnmanagedInstancesCommand command, LibvirtComputingResource libvirtComputingResource) { + LOGGER.info("Fetching unmanaged instance on host"); + + HashMap unmanagedInstances = new HashMap<>(); + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + final Connect conn = libvirtUtilitiesHelper.getConnection(); + final List domains = getDomains(command, libvirtComputingResource, conn); + + for (Domain domain : domains) { + UnmanagedInstanceTO instance = getUnmanagedInstance(libvirtComputingResource, domain, conn); + if (instance != null) { + unmanagedInstances.put(instance.getName(), instance); + domain.free(); + } + } + } catch (Exception e) { + String err = String.format("Error listing unmanaged instances: %s", e.getMessage()); + LOGGER.error(err, e); + return new GetUnmanagedInstancesAnswer(command, err); + } + + return new GetUnmanagedInstancesAnswer(command, "OK", unmanagedInstances); + } + + private List getDomains(GetUnmanagedInstancesCommand command, + LibvirtComputingResource libvirtComputingResource, + Connect conn) throws LibvirtException, CloudRuntimeException { + final List domains = new ArrayList<>(); + final String vmNameCmd = command.getInstanceName(); + if (StringUtils.isNotBlank(vmNameCmd)) { + final Domain domain = libvirtComputingResource.getDomain(conn, vmNameCmd); + if (domain == null) { + String msg = String.format("VM %s not found", vmNameCmd); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + + checkIfVmExists(vmNameCmd,domain); + checkIfVmIsManaged(command,vmNameCmd,domain); + + domains.add(domain); + } else { + final List allVmNames = libvirtComputingResource.getAllVmNames(conn); + for (String name : allVmNames) { + if (!command.hasManagedInstance(name)) { + final Domain domain = libvirtComputingResource.getDomain(conn, name); + domains.add(domain); + } + } + } + return domains; + } + + private void checkIfVmExists(String vmNameCmd,final Domain domain) throws LibvirtException { + if (StringUtils.isNotEmpty(vmNameCmd) && + !vmNameCmd.equals(domain.getName())) { + LOGGER.error("GetUnmanagedInstancesCommand: exact vm name not found " + vmNameCmd); + throw new CloudRuntimeException("GetUnmanagedInstancesCommand: exact vm name not found " + vmNameCmd); + } + } + + private void checkIfVmIsManaged(GetUnmanagedInstancesCommand command,String vmNameCmd,final Domain domain) throws LibvirtException { + if (command.hasManagedInstance(domain.getName())) { + LOGGER.error("GetUnmanagedInstancesCommand: vm already managed " + vmNameCmd); + throw new CloudRuntimeException("GetUnmanagedInstancesCommand: vm already managed " + vmNameCmd); + } + } + private UnmanagedInstanceTO getUnmanagedInstance(LibvirtComputingResource libvirtComputingResource, Domain domain, Connect conn) { + try { + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + parser.parseDomainXML(domain.getXMLDesc(1)); + + final UnmanagedInstanceTO instance = new UnmanagedInstanceTO(); + instance.setName(domain.getName()); + + instance.setCpuCores((int) LibvirtComputingResource.countDomainRunningVcpus(domain)); + instance.setCpuSpeed(parser.getCpuTuneDef().getShares()/instance.getCpuCores()); + + if (parser.getCpuModeDef() != null) { + instance.setCpuCoresPerSocket(parser.getCpuModeDef().getCoresPerSocket()); + } + instance.setPowerState(getPowerState(libvirtComputingResource.getVmState(conn,domain.getName()))); + instance.setMemory((int) LibvirtComputingResource.getDomainMemory(domain) / 1024); + instance.setNics(getUnmanagedInstanceNics(parser.getInterfaces())); + instance.setDisks(getUnmanagedInstanceDisks(parser.getDisks(),libvirtComputingResource)); + instance.setVncPassword(parser.getVncPasswd() + "aaaaaaaaaaaaaa"); // Suffix back extra characters for DB compatibility + + return instance; + } catch (Exception e) { + LOGGER.info("Unable to retrieve unmanaged instance info. " + e.getMessage(), e); + return null; + } + } + + private UnmanagedInstanceTO.PowerState getPowerState(VirtualMachine.PowerState vmPowerState) { + switch (vmPowerState) { + case PowerOn: + return UnmanagedInstanceTO.PowerState.PowerOn; + case PowerOff: + return UnmanagedInstanceTO.PowerState.PowerOff; + default: + return UnmanagedInstanceTO.PowerState.PowerUnknown; + + } + } + + private List getUnmanagedInstanceNics(List interfaces) { + final ArrayList nics = new ArrayList<>(interfaces.size()); + int counter = 0; + for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { + final UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); + nic.setNicId(String.valueOf(counter++)); + nic.setMacAddress(interfaceDef.getMacAddress()); + nic.setAdapterType(interfaceDef.getModel().toString()); + nic.setNetwork(interfaceDef.getDevName()); + nic.setPciSlot(interfaceDef.getSlot().toString()); + nic.setVlan(interfaceDef.getVlanTag()); + nics.add(nic); + } + return nics; + } + + private List getUnmanagedInstanceDisks(List disksInfo, LibvirtComputingResource libvirtComputingResource){ + final ArrayList disks = new ArrayList<>(disksInfo.size()); + int counter = 0; + for (LibvirtVMDef.DiskDef diskDef : disksInfo) { + if (diskDef.getDeviceType() != LibvirtVMDef.DiskDef.DeviceType.DISK) { + continue; + } + + final UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); + Long size = null; + String imagePath = null; + try { + QemuImgFile file = new QemuImgFile(diskDef.getSourcePath()); + QemuImg qemu = new QemuImg(0); + Map info = qemu.info(file); + size = Long.parseLong(info.getOrDefault("virtual_size", "0")); + imagePath = info.getOrDefault("image", null); + } catch (QemuImgException | LibvirtException e) { + throw new RuntimeException(e); + } + + disk.setPosition(counter); + disk.setCapacity(size); + disk.setDiskId(String.valueOf(counter++)); + disk.setLabel(diskDef.getDiskLabel()); + disk.setController(diskDef.getBusType().toString()); + + + Pair sourceHostPath = getSourceHostPath(libvirtComputingResource, diskDef.getSourcePath()); + if (sourceHostPath != null) { + disk.setDatastoreHost(sourceHostPath.first()); + disk.setDatastorePath(sourceHostPath.second()); + } else { + disk.setDatastorePath(diskDef.getSourcePath()); + disk.setDatastoreHost(diskDef.getSourceHost()); + } + + disk.setDatastoreType(diskDef.getDiskType().toString()); + disk.setDatastorePort(diskDef.getSourceHostPort()); + disk.setImagePath(imagePath); + disk.setDatastoreName(imagePath.substring(imagePath.lastIndexOf("/"))); + disks.add(disk); + } + return disks; + } + + private Pair getSourceHostPath(LibvirtComputingResource libvirtComputingResource, String diskPath) { + int pathEnd = diskPath.lastIndexOf("/"); + if (pathEnd >= 0) { + diskPath = diskPath.substring(0, pathEnd); + return libvirtComputingResource.getSourceHostPath(diskPath); + } + return null; + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index d0ab77829af..fb526626ef8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -23,6 +23,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Set; @@ -211,6 +212,8 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper + *
  • + * If both hosts utilize cgroup v1; then, the shares value of the VM is equal in both hosts, and there is no need to update the VM CPU shares value for the + * migration.
    • + *
  • + * If, at least, one of the hosts utilize cgroup v2, the VM CPU shares must be recalculated for the migration, accordingly to + * method {@link LibvirtComputingResource#calculateCpuShares(VirtualMachineTO)}. + *
    • + * + */ + protected String updateVmSharesIfNeeded(MigrateCommand migrateCommand, String xmlDesc, LibvirtComputingResource libvirtComputingResource) + throws ParserConfigurationException, IOException, SAXException, TransformerException { + Integer newVmCpuShares = migrateCommand.getNewVmCpuShares(); + int currentCpuShares = libvirtComputingResource.calculateCpuShares(migrateCommand.getVirtualMachine()); + + if (newVmCpuShares == currentCpuShares) { + s_logger.info(String.format("Current CPU shares [%s] is equal in both hosts; therefore, there is no need to update the CPU shares for the new host.", + currentCpuShares)); + return xmlDesc; + } + + InputStream inputStream = IOUtils.toInputStream(xmlDesc, StandardCharsets.UTF_8); + DocumentBuilderFactory docFactory = ParserUtils.getSaferDocumentBuilderFactory(); + DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); + Document document = docBuilder.parse(inputStream); + + Element root = document.getDocumentElement(); + Node sharesNode = root.getElementsByTagName("shares").item(0); + String currentShares = sharesNode.getTextContent(); + + s_logger.info(String.format("VM [%s] will have CPU shares altered from [%s] to [%s] as part of migration because the cgroups version differs between hosts.", + migrateCommand.getVmName(), currentShares, newVmCpuShares)); + sharesNode.setTextContent(String.valueOf(newVmCpuShares)); + return getXml(document); + } + /** * Replace DPDK source path and target before migrations */ diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java index 5c893e5d12f..2a09c340891 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateVolumeCommandWrapper.java @@ -279,6 +279,10 @@ public class LibvirtMigrateVolumeCommandWrapper extends CommandWrapper srcDetails = command.getSrcDetails(); String srcPath = srcDetails != null ? srcDetails.get(DiskTO.IQN) : srcVolumeObjectTO.getPath(); + // its possible a volume has details but is not using IQN addressing... + if (srcPath == null) { + srcPath = srcVolumeObjectTO.getPath(); + } VolumeObjectTO destVolumeObjectTO = (VolumeObjectTO)command.getDestData(); PrimaryDataStoreTO destPrimaryDataStore = (PrimaryDataStoreTO)destVolumeObjectTO.getDataStore(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java index ec9e67e894c..6292ca71c2e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareForMigrationCommandWrapper.java @@ -125,11 +125,7 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp return new PrepareForMigrationAnswer(command, "failed to connect physical disks to host"); } - PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command); - if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { - answer.setDpdkInterfaceMapping(dpdkInterfaceMapping); - } - return answer; + return createPrepareForMigrationAnswer(command, dpdkInterfaceMapping, libvirtComputingResource, vm); } catch (final LibvirtException | CloudRuntimeException | InternalErrorException | URISyntaxException e) { if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { for (DpdkTO to : dpdkInterfaceMapping.values()) { @@ -146,6 +142,22 @@ public final class LibvirtPrepareForMigrationCommandWrapper extends CommandWrapp } } + protected PrepareForMigrationAnswer createPrepareForMigrationAnswer(PrepareForMigrationCommand command, Map dpdkInterfaceMapping, + LibvirtComputingResource libvirtComputingResource, VirtualMachineTO vm) { + PrepareForMigrationAnswer answer = new PrepareForMigrationAnswer(command); + + if (MapUtils.isNotEmpty(dpdkInterfaceMapping)) { + s_logger.debug(String.format("Setting DPDK interface for the migration of VM [%s].", vm)); + answer.setDpdkInterfaceMapping(dpdkInterfaceMapping); + } + + int newCpuShares = libvirtComputingResource.calculateCpuShares(vm); + s_logger.debug(String.format("Setting CPU shares to [%s] for the migration of VM [%s].", newCpuShares, vm)); + answer.setNewVmCpuShares(newCpuShares); + + return answer; + } + private Answer handleRollback(PrepareForMigrationCommand command, LibvirtComputingResource libvirtComputingResource) { KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); VirtualMachineTO vmTO = command.getVirtualMachine(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareUnmanageVMInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareUnmanageVMInstanceCommandWrapper.java new file mode 100644 index 00000000000..68373089038 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPrepareUnmanageVMInstanceCommandWrapper.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.PrepareUnmanageVMInstanceAnswer; +import com.cloud.agent.api.PrepareUnmanageVMInstanceCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import org.apache.log4j.Logger; +import org.libvirt.Connect; +import org.libvirt.Domain; + +@ResourceWrapper(handles=PrepareUnmanageVMInstanceCommand.class) +public final class LibvirtPrepareUnmanageVMInstanceCommandWrapper extends CommandWrapper { + private static final Logger LOGGER = Logger.getLogger(LibvirtPrepareUnmanageVMInstanceCommandWrapper.class); + @Override + public PrepareUnmanageVMInstanceAnswer execute(PrepareUnmanageVMInstanceCommand command, LibvirtComputingResource libvirtComputingResource) { + final String vmName = command.getInstanceName(); + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + LOGGER.debug(String.format("Verify if KVM instance: [%s] is available before Unmanaging VM.", vmName)); + try { + final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); + final Domain domain = libvirtComputingResource.getDomain(conn, vmName); + if (domain == null) { + LOGGER.error("Prepare Unmanage VMInstanceCommand: vm not found " + vmName); + new PrepareUnmanageVMInstanceAnswer(command, false, String.format("Cannot find VM with name [%s] in KVM host.", vmName)); + } + } catch (Exception e){ + LOGGER.error("PrepareUnmanagedInstancesCommand failed due to " + e.getMessage()); + return new PrepareUnmanageVMInstanceAnswer(command, false, "Error: " + e.getMessage()); + } + + return new PrepareUnmanageVMInstanceAnswer(command, true, "OK"); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 36ff69d83af..4f1ad728b5d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -50,6 +50,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.hypervisor.kvm.storage.MultipathSCSIPool; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage.StoragePoolType; @@ -84,6 +85,10 @@ public final class LibvirtResizeVolumeCommandWrapper extends CommandWrapper; connid= + String type = null; + String address = null; + String connectionId = null; + String path = null; + String[] parts = inPath.split(";"); + // handle initial code of wwn only + if (parts.length == 1) { + type = "FIBERWWN"; + address = parts[0]; + } else { + for (String part: parts) { + String[] pair = part.split("="); + if (pair.length == 2) { + String key = pair[0].trim(); + String value = pair[1].trim(); + if (key.equals("type")) { + type = value.toUpperCase(); + } else if (key.equals("address")) { + address = value; + } else if (key.equals("connid")) { + connectionId = value; + } + } + } + } + + if ("FIBERWWN".equals(type)) { + path = "/dev/mapper/3" + address; + } else { + throw new CloudRuntimeException("Invalid address type provided for target disk: " + type); + } + + return new AddressInfo(type, address, connectionId, path); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index dd31025d35f..1be4a8b6185 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -290,9 +290,12 @@ public class KVMStorageProcessor implements StorageProcessor { final TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(primaryVol.getName()); newTemplate.setSize(primaryVol.getSize()); - if (primaryPool.getType() == StoragePoolType.RBD || - primaryPool.getType() == StoragePoolType.PowerFlex || - primaryPool.getType() == StoragePoolType.Linstor) { + + if(List.of( + StoragePoolType.RBD, + StoragePoolType.PowerFlex, + StoragePoolType.Linstor, + StoragePoolType.FiberChannel).contains(primaryPool.getType())) { newTemplate.setFormat(ImageFormat.RAW); } else { newTemplate.setFormat(ImageFormat.QCOW2); @@ -584,7 +587,9 @@ public class KVMStorageProcessor implements StorageProcessor { public Answer createTemplateFromVolume(final CopyCommand cmd) { Map details = cmd.getOptions(); - if (details != null && details.get(DiskTO.IQN) != null) { + // handle cases where the managed storage driver had to make a temporary volume from + // the snapshot in order to support the copy + if (details != null && (details.get(DiskTO.IQN) != null || details.get(DiskTO.PATH) != null)) { // use the managed-storage approach return createTemplateFromVolumeOrSnapshot(cmd); } @@ -712,7 +717,7 @@ public class KVMStorageProcessor implements StorageProcessor { public Answer createTemplateFromSnapshot(CopyCommand cmd) { Map details = cmd.getOptions(); - if (details != null && details.get(DiskTO.IQN) != null) { + if (details != null && (details.get(DiskTO.IQN) != null || details.get(DiskTO.PATH) != null)) { // use the managed-storage approach return createTemplateFromVolumeOrSnapshot(cmd); } @@ -750,12 +755,15 @@ public class KVMStorageProcessor implements StorageProcessor { KVMStoragePool secondaryStorage = null; try { + // look for options indicating an overridden path or IQN. Used when snapshots have to be + // temporarily copied on the manaaged storage device before the actual copy to target object Map details = cmd.getOptions(); - - String path = details != null ? details.get(DiskTO.IQN) : null; - + String path = details != null ? details.get(DiskTO.PATH) : null; if (path == null) { - new CloudRuntimeException("The 'path' field must be specified."); + path = details != null ? details.get(DiskTO.IQN) : null; + if (path == null) { + new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + } } storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details); @@ -2188,7 +2196,16 @@ public class KVMStorageProcessor implements StorageProcessor { Map details = cmd.getOptions2(); - String path = details != null ? details.get(DiskTO.IQN) : null; + String path = cmd.getDestTO().getPath(); + if (path == null) { + path = details != null ? details.get(DiskTO.PATH) : null; + if (path == null) { + path = details != null ? details.get(DiskTO.IQN) : null; + if (path == null) { + new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + } + } + } storagePoolMgr.connectPhysicalDisk(pool.getPoolType(), pool.getUuid(), path, details); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java new file mode 100644 index 00000000000..06dea46a98d --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -0,0 +1,758 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.log4j.Logger; + +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; +import org.libvirt.LibvirtException; +import org.joda.time.Duration; + +public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { + static final Logger LOGGER = Logger.getLogger(MultipathSCSIAdapterBase.class); + static final Map MapStorageUuidToStoragePool = new HashMap<>(); + + /** + * A lock to avoid any possiblity of multiple requests for a scan + */ + static byte[] CLEANUP_LOCK = new byte[0]; + + /** + * Property keys and defaults + */ + static final Property CLEANUP_FREQUENCY_SECS = new Property("multimap.cleanup.frequency.secs", 60); + static final Property CLEANUP_TIMEOUT_SECS = new Property("multimap.cleanup.timeout.secs", 4); + static final Property CLEANUP_ENABLED = new Property("multimap.cleanup.enabled", true); + static final Property CLEANUP_SCRIPT = new Property("multimap.cleanup.script", "cleanStaleMaps.sh"); + static final Property CONNECT_SCRIPT = new Property("multimap.connect.script", "connectVolume.sh"); + static final Property COPY_SCRIPT = new Property("multimap.copy.script", "copyVolume.sh"); + static final Property DISCONNECT_SCRIPT = new Property("multimap.disconnect.script", "disconnectVolume.sh"); + static final Property RESIZE_SCRIPT = new Property("multimap.resize.script", "resizeVolume.sh"); + static final Property DISK_WAIT_SECS = new Property("multimap.disk.wait.secs", 240); + static final Property STORAGE_SCRIPTS_DIR = new Property("multimap.storage.scripts.dir", "scripts/storage/multipath"); + + static Timer cleanupTimer = new Timer(); + private static int cleanupTimeoutSecs = CLEANUP_TIMEOUT_SECS.getFinalValue(); + private static String connectScript = CONNECT_SCRIPT.getFinalValue(); + private static String disconnectScript = DISCONNECT_SCRIPT.getFinalValue(); + private static String cleanupScript = CLEANUP_SCRIPT.getFinalValue(); + private static String resizeScript = RESIZE_SCRIPT.getFinalValue(); + private static String copyScript = COPY_SCRIPT.getFinalValue(); + private static int diskWaitTimeSecs = DISK_WAIT_SECS.getFinalValue(); + + /** + * Initialize static program-wide configurations and background jobs + */ + static { + long cleanupFrequency = CLEANUP_FREQUENCY_SECS.getFinalValue() * 1000; + boolean cleanupEnabled = CLEANUP_ENABLED.getFinalValue(); + + + connectScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), connectScript); + if (connectScript == null) { + throw new Error("Unable to find the connectVolume.sh script"); + } + + disconnectScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), disconnectScript); + if (disconnectScript == null) { + throw new Error("Unable to find the disconnectVolume.sh script"); + } + + resizeScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), resizeScript); + if (resizeScript == null) { + throw new Error("Unable to find the resizeVolume.sh script"); + } + + copyScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), copyScript); + if (copyScript == null) { + throw new Error("Unable to find the copyVolume.sh script"); + } + + if (cleanupEnabled) { + cleanupScript = Script.findScript(STORAGE_SCRIPTS_DIR.getFinalValue(), cleanupScript); + if (cleanupScript == null) { + throw new Error("Unable to find the cleanStaleMaps.sh script and " + CLEANUP_ENABLED.getName() + " is true"); + } + + TimerTask task = new TimerTask() { + @Override + public void run() { + try { + MultipathSCSIAdapterBase.cleanupStaleMaps(); + } catch (Throwable e) { + LOGGER.warn("Error running stale multipath map cleanup", e); + } + } + }; + + cleanupTimer = new Timer("MultipathMapCleanupJob"); + cleanupTimer.scheduleAtFixedRate(task, 0, cleanupFrequency); + } + } + + @Override + public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { + return getStoragePool(uuid); + } + + public abstract String getName(); + + public abstract boolean isStoragePoolTypeSupported(Storage.StoragePoolType type); + + /** + * We expect WWN values in the volumePath so need to convert it to an actual physical path + */ + public abstract AddressInfo parseAndValidatePath(String path); + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumePath, KVMStoragePool pool) { + LOGGER.debug(String.format("getPhysicalDisk(volumePath,pool) called with args (%s,%s)", volumePath, pool)); + + if (StringUtils.isEmpty(volumePath) || pool == null) { + LOGGER.error("Unable to get physical disk, volume path or pool not specified"); + return null; + } + + AddressInfo address = parseAndValidatePath(volumePath); + return getPhysicalDisk(address, pool); + } + + private KVMPhysicalDisk getPhysicalDisk(AddressInfo address, KVMStoragePool pool) { + LOGGER.debug(String.format("getPhysicalDisk(addressInfo,pool) called with args (%s,%s)", address.getPath(), pool)); + KVMPhysicalDisk disk = new KVMPhysicalDisk(address.getPath(), address.toString(), pool); + disk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + + long diskSize = getPhysicalDiskSize(address.getPath()); + disk.setSize(diskSize); + disk.setVirtualSize(diskSize); + LOGGER.debug("Physical disk " + disk.getPath() + " with format " + disk.getFormat() + " and size " + disk.getSize() + " provided"); + return disk; + } + + @Override + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details) { + LOGGER.info(String.format("createStoragePool(uuid,host,port,path,type) called with args (%s, %s, %s, %s, %s)", uuid, host, ""+port, path, type)); + MultipathSCSIPool storagePool = new MultipathSCSIPool(uuid, host, port, path, type, details, this); + MapStorageUuidToStoragePool.put(uuid, storagePool); + return storagePool; + } + + @Override + public boolean deleteStoragePool(String uuid) { + return MapStorageUuidToStoragePool.remove(uuid) != null; + } + + @Override + public boolean connectPhysicalDisk(String volumePath, KVMStoragePool pool, Map details) { + LOGGER.info("connectPhysicalDisk called for [" + volumePath + "]"); + + if (StringUtils.isEmpty(volumePath)) { + LOGGER.error("Unable to connect physical disk due to insufficient data - volume path is undefined"); + throw new CloudRuntimeException("Unable to connect physical disk due to insufficient data - volume path is underfined"); + } + + if (pool == null) { + LOGGER.error("Unable to connect physical disk due to insufficient data - pool is not set"); + throw new CloudRuntimeException("Unable to connect physical disk due to insufficient data - pool is not set"); + } + + AddressInfo address = this.parseAndValidatePath(volumePath); + int waitTimeInSec = diskWaitTimeSecs; + if (details != null && details.containsKey(StorageManager.STORAGE_POOL_DISK_WAIT.toString())) { + String waitTime = details.get(StorageManager.STORAGE_POOL_DISK_WAIT.toString()); + if (StringUtils.isNotEmpty(waitTime)) { + waitTimeInSec = Integer.valueOf(waitTime).intValue(); + } + } + return waitForDiskToBecomeAvailable(address, pool, waitTimeInSec); + } + + @Override + public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool) { + LOGGER.debug(String.format("disconnectPhysicalDiskByPath(volumePath,pool) called with args (%s, %s) START", volumePath, pool.getUuid())); + AddressInfo address = this.parseAndValidatePath(volumePath); + ScriptResult result = runScript(disconnectScript, 60000L, address.getAddress().toLowerCase()); + if (LOGGER.isDebugEnabled()) LOGGER.debug("multipath flush output: " + result.getResult()); + LOGGER.debug(String.format("disconnectPhysicalDiskByPath(volumePath,pool) called with args (%s, %s) COMPLETE [rc=%s]", volumePath, pool.getUuid(), result.getResult())); return true; + } + + @Override + public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { + LOGGER.debug(String.format("disconnectPhysicalDiskByPath(volumeToDisconnect) called with arg bag [not implemented]:") + " " + volumeToDisconnect); + return false; + } + + @Override + public boolean disconnectPhysicalDiskByPath(String localPath) { + LOGGER.debug(String.format("disconnectPhysicalDiskByPath(localPath) called with args (%s) STARTED", localPath)); + ScriptResult result = runScript(disconnectScript, 60000L, localPath.replace("/dev/mapper/3", "")); + if (LOGGER.isDebugEnabled()) LOGGER.debug("multipath flush output: " + result.getResult()); + LOGGER.debug(String.format("disconnectPhysicalDiskByPath(localPath) called with args (%s) COMPLETE [rc=%s]", localPath, result.getExitCode())); return true; + } + + @Override + public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.ImageFormat format) { + LOGGER.info(String.format("deletePhysicalDisk(uuid,pool,format) called with args (%s, %s, %s) [not implemented]", uuid, pool.getUuid(), format.toString())); + return true; + } + + @Override + public KVMPhysicalDisk createTemplateFromDisk(KVMPhysicalDisk disk, String name, QemuImg.PhysicalDiskFormat format, long size, KVMStoragePool destPool) { + LOGGER.info(String.format("createTemplateFromDisk(disk,name,format,size,destPool) called with args (%s, %s, %s, %s, %s) [not implemented]", disk.getPath(), name, format.toString(), ""+size, destPool.getUuid())); + return null; + } + + @Override + public List listPhysicalDisks(String storagePoolUuid, KVMStoragePool pool) { + LOGGER.info(String.format("listPhysicalDisks(uuid,pool) called with args (%s, %s) [not implemented]", storagePoolUuid, pool.getUuid())); + return null; + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout) { + return copyPhysicalDisk(disk, name, destPool, timeout, null, null, null); + } + + @Override + public boolean refresh(KVMStoragePool pool) { + LOGGER.info(String.format("refresh(pool) called with args (%s)", pool.getUuid())); + return true; + } + + @Override + public boolean deleteStoragePool(KVMStoragePool pool) { + LOGGER.info(String.format("deleteStroagePool(pool) called with args (%s)", pool.getUuid())); + return deleteStoragePool(pool.getUuid()); + } + + @Override + public boolean createFolder(String uuid, String path) { + LOGGER.info(String.format("createFolder(uuid,path) called with args (%s, %s) [not implemented]", uuid, path)); + return createFolder(uuid, path, null); + } + + @Override + public boolean createFolder(String uuid, String path, String localPath) { + LOGGER.info(String.format("createFolder(uuid,path,localPath) called with args (%s, %s, %s) [not implemented]", uuid, path, localPath)); + return true; + } + + /** + * Validate inputs and return the source file for a template copy + * @param templateFilePath + * @param destTemplatePath + * @param destPool + * @param format + * @return + */ + File createTemplateFromDirectDownloadFileValidate(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format) { + if (StringUtils.isAnyEmpty(templateFilePath, destTemplatePath) || destPool == null) { + LOGGER.error("Unable to create template from direct download template file due to insufficient data"); + throw new CloudRuntimeException("Unable to create template from direct download template file due to insufficient data"); + } + + LOGGER.debug("Create template from direct download template - file path: " + templateFilePath + ", dest path: " + destTemplatePath + ", format: " + format.toString()); + + File sourceFile = new File(templateFilePath); + if (!sourceFile.exists()) { + throw new CloudRuntimeException("Direct download template file " + templateFilePath + " does not exist on this host"); + } + + if (destTemplatePath == null || destTemplatePath.isEmpty()) { + LOGGER.error("Failed to create template, target template disk path not provided"); + throw new CloudRuntimeException("Target template disk path not provided"); + } + + if (this.isStoragePoolTypeSupported(destPool.getType())) { + throw new CloudRuntimeException("Unsupported storage pool type: " + destPool.getType().toString()); + } + + if (Storage.ImageFormat.RAW.equals(format) && Storage.ImageFormat.QCOW2.equals(format)) { + LOGGER.error("Failed to create template, unsupported template format: " + format.toString()); + throw new CloudRuntimeException("Unsupported template format: " + format.toString()); + } + return sourceFile; + } + + String extractSourceTemplateIfNeeded(File sourceFile, String templateFilePath) { + String srcTemplateFilePath = templateFilePath; + if (isTemplateExtractable(templateFilePath)) { + srcTemplateFilePath = sourceFile.getParent() + "/" + UUID.randomUUID().toString(); + LOGGER.debug("Extract the downloaded template " + templateFilePath + " to " + srcTemplateFilePath); + String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); + Script.runSimpleBashScript(extractCommand); + Script.runSimpleBashScript("rm -f " + templateFilePath); + } + return srcTemplateFilePath; + } + + QemuImg.PhysicalDiskFormat deriveImgFileFormat(Storage.ImageFormat format) { + if (format == Storage.ImageFormat.RAW) { + return QemuImg.PhysicalDiskFormat.RAW; + } else if (format == Storage.ImageFormat.QCOW2) { + return QemuImg.PhysicalDiskFormat.QCOW2; + } else { + return QemuImg.PhysicalDiskFormat.RAW; + } + } + + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, KVMStoragePool destPool, Storage.ImageFormat format, int timeout) { + File sourceFile = createTemplateFromDirectDownloadFileValidate(templateFilePath, destTemplatePath, destPool, format); + LOGGER.debug("Create template from direct download template - file path: " + templateFilePath + ", dest path: " + destTemplatePath + ", format: " + format.toString()); + KVMPhysicalDisk sourceDisk = destPool.getPhysicalDisk(sourceFile.getAbsolutePath()); + return copyPhysicalDisk(sourceDisk, destTemplatePath, destPool, timeout, null, null, Storage.ProvisioningType.THIN); + } + + @Override + public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPool, int timeout, + byte[] srcPassphrase, byte[] dstPassphrase, Storage.ProvisioningType provisioningType) { + + validateForDiskCopy(disk, name, destPool); + LOGGER.info("Copying FROM source physical disk " + disk.getPath() + ", size: " + disk.getSize() + ", virtualsize: " + disk.getVirtualSize()+ ", format: " + disk.getFormat()); + + KVMPhysicalDisk destDisk = destPool.getPhysicalDisk(name); + if (destDisk == null) { + LOGGER.error("Failed to find the disk: " + name + " of the storage pool: " + destPool.getUuid()); + throw new CloudRuntimeException("Failed to find the disk: " + name + " of the storage pool: " + destPool.getUuid()); + } + + if (srcPassphrase != null || dstPassphrase != null) { + throw new CloudRuntimeException("Storage provider does not support user-space encrypted source or destination volumes"); + } + + destDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + destDisk.setVirtualSize(disk.getVirtualSize()); + destDisk.setSize(disk.getSize()); + + LOGGER.info("Copying TO destination physical disk " + destDisk.getPath() + ", size: " + destDisk.getSize() + ", virtualsize: " + destDisk.getVirtualSize()+ ", format: " + destDisk.getFormat()); + QemuImgFile srcFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + QemuImgFile destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); + LOGGER.debug("Starting COPY from source downloaded template " + srcFile.getFileName() + " to Primera volume: " + destDisk.getPath()); + ScriptResult result = runScript(copyScript, timeout, destDisk.getFormat().toString().toLowerCase(), srcFile.getFileName(), destFile.getFileName()); + int rc = result.getExitCode(); + if (rc != 0) { + throw new CloudRuntimeException("Failed to convert from " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + rc + " - " + result.getResult()); + } + LOGGER.debug("Successfully converted source downloaded template " + srcFile.getFileName() + " to Primera volume: " + destDisk.getPath() + " " + result.getResult()); + + return destDisk; + } + + void validateForDiskCopy(KVMPhysicalDisk disk, String name, KVMStoragePool destPool) { + if (StringUtils.isEmpty(name) || disk == null || destPool == null) { + LOGGER.error("Unable to copy physical disk due to insufficient data"); + throw new CloudRuntimeException("Unable to copy physical disk due to insufficient data"); + } + } + + /** + * Copy a disk path to another disk path using QemuImg command + * @param disk + * @param destDisk + * @param name + * @param timeout + */ + void qemuCopy(KVMPhysicalDisk disk, KVMPhysicalDisk destDisk, String name, int timeout) { + QemuImg qemu; + try { + qemu = new QemuImg(timeout); + } catch (LibvirtException | QemuImgException e) { + throw new CloudRuntimeException (e); + } + QemuImgFile srcFile = null; + QemuImgFile destFile = null; + + try { + srcFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + destFile = new QemuImgFile(destDisk.getPath(), destDisk.getFormat()); + + LOGGER.debug("Starting copy from source disk image " + srcFile.getFileName() + " to volume: " + destDisk.getPath()); + qemu.convert(srcFile, destFile, true); + LOGGER.debug("Successfully converted source disk image " + srcFile.getFileName() + " to volume: " + destDisk.getPath()); + } catch (QemuImgException | LibvirtException e) { + try { + Map srcInfo = qemu.info(srcFile); + LOGGER.debug("Source disk info: " + Arrays.asList(srcInfo)); + } catch (Exception ignored) { + LOGGER.warn("Unable to get info from source disk: " + disk.getName()); + } + + String errMsg = String.format("Unable to convert/copy from %s to %s, due to: %s", disk.getName(), name, ((StringUtils.isEmpty(e.getMessage())) ? "an unknown error" : e.getMessage())); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg, e); + } + } + + @Override + public KVMPhysicalDisk createDiskFromTemplate(KVMPhysicalDisk template, + String name, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplate'"); + } + + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, + String name, PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createDiskFromTemplateBacking'"); + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, + PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase) { + throw new UnsupportedOperationException("Unimplemented method 'createPhysicalDisk'"); + } + + boolean isTemplateExtractable(String templatePath) { + ScriptResult result = runScript("file", 5000L, templatePath, "| awk -F' ' '{print $2}'"); + String type = result.getResult(); + return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); + } + + String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) { + if (downloadedTemplateFile.endsWith(".zip")) { + return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile; + } else if (downloadedTemplateFile.endsWith(".bz2")) { + return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile; + } else if (downloadedTemplateFile.endsWith(".gz")) { + return "gunzip -c " + downloadedTemplateFile + " > " + templateFile; + } else { + throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); + } + } + + private static final ScriptResult runScript(String script, long timeout, String...args) { + ScriptResult result = new ScriptResult(); + Script cmd = new Script(script, Duration.millis(timeout), LOGGER); + cmd.add(args); + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String output = cmd.execute(parser); + // its possible the process never launches which causes an NPE on getExitValue below + if (output != null && output.contains("Unable to execute the command")) { + result.setResult(output); + result.setExitCode(-1); + return result; + } + result.setResult(output); + result.setExitCode(cmd.getExitValue()); + return result; + } + + boolean waitForDiskToBecomeAvailable(AddressInfo address, KVMStoragePool pool, long waitTimeInSec) { + LOGGER.debug("Waiting for the volume with id: " + address.getPath() + " of the storage pool: " + pool.getUuid() + " to become available for " + waitTimeInSec + " secs"); + long scriptTimeoutSecs = 30; // how long to wait for each script execution to run + long maxTries = 10; // how many max retries to attempt the script + long waitTimeInMillis = waitTimeInSec * 1000; // how long overall to wait + int timeBetweenTries = 1000; // how long to sleep between tries + // wait at least 60 seconds even if input was lower + if (waitTimeInSec < 60) { + waitTimeInSec = 60; + } + KVMPhysicalDisk physicalDisk = null; + + // Rescan before checking for the physical disk + int tries = 0; + while (waitTimeInMillis > 0 && tries < maxTries) { + tries++; + long start = System.currentTimeMillis(); + String lun; + if (address.getConnectionId() == null) { + lun = "-"; + } else { + lun = address.getConnectionId(); + } + + Process p = null; + try { + ProcessBuilder builder = new ProcessBuilder(connectScript, lun, address.getAddress()); + p = builder.start(); + if (p.waitFor(scriptTimeoutSecs, TimeUnit.SECONDS)) { + int rc = p.exitValue(); + StringBuffer output = new StringBuffer(); + if (rc == 0) { + BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); + String line = null; + while ((line = input.readLine()) != null) { + output.append(line); + output.append(" "); + } + + physicalDisk = getPhysicalDisk(address, pool); + if (physicalDisk != null && physicalDisk.getSize() > 0) { + LOGGER.debug("Found the volume using id: " + address.getPath() + " of the storage pool: " + pool.getUuid()); + return true; + } + + break; + } else { + LOGGER.warn("Failure discovering LUN via " + connectScript); + BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream())); + String line = null; + while ((line = error.readLine()) != null) { + LOGGER.warn("error --> " + line); + } + } + } else { + LOGGER.debug("Timeout waiting for " + connectScript + " to complete - try " + tries); + } + } catch (IOException | InterruptedException | IllegalThreadStateException e) { + LOGGER.warn("Problem performing scan on SCSI hosts - try " + tries, e); + } finally { + if (p != null && p.isAlive()) { + p.destroyForcibly(); + } + } + + long elapsed = System.currentTimeMillis() - start; + waitTimeInMillis = waitTimeInMillis - elapsed; + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ex) { + // don't do anything + } + } + + LOGGER.debug("Unable to find the volume with id: " + address.getPath() + " of the storage pool: " + pool.getUuid()); + return false; + } + + void runConnectScript(String lun, AddressInfo address) { + try { + ProcessBuilder builder = new ProcessBuilder(connectScript, lun, address.getAddress()); + Process p = builder.start(); + int rc = p.waitFor(); + StringBuffer output = new StringBuffer(); + if (rc == 0) { + BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream())); + String line = null; + while ((line = input.readLine()) != null) { + output.append(line); + output.append(" "); + } + } else { + LOGGER.warn("Failure discovering LUN via " + connectScript); + BufferedReader error = new BufferedReader(new InputStreamReader(p.getErrorStream())); + String line = null; + while ((line = error.readLine()) != null) { + LOGGER.warn("error --> " + line); + } + } + } catch (IOException | InterruptedException e) { + throw new CloudRuntimeException("Problem performing scan on SCSI hosts", e); + } + } + + void sleep(long sleepTimeMs) { + try { + Thread.sleep(sleepTimeMs); + } catch (Exception ex) { + // don't do anything + } + } + + long getPhysicalDiskSize(String diskPath) { + if (StringUtils.isEmpty(diskPath)) { + return 0; + } + + Script diskCmd = new Script("blockdev", LOGGER); + diskCmd.add("--getsize64", diskPath); + + OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); + String result = diskCmd.execute(parser); + + if (result != null) { + LOGGER.debug("Unable to get the disk size at path: " + diskPath); + return 0; + } + + Long size = Long.parseLong(parser.getLine()); + + if (size <= 0) { + // its possible the path can't be seen on the host yet, lets rescan + // now rerun the command + parser = new OutputInterpreter.OneLineParser(); + result = diskCmd.execute(parser); + + if (result != null) { + LOGGER.debug("Unable to get the disk size at path: " + diskPath); + return 0; + } + + size = Long.parseLong(parser.getLine()); + } + + return size; + } + + public void resize(String path, String vmName, long newSize) { + if (LOGGER.isDebugEnabled()) LOGGER.debug("Executing resize of " + path + " to " + newSize + " bytes for VM " + vmName); + + // extract wwid + AddressInfo address = parseAndValidatePath(path); + if (address == null || address.getAddress() == null) { + LOGGER.error("Unable to resize volume, address value is not valid"); + throw new CloudRuntimeException("Unable to resize volume, address value is not valid"); + } + + if (LOGGER.isDebugEnabled()) LOGGER.debug(String.format("Running %s %s %s %s", resizeScript, address.getAddress(), vmName, newSize)); + + // call resizeVolume.sh + ScriptResult result = runScript(resizeScript, 60000L, address.getAddress(), vmName, ""+newSize); + + if (result.getExitCode() != 0) { + throw new CloudRuntimeException("Failed to resize volume at address " + address.getAddress() + " to " + newSize + " bytes for VM " + vmName + ": " + result.getResult()); + } + + LOGGER.info("Resize of volume at address " + address.getAddress() + " completed successfully: " + result.getResult()); + } + + static void cleanupStaleMaps() { + synchronized(CLEANUP_LOCK) { + long start = System.currentTimeMillis(); + ScriptResult result = runScript(cleanupScript, cleanupTimeoutSecs * 1000); + LOGGER.debug("Multipath Cleanup Job elapsed time (ms): "+ (System.currentTimeMillis() - start) + "; result: " + result.getExitCode(), null); + } + } + + public static final class AddressInfo { + String type; + String address; + String connectionId; + String path; + + public AddressInfo(String type, String address, String connectionId, String path) { + this.type = type; + this.address = address; + this.connectionId = connectionId; + this.path = path; + } + + public String getType() { + return type; + } + + public String getAddress() { + return address; + } + + public String getConnectionId() { + return connectionId; + } + + public String getPath() { + return path; + } + + public String toString() { + return String.format("type=%s; address=%s; connid=%s", getType(), getAddress(), getConnectionId()); + } + } + + public static class Property { + private String name; + private T defaultValue; + + Property(String name, T value) { + this.name = name; + this.defaultValue = value; + } + + public String getName() { + return this.name; + } + + public T getDefaultValue() { + return this.defaultValue; + } + + public T getFinalValue() { + File agentPropertiesFile = PropertiesUtil.findConfigFile("agent.properties"); + if (agentPropertiesFile == null) { + LOGGER.debug(String.format("File [%s] was not found, we will use default defined values. Property [%s]: [%s].", "agent.properties", name, defaultValue)); + return defaultValue; + } else { + try { + String configValue = PropertiesUtil.loadFromFile(agentPropertiesFile).getProperty(name); + if (StringUtils.isBlank(configValue)) { + LOGGER.debug(String.format("Property [%s] has empty or null value. Using default value [%s].", name, defaultValue)); + return defaultValue; + } else { + if (defaultValue instanceof Integer) { + return (T)Integer.getInteger(configValue); + } else if (defaultValue instanceof Long) { + return (T)Long.getLong(configValue); + } else if (defaultValue instanceof String) { + return (T)configValue; + } else if (defaultValue instanceof Boolean) { + return (T)Boolean.valueOf(configValue); + } else { + return null; + } + } + } catch (IOException var5) { + LOGGER.debug(String.format("Failed to get property [%s]. Using default value [%s].", name, defaultValue), var5); + return defaultValue; + } + } + } + } + + public static class ScriptResult { + private int exitCode = -1; + private String result = null; + public int getExitCode() { + return exitCode; + } + public void setExitCode(int exitCode) { + this.exitCode = exitCode; + } + public String getResult() { + return result; + } + public void setResult(String result) { + this.result = result; + } + } + +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java new file mode 100644 index 00000000000..bc2f072f719 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.storage; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.joda.time.Duration; + +import com.cloud.agent.api.to.HostTO; +import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ProvisioningType; + +public class MultipathSCSIPool implements KVMStoragePool { + private String uuid; + private String sourceHost; + private int sourcePort; + private String sourceDir; + private Storage.StoragePoolType storagePoolType; + private StorageAdaptor storageAdaptor; + private long capacity; + private long used; + private long available; + private Map details; + + public MultipathSCSIPool(String uuid, String host, int port, String path, Storage.StoragePoolType poolType, Map poolDetails, StorageAdaptor adaptor) { + this.uuid = uuid; + sourceHost = host; + sourcePort = port; + sourceDir = path; + storagePoolType = poolType; + storageAdaptor = adaptor; + capacity = 0; + used = 0; + available = 0; + details = poolDetails; + } + + public MultipathSCSIPool(String uuid, StorageAdaptor adapter) { + this.uuid = uuid; + sourceHost = null; + sourcePort = -1; + sourceDir = null; + storagePoolType = Storage.StoragePoolType.FiberChannel; + details = new HashMap(); + this.storageAdaptor = adapter; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String arg0, ProvisioningType arg1, long arg2, byte[] arg3) { + return null; + } + + @Override + public KVMPhysicalDisk createPhysicalDisk(String arg0, PhysicalDiskFormat arg1, ProvisioningType arg2, long arg3, + byte[] arg4) { + return null; + } + + @Override + public boolean connectPhysicalDisk(String volumeUuid, Map details) { + return storageAdaptor.connectPhysicalDisk(volumeUuid, this, details); + } + + @Override + public KVMPhysicalDisk getPhysicalDisk(String volumeId) { + return storageAdaptor.getPhysicalDisk(volumeId, this); + } + + @Override + public boolean disconnectPhysicalDisk(String volumeUuid) { + return storageAdaptor.disconnectPhysicalDisk(volumeUuid, this); + } + + @Override + public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format) { + return true; + } + + @Override + public List listPhysicalDisks() { + return null; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setCapacity(long capacity) { + this.capacity = capacity; + } + + @Override + public long getCapacity() { + return this.capacity; + } + + public void setUsed(long used) { + this.used = used; + } + + @Override + public long getUsed() { + return this.used; + } + + public void setAvailable(long available) { + this.available = available; + } + + @Override + public long getAvailable() { + return this.available; + } + + @Override + public boolean refresh() { + return false; + } + + @Override + public boolean isExternalSnapshot() { + return true; + } + + @Override + public String getLocalPath() { + return null; + } + + @Override + public String getSourceHost() { + return this.sourceHost; + } + + @Override + public String getSourceDir() { + return this.sourceDir; + } + + @Override + public int getSourcePort() { + return this.sourcePort; + } + + @Override + public String getAuthUserName() { + return null; + } + + @Override + public String getAuthSecret() { + return null; + } + + @Override + public Storage.StoragePoolType getType() { + return storagePoolType; + } + + @Override + public boolean delete() { + return false; + } + + @Override + public QemuImg.PhysicalDiskFormat getDefaultFormat() { + return QemuImg.PhysicalDiskFormat.RAW; + } + + @Override + public boolean createFolder(String path) { + return false; + } + + @Override + public boolean supportsConfigDriveIso() { + return false; + } + + @Override + public Map getDetails() { + return this.details; + } + + @Override + public boolean isPoolSupportHA() { + return false; + } + + @Override + public String getHearthBeatPath() { + return null; + } + + @Override + public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, + boolean hostValidation) { + return null; + } + + @Override + public String getStorageNodeId() { + return null; + } + + @Override + public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + return null; + } + + @Override + public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, + String volumeUUIDListString, String vmActivityCheckPath, long duration) { + return null; + } + + public void resize(String path, String vmName, long newSize) { + ((MultipathSCSIAdapterBase)storageAdaptor).resize(path, vmName, newSize); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index fc462adc86d..aac7f7343af 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -6200,4 +6200,99 @@ public class LibvirtComputingResourceTest { Mockito.verify(loggerMock).debug("Skipping the memory balloon stats period setting for the VM (Libvirt Domain) with ID [1] and name [fake-VM-name] because this" + " VM has no memory balloon."); } + + @Test + public void calculateCpuSharesTestMinSpeedNullAndHostCgroupV1ShouldNotConsiderCgroupLimit() { + int cpuCores = 2; + int cpuSpeed = 2000; + int maxCpuShares = 0; + int expectedCpuShares = 4000; + + Mockito.doReturn(cpuCores).when(vmTO).getCpus(); + Mockito.doReturn(null).when(vmTO).getMinSpeed(); + Mockito.doReturn(cpuSpeed).when(vmTO).getSpeed(); + Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity(); + int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO); + + Assert.assertEquals(expectedCpuShares, calculatedCpuShares); + } + + @Test + public void calculateCpuSharesTestMinSpeedNotNullAndHostCgroupV1ShouldNotConsiderCgroupLimit() { + int cpuCores = 2; + int cpuSpeed = 2000; + int maxCpuShares = 0; + int expectedCpuShares = 4000; + + Mockito.doReturn(cpuCores).when(vmTO).getCpus(); + Mockito.doReturn(cpuSpeed).when(vmTO).getMinSpeed(); + Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity(); + int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO); + + Assert.assertEquals(expectedCpuShares, calculatedCpuShares); + } + + + @Test + public void calculateCpuSharesTestMinSpeedNullAndHostCgroupV2ShouldConsiderCgroupLimit() { + int cpuCores = 2; + int cpuSpeed = 2000; + int maxCpuShares = 5000; + int expectedCpuShares = 8000; + + Mockito.doReturn(cpuCores).when(vmTO).getCpus(); + Mockito.doReturn(null).when(vmTO).getMinSpeed(); + Mockito.doReturn(cpuSpeed).when(vmTO).getSpeed(); + Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity(); + int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO); + + Assert.assertEquals(expectedCpuShares, calculatedCpuShares); + } + + @Test + public void calculateCpuSharesTestMinSpeedNotNullAndHostCgroupV2ShouldConsiderCgroupLimit() { + int cpuCores = 2; + int cpuSpeed = 2000; + int maxCpuShares = 5000; + int expectedCpuShares = 8000; + + Mockito.doReturn(cpuCores).when(vmTO).getCpus(); + Mockito.doReturn(cpuSpeed).when(vmTO).getMinSpeed(); + Mockito.doReturn(maxCpuShares).when(libvirtComputingResourceSpy).getHostCpuMaxCapacity(); + int calculatedCpuShares = libvirtComputingResourceSpy.calculateCpuShares(vmTO); + + Assert.assertEquals(expectedCpuShares, calculatedCpuShares); + } + + @Test + public void setMaxHostCpuSharesIfCGroupV2TestShouldCalculateMaxCpuCapacityIfHostUtilizesCgroupV2() { + int cpuCores = 2; + long cpuSpeed = 2500L; + int expectedShares = 5000; + + String hostCgroupVersion = LibvirtComputingResource.CGROUP_V2; + try (MockedStatic diff --git a/ui/src/components/view/AnnotationsTab.vue b/ui/src/components/view/AnnotationsTab.vue index 97cf926dee3..5b42af8eabc 100644 --- a/ui/src/components/view/AnnotationsTab.vue +++ b/ui/src/components/view/AnnotationsTab.vue @@ -193,6 +193,7 @@ export default { case 'VirtualRouter': return 'VR' case 'AutoScaleVmGroup': return 'AUTOSCALE_VM_GROUP' case 'ManagementServer': return 'MANAGEMENT_SERVER' + case 'ObjectStorage': return 'OBJECT_STORAGE' default: return '' } }, diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index cf52f058075..53c6efb321d 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -279,6 +279,9 @@ + @@ -585,7 +588,7 @@ export default { '/volume', '/snapshot', '/vmsnapshot', '/backup', '/guestnetwork', '/vpc', '/vpncustomergateway', '/vnfapp', '/template', '/iso', - '/project', '/account', + '/project', '/account', 'buckets', 'objectstore', '/zone', '/pod', '/cluster', '/host', '/storagepool', '/imagestore', '/systemvm', '/router', '/ilbvm', '/annotation', '/computeoffering', '/systemoffering', '/diskoffering', '/backupoffering', '/networkoffering', '/vpcoffering', '/tungstenfabric', '/oauthsetting', '/guestos', '/guestoshypervisormapping'].join('|')) @@ -595,7 +598,7 @@ export default { return ['vm', 'alert', 'vmgroup', 'ssh', 'userdata', 'affinitygroup', 'autoscalevmgroup', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork', 'vpc', 'publicip', 'vpnuser', 'vpncustomergateway', 'vnfapp', 'project', 'account', 'systemvm', 'router', 'computeoffering', 'systemoffering', - 'diskoffering', 'backupoffering', 'networkoffering', 'vpcoffering', 'ilbvm', 'kubernetes', 'comment' + 'diskoffering', 'backupoffering', 'networkoffering', 'vpcoffering', 'ilbvm', 'kubernetes', 'comment', 'buckets' ].includes(this.$route.name) }, getDateAtTimeZone (date, timezone) { diff --git a/ui/src/components/view/ObjectStoreBrowser.vue b/ui/src/components/view/ObjectStoreBrowser.vue new file mode 100644 index 00000000000..f2e68c8b954 --- /dev/null +++ b/ui/src/components/view/ObjectStoreBrowser.vue @@ -0,0 +1,541 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/components/view/ResourceView.vue b/ui/src/components/view/ResourceView.vue index 440b45e640a..367c58961bc 100644 --- a/ui/src/components/view/ResourceView.vue +++ b/ui/src/components/view/ResourceView.vue @@ -44,7 +44,7 @@ @@ -104,6 +104,10 @@ export default { autoSelectLabel: { type: String, default: '' + }, + isKVMUnmanage: { + type: Boolean, + default: false } }, data () { diff --git a/ui/src/views/compute/wizard/MultiNetworkSelection.vue b/ui/src/views/compute/wizard/MultiNetworkSelection.vue index f2397d5c48c..048a70671d9 100644 --- a/ui/src/views/compute/wizard/MultiNetworkSelection.vue +++ b/ui/src/views/compute/wizard/MultiNetworkSelection.vue @@ -36,7 +36,16 @@