diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 0b6e3af3552..0a459d8d4fc 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -468,3 +468,15 @@ iscsi.session.cleanup.enabled=false # Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. # incremental.snapshot.retry.rebase.wait=60 + +# Path to the VDDK library directory for VMware to KVM conversion via VDDK, +# passed to virt-v2v as -io vddk-libdir= +#vddk.lib.dir= + +# Ordered VDDK transport preference for VMware to KVM conversion via VDDK, passed as +# -io vddk-transports= to virt-v2v. Example: nbd:nbdssl +#vddk.transports= + +# Optional vCenter SHA1 thumbprint for VMware to KVM conversion via VDDK, passed as +# -io vddk-thumbprint=. If unset, CloudStack computes it on the KVM host via openssl. +#vddk.thumbprint= diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index f0a10f80a9a..18a6b6df100 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -822,6 +822,30 @@ public class AgentProperties{ */ public static final Property CONVERT_ENV_VIRTV2V_TMPDIR = new Property<>("convert.instance.env.virtv2v.tmpdir", null, String.class); + /** + * Path to the VDDK library directory on the KVM conversion host, used when converting VMs from VMware to KVM via VDDK. + * This directory is passed to virt-v2v as -io vddk-libdir=<path>. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_LIB_DIR = new Property<>("vddk.lib.dir", null, String.class); + + /** + * Ordered list of VDDK transports for virt-v2v, passed as -io vddk-transports=<value>. + * Example: nbd:nbdssl. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_TRANSPORTS = new Property<>("vddk.transports", null, String.class); + + /** + * vCenter TLS certificate thumbprint used by virt-v2v VDDK mode, passed as -io vddk-thumbprint=<value>. + * If unset, the KVM host computes it at runtime from the vCenter endpoint. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_THUMBPRINT = new Property<>("vddk.thumbprint", null, String.class); + /** * BGP controll CIDR * Data type: String.
diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java index 18737c584b3..7daeb964917 100644 --- a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -36,13 +36,17 @@ public class RemoteInstanceTO implements Serializable { private String vcenterPassword; private String vcenterHost; private String datacenterName; + private String clusterName; + private String hostName; public RemoteInstanceTO() { } - public RemoteInstanceTO(String instanceName) { + public RemoteInstanceTO(String instanceName, String clusterName, String hostName) { this.hypervisorType = Hypervisor.HypervisorType.VMware; this.instanceName = instanceName; + this.clusterName = clusterName; + this.hostName = hostName; } public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) { @@ -55,6 +59,12 @@ public class RemoteInstanceTO implements Serializable { this.datacenterName = datacenterName; } + public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName, String clusterName, String hostName) { + this(instanceName, instancePath, vcenterHost, vcenterUsername, vcenterPassword, datacenterName); + this.clusterName = clusterName; + this.hostName = hostName; + } + public Hypervisor.HypervisorType getHypervisorType() { return this.hypervisorType; } @@ -82,4 +92,12 @@ public class RemoteInstanceTO implements Serializable { public String getDatacenterName() { return datacenterName; } + + public String getClusterName() { + return clusterName; + } + + public String getHostName() { + return hostName; + } } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 9c011bac319..b5234820151 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -57,6 +57,9 @@ public interface Host extends StateObject, Identity, Partition, HAResour String HOST_UEFI_ENABLE = "host.uefi.enable"; String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + String HOST_VDDK_SUPPORT = "host.vddk.support"; + String HOST_VDDK_LIB_DIR = "vddk.lib.dir"; + String HOST_VDDK_VERSION = "host.vddk.version"; String HOST_OVFTOOL_VERSION = "host.ovftool.version"; String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; String HOST_SSH_PORT = "host.ssh.port"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4f0b39fb1ce..19af0831c83 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -158,6 +158,7 @@ public class ApiConstants { public static final String CUSTOM_ID = "customid"; public static final String CUSTOM_ACTION_ID = "customactionid"; public static final String CUSTOM_JOB_ID = "customjobid"; + public static final String CURRENCY = "currency"; public static final String CURRENT_START_IP = "currentstartip"; public static final String CURRENT_END_IP = "currentendip"; public static final String ENCRYPT = "encrypt"; @@ -530,7 +531,6 @@ public class ApiConstants { public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; - public static final String USER_SECRET_KEY = "usersecretkey"; public static final String SEARCH_BASE = "searchbase"; public static final String SECONDARY_IP = "secondaryip"; public static final String SECURITY_GROUP_IDS = "securitygroupids"; @@ -546,6 +546,7 @@ public class ApiConstants { public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; public static final String SHOW_REMOVED = "showremoved"; + public static final String SHOW_RESOURCES = "showresources"; public static final String SHOW_RESOURCE_ICON = "showicon"; public static final String SHOW_INACTIVE = "showinactive"; public static final String SHOW_UNIQUE = "showunique"; @@ -612,9 +613,11 @@ public class ApiConstants { public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; public static final String TO_CHECKPOINT_ID = "tocheckpointid"; + public static final String TOTAL_QUOTA = "totalquota"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; public static final String TRUST_STORE_PASSWORD = "truststorepass"; + public static final String UNIT = "unit"; public static final String URL = "url"; public static final String USAGE_INTERFACE = "usageinterface"; public static final String USED = "used"; @@ -635,6 +638,8 @@ public class ApiConstants { public static final String USERNAME = "username"; public static final String USER_CONFIGURABLE = "userconfigurable"; public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; + public static final String USER_SECRET_KEY = "usersecretkey"; + public static final String USE_VDDK = "usevddk"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver"; public static final String UPDATE_IN_SEQUENCE = "updateinsequence"; @@ -1304,6 +1309,8 @@ public class ApiConstants { public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; public static final String QUOTA = "quota"; + public static final String QUOTA_CONSUMED = "quotaconsumed"; + public static final String QUOTA_USAGE = "quotausage"; public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index e495cf28413..00b1bc310d5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -27,7 +27,6 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.regex.Pattern; import javax.inject.Inject; @@ -504,12 +503,6 @@ public abstract class BaseCmd { } public String getResourceUuid(String parameterName) { - UUID resourceUuid = CallContext.current().getApiResourceUuid(parameterName); - - if (resourceUuid != null) { - return resourceUuid.toString(); - } - - return null; + return CallContext.current().getApiResourceUuid(parameterName); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 50ccfbd69c5..db7dcc3fb44 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -179,6 +179,14 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "(only for importing VMs from VMware to KVM) optional - the ID of the guest OS for the imported VM.") private Long guestOsId; + @Parameter(name = ApiConstants.USE_VDDK, + type = CommandType.BOOLEAN, + since = "4.22.1", + description = "(only for importing VMs from VMware to KVM) optional - if true, uses VDDK on the KVM conversion host for converting the VM. " + + "This parameter is mutually exclusive with " + ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES + ".") + private Boolean useVddk; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -255,6 +263,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return storagePoolId; } + public boolean getUseVddk() { + return BooleanUtils.toBooleanDefaultIfNull(useVddk, true); + } + public String getTmpPath() { return tmpPath; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java index 4a9a0569c3b..abbb1760f9d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.bucket; import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.user.Account; @@ -82,7 +83,7 @@ public class DeleteBucketCmd extends BaseCmd { } @Override - public void execute() throws ConcurrentOperationException { + public void execute() throws ConcurrentOperationException, ResourceAllocationException { CallContext.current().setEventDetails("Bucket ID: " + getResourceUuid(ApiConstants.ID)); boolean result = _bucketService.deleteBucket(id, CallContext.current().getCallingAccount()); SuccessResponse response = new SuccessResponse(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index f3bd535a6b8..587b7c32105 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -235,7 +235,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer * @param forced Indicates if backup will be force removed or not * @return returns operation success */ - boolean deleteBackup(final Long backupId, final Boolean forced); + boolean deleteBackup(final Long backupId, final Boolean forced) throws ResourceAllocationException; void validateBackupForZone(Long zoneId); diff --git a/api/src/main/java/org/apache/cloudstack/context/CallContext.java b/api/src/main/java/org/apache/cloudstack/context/CallContext.java index 5e0c60184f4..fcfb5b6b1e0 100644 --- a/api/src/main/java/org/apache/cloudstack/context/CallContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/CallContext.java @@ -63,7 +63,7 @@ public class CallContext { private User user; private long userId; private final Map context = new HashMap(); - private final Map apiResourcesUuids = new HashMap<>(); + private final Map apiResourcesUuids = new HashMap<>(); private Project project; private String apiName; @@ -389,11 +389,11 @@ public class CallContext { isEventDisplayEnabled = eventDisplayEnabled; } - public UUID getApiResourceUuid(String paramName) { + public String getApiResourceUuid(String paramName) { return apiResourcesUuids.get(paramName); } - public void putApiResourceUuid(String paramName, UUID uuid) { + public void putApiResourceUuid(String paramName, String uuid) { apiResourcesUuids.put(paramName, uuid); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java index e27ef308d7f..8c164133db8 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -95,7 +95,7 @@ public interface BucketApiService { */ Bucket createBucket(CreateBucketCmd cmd); - boolean deleteBucket(long bucketId, Account caller); + boolean deleteBucket(long bucketId, Account caller) throws ResourceAllocationException; boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java index 4039ca6dc94..38d0df2e8b8 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java @@ -40,7 +40,7 @@ public class CreateIpv4SubnetForGuestNetworkCmdTest { @Test public void testCreateIpv4SubnetForGuestNetworkCmd() { Long parentId = 1L; - UUID parentUuid = UUID.randomUUID(); + String parentUuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; Integer cidrSize = 26; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java index bb324aca0e7..560ecdc3b29 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java @@ -40,7 +40,7 @@ public class CreateIpv4SubnetForZoneCmdTest { @Test public void testCreateIpv4SubnetForZoneCmd() { Long zoneId = 1L; - UUID zoneUuid = UUID.randomUUID(); + String zoneUuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; String accountName = "user"; Long projectId = 10L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java index 31458d2833f..4640510ccda 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class DedicateIpv4SubnetForZoneCmdTest { @Test public void testDedicateIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java index 48aceeaaeec..cd25d8d2401 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java @@ -39,7 +39,7 @@ public class DeleteIpv4SubnetForGuestNetworkCmdTest { @Test public void testDeleteIpv4SubnetForGuestNetworkCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteIpv4SubnetForGuestNetworkCmd cmd = new DeleteIpv4SubnetForGuestNetworkCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java index 5c3593a8f1b..269fb3f3c19 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class DeleteIpv4SubnetForZoneCmdTest { @Test public void testDeleteIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteIpv4SubnetForZoneCmd cmd = new DeleteIpv4SubnetForZoneCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java index 29d6d8e735b..6e5d2a95f3a 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class ReleaseDedicatedIpv4SubnetForZoneCmdTest { @Test public void testReleaseDedicatedIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReleaseDedicatedIpv4SubnetForZoneCmd cmd = new ReleaseDedicatedIpv4SubnetForZoneCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java index 399b77de6e8..af37006eafd 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java @@ -40,7 +40,7 @@ public class UpdateIpv4SubnetForZoneCmdTest { @Test public void testUpdateIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; UpdateIpv4SubnetForZoneCmd cmd = new UpdateIpv4SubnetForZoneCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java index 9cd403ddd1d..3db1fab466f 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java @@ -46,7 +46,7 @@ public class ChangeBgpPeersForNetworkCmdTest { @Test public void testChangeBgpPeersForNetworkCmd() { Long networkId = 10L; - UUID networkUuid = UUID.randomUUID(); + String networkUuid = UUID.randomUUID().toString(); List bgpPeerIds = Arrays.asList(20L, 21L); ChangeBgpPeersForNetworkCmd cmd = new ChangeBgpPeersForNetworkCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java index 545523e3ab9..fb85f706068 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java @@ -46,7 +46,7 @@ public class ChangeBgpPeersForVpcCmdTest { @Test public void testChangeBgpPeersForVpcCmd() { Long VpcId = 10L; - UUID vpcUuid = UUID.randomUUID(); + String vpcUuid = UUID.randomUUID().toString(); List bgpPeerIds = Arrays.asList(20L, 21L); ChangeBgpPeersForVpcCmd cmd = new ChangeBgpPeersForVpcCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java index 866824f6293..3abc5f57d01 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java @@ -40,7 +40,7 @@ public class CreateBgpPeerCmdTest { @Test public void testCreateBgpPeerCmd() { Long zoneId = 1L; - UUID zoneUuid = UUID.randomUUID(); + String zoneUuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java index a8046d3d745..c5edb1b8f53 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class DedicateBgpPeerCmdTest { @Test public void testDedicateBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java index d54be9e859e..5228a63dc92 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class DeleteBgpPeerCmdTest { @Test public void testDeleteBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteBgpPeerCmd cmd = new DeleteBgpPeerCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java index 1cf8ead706d..60a814d6305 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class ReleaseDedicatedBgpPeerCmdTest { @Test public void testReleaseDedicatedBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReleaseDedicatedBgpPeerCmd cmd = new ReleaseDedicatedBgpPeerCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java index 1601fcb4c5a..d594bc5718b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java @@ -40,7 +40,7 @@ public class UpdateBgpPeerCmdTest { @Test public void testUpdateBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String ip4Address = "ip4-address"; String ip6Address = "ip6-address"; Long peerAsNumber = 15000L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java index 3b2b49cae4e..8d2771c969b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java @@ -97,7 +97,7 @@ public class DownloadImageStoreObjectCmdTest { @Test public void testGetEventDescription() { - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReflectionTestUtils.setField(cmd, "storeId", 1L); ReflectionTestUtils.setField(cmd, "path", "path/to/object"); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java index ecca507a6b9..59a61806e86 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java @@ -44,7 +44,7 @@ public class UnmanageVolumeCmdTest { public void testUnmanageVolumeCmd() { long accountId = 2L; Long volumeId = 3L; - UUID volumeUuid = UUID.randomUUID(); + String volumeUuid = UUID.randomUUID().toString(); Volume volume = Mockito.mock(Volume.class); Mockito.when(responseGenerator.findVolumeById(volumeId)).thenReturn(volume); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index b70efaf9a6c..032dca8d800 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -118,7 +118,7 @@ public class CreateSnapshotCmdTest extends TestCase { AccountService accountService = Mockito.mock(AccountService.class); Account account = Mockito.mock(Account.class); Mockito.when(accountService.getAccount(anyLong())).thenReturn(account); - UUID volumeUuid = UUID.randomUUID(); + String volumeUuid = UUID.randomUUID().toString(); CallContext.current().putApiResourceUuid("volumeid", volumeUuid); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java index 3dfb29dadd3..c78dbe9b56b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java @@ -56,7 +56,7 @@ public class UpdateConditionCmdTest { private static final Long threshold = 100L; private static final long accountId = 5L; - private static final UUID conditionUuid = UUID.randomUUID(); + private static final String conditionUuid = UUID.randomUUID().toString(); @Before public void setUp() { diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java index d3cf5dd6cd6..dbe7669431d 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java @@ -49,7 +49,7 @@ public class DeleteRoutingFirewallRuleCmdTest { ReflectionTestUtils.setField(cmd, "_firewallService", _firewallService); long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); long accountId = 2L; long networkId = 3L; diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java index fc066e5c589..c46fb697a3c 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java @@ -18,6 +18,8 @@ package com.cloud.agent.api; public class CheckConvertInstanceCommand extends Command { boolean checkWindowsGuestConversionSupport = false; + boolean useVddk = false; + String vddkLibDir; public CheckConvertInstanceCommand() { } @@ -26,6 +28,11 @@ public class CheckConvertInstanceCommand extends Command { this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; } + public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport, boolean useVddk) { + this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; + this.useVddk = useVddk; + } + @Override public boolean executeInSequence() { return false; @@ -34,4 +41,20 @@ public class CheckConvertInstanceCommand extends Command { public boolean getCheckWindowsGuestConversionSupport() { return checkWindowsGuestConversionSupport; } + + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index 24336747ccf..38e0dca7736 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -31,6 +31,10 @@ public class ConvertInstanceCommand extends Command { private boolean exportOvfToConversionLocation; private int threadsCountToExportOvf = 0; private String extraParams; + private boolean useVddk; + private String vddkLibDir; + private String vddkTransports; + private String vddkThumbprint; public ConvertInstanceCommand() { } @@ -90,6 +94,38 @@ public class ConvertInstanceCommand extends Command { this.extraParams = extraParams; } + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public void setVddkTransports(String vddkTransports) { + this.vddkTransports = vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public void setVddkThumbprint(String vddkThumbprint) { + this.vddkThumbprint = vddkThumbprint; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java index ed337885bee..21c4e7b97d0 100644 --- a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java +++ b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java @@ -24,6 +24,8 @@ import com.cloud.resource.ResourceState; public class PropagateResourceEventCommand extends Command { long hostId; ResourceState.Event event; + boolean forced; + boolean forceDeleteStorage; protected PropagateResourceEventCommand() { @@ -34,6 +36,13 @@ public class PropagateResourceEventCommand extends Command { this.event = event; } + public PropagateResourceEventCommand(long hostId, ResourceState.Event event, boolean forced, boolean forceDeleteStorage) { + this.hostId = hostId; + this.event = event; + this.forced = forced; + this.forceDeleteStorage = forceDeleteStorage; + } + public long getHostId() { return hostId; } @@ -42,6 +51,14 @@ public class PropagateResourceEventCommand extends Command { return event; } + public boolean isForced() { + return forced; + } + + public boolean isForceDeleteStorage() { + return forceDeleteStorage; + } + @Override public boolean executeInSequence() { // TODO Auto-generated method stub diff --git a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java index d8cc74817d7..96d73e11990 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java @@ -36,6 +36,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { public String lbStatsAuth = "admin1:AdMiN123"; public String lbStatsUri = "/admin?stats"; public String maxconn = ""; + public Long idleTimeout = 50000L; /* 0=infinite, >0 = timeout in milliseconds */ public String lbProtocol; public boolean keepAliveEnabled = false; NicTO nic; @@ -50,7 +51,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { } public LoadBalancerConfigCommand(LoadBalancerTO[] loadBalancers, String publicIp, String guestIp, String privateIp, NicTO nic, Long vpcId, String maxconn, - boolean keepAliveEnabled) { + boolean keepAliveEnabled, Long idleTimeout) { this.loadBalancers = loadBalancers; this.lbStatsPublicIP = publicIp; this.lbStatsPrivateIP = privateIp; @@ -59,6 +60,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { this.vpcId = vpcId; this.maxconn = maxconn; this.keepAliveEnabled = keepAliveEnabled; + this.idleTimeout = idleTimeout; } public NicTO getNic() { diff --git a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java index 128652fc64f..7d544c2e49c 100644 --- a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java +++ b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java @@ -635,6 +635,19 @@ public class HAProxyConfigurator implements LoadBalancerConfigurator { if (lbCmd.keepAliveEnabled) { dSection.set(7, "\tno option httpclose"); } + if (lbCmd.idleTimeout > 0) { + dSection.set(9, "\ttimeout client " + Long.toString(lbCmd.idleTimeout)); + dSection.set(10, "\ttimeout server " + Long.toString(lbCmd.idleTimeout)); + } else if (lbCmd.idleTimeout == 0) { + // .remove() is not allowed, only .set() operations are allowed as the list + // is a fixed size. So lets just mark the entry as blank. + dSection.set(9, ""); + dSection.set(10, ""); + } else { + // Negative idleTimeout values are considered invalid; retain the + // default HAProxy timeout values from defaultsSection for predictability. + logger.warn("Negative idleTimeout ({}) configured; retaining default HAProxy timeouts.", lbCmd.idleTimeout); + } if (logger.isDebugEnabled()) { for (final String s : dSection) { diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java index 6d4c6234c42..4ddadab999a 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java @@ -235,7 +235,7 @@ public class ConfigHelperTest { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 0L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java index 4196587cc3f..ed819bb7c68 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java @@ -779,7 +779,7 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer { final LoadBalancerTO[] arrayLbs = new LoadBalancerTO[lbs.size()]; lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; @@ -795,7 +795,7 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); nic.setIp("10.1.10.2"); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; diff --git a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java index 72361c2880e..073f976719b 100644 --- a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java +++ b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java @@ -79,13 +79,14 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("keepalive disabled should result in 'option httpclose' in the resulting haproxy config", result.contains("\toption httpclose")); - cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true); + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); result = genConfig(hpg, cmd); assertTrue("keepalive enabled should result in 'no option httpclose' in the resulting haproxy config", result.contains("\tno option httpclose")); + // TODO // create lb command // setup tests for @@ -93,6 +94,27 @@ public class HAProxyConfiguratorTest { // httpmode } + /** + * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. + */ + @Test + public void testGenerateConfigurationLoadBalancerIdleTimeoutConfigCommand() { + LoadBalancerTO lb = new LoadBalancerTO("1", "10.2.0.1", 80, "http", "bla", false, false, false, null); + LoadBalancerTO[] lba = new LoadBalancerTO[1]; + lba[0] = lb; + HAProxyConfigurator hpg = new HAProxyConfigurator(); + + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); + String result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 0 should not generate 'timeout server' in the resulting haproxy config", !result.contains("\ttimeout server")); + assertTrue("idleTimeout of 0 should not generate 'timeout client' in the resulting haproxy config", !result.contains("\ttimeout client")); + + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 1234L); + result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 1234 should result in 'timeout server 1234' in the resulting haproxy config", result.contains("\ttimeout server 1234")); + assertTrue("idleTimeout of 1234 should result in 'timeout client 1234' in the resulting haproxy config", result.contains("\ttimeout client 1234")); + } + /** * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. */ @@ -106,7 +128,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("'send-proxy' should result if protocol is 'tcp-proxy'", result.contains("send-proxy")); } @@ -118,7 +140,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("acl network_allowed src 1.1.1.1 2.2.2.2/24 \n\ttcp-request connection reject if !network_allowed")); } @@ -131,7 +153,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("bind 10.2.0.1:443 ssl crt /etc/cloudstack/ssl/10_2_0_1-443.pem")); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 615320e1788..b7b548fb940 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -122,6 +122,14 @@ public interface NetworkOrchestrationService { "Load Balancer(haproxy) maximum number of concurrent connections(global max)", true, Scope.Global); + ConfigKey NETWORK_LB_HAPROXY_IDLE_TIMEOUT = new ConfigKey<>( + "Network", + Long.class, + "network.loadbalancer.haproxy.idle.timeout", + "50000", + "Load Balancer(haproxy) idle timeout in milliseconds. Use 0 for infinite.", + true, + Scope.Global); List setupNetwork(Account owner, NetworkOffering offering, DeploymentPlan plan, String name, String displayText, boolean isDefault) throws ConcurrentOperationException; diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index e724f5d081b..4767e86e8ab 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -122,6 +122,8 @@ public interface ResourceManager extends ResourceService, Configurable { public boolean executeUserRequest(long hostId, ResourceState.Event event) throws AgentUnavailableException; + boolean executeUserRequest(long hostId, ResourceState.Event event, boolean isForced, boolean isForceDeleteStorage) throws AgentUnavailableException; + boolean resourceStateTransitTo(Host host, Event event, long msId) throws NoTransitionException; boolean umanageHost(long hostId); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 527c5259376..1215829d92f 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -805,8 +805,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); + String vddkSupport = detailsMap.get(Host.HOST_VDDK_SUPPORT); + String vddkLibDir = detailsMap.get(Host.HOST_VDDK_LIB_DIR); + String vddkVersion = detailsMap.get(Host.HOST_VDDK_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion)) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) { _hostDao.loadDetails(host); boolean updateNeeded = false; if (StringUtils.isNotBlank(uefiEnabled) && !uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { @@ -821,6 +824,26 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl host.getDetails().put(Host.HOST_OVFTOOL_VERSION, ovftoolVersion); updateNeeded = true; } + if (StringUtils.isNotBlank(vddkSupport) && !vddkSupport.equals(host.getDetails().get(Host.HOST_VDDK_SUPPORT))) { + host.getDetails().put(Host.HOST_VDDK_SUPPORT, vddkSupport); + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkLibDir).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_LIB_DIR)))) { + if (StringUtils.isBlank(vddkLibDir)) { + host.getDetails().remove(Host.HOST_VDDK_LIB_DIR); + } else { + host.getDetails().put(Host.HOST_VDDK_LIB_DIR, vddkLibDir); + } + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkVersion).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_VERSION)))) { + if (StringUtils.isBlank(vddkVersion)) { + host.getDetails().remove(Host.HOST_VDDK_VERSION); + } else { + host.getDetails().put(Host.HOST_VDDK_VERSION, vddkVersion); + } + updateNeeded = true; + } if (updateNeeded) { _hostDao.saveDetails(host); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index cfa0949883f..38a198b7304 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -1306,11 +1306,20 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust boolean result; try { - result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent()); + result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent(), cmd.isForced(), cmd.isForceDeleteStorage()); logger.debug("Result is {}", result); } catch (final AgentUnavailableException ex) { logger.warn("Agent is unavailable", ex); return null; + } catch (final RuntimeException ex) { + logger.error(String.format("Failed to execute propagated event %s for host %d", cmd.getEvent().name(), cmd.getHostId()), ex); + final Answer[] answers = new Answer[1]; + String details = ex.getMessage(); + if (details == null || details.isEmpty()) { + details = ex.toString(); + } + answers[0] = new Answer(cmd, false, details); + return _gson.toJson(answers); } final Answer[] answers = new Answer[1]; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 2ae228adba9..7d455e7d6dc 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -2776,219 +2776,218 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra boolean ipv6 = false; try (CheckedReservation networkReservation = new CheckedReservation(owner, domainId, Resource.ResourceType.network, null, null, 1L, reservationDao, _resourceLimitMgr)) { - - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - ipv6 = true; - } - // Validate zone - if (zone.getNetworkType() == NetworkType.Basic) { - // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true - if (aclType == null || aclType != ACLType.Domain) { - throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + ipv6 = true; } + // Validate zone + if (zone.getNetworkType() == NetworkType.Basic) { + // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true + if (aclType == null || aclType != ACLType.Domain) { + throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); + } - // Only one guest network is supported in Basic zone - final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); - if (!guestNetworks.isEmpty()) { - throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); - } + // Only one guest network is supported in Basic zone + final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); + if (!guestNetworks.isEmpty()) { + throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); + } - // if zone is basic, only Shared network offerings w/o source nat service are allowed - if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { - throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " - + Service.SourceNat.getName() + " service are allowed"); - } + // if zone is basic, only Shared network offerings w/o source nat service are allowed + if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { + throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " + + Service.SourceNat.getName() + " service are allowed"); + } - if (domainId == null || domainId != Domain.ROOT_DOMAIN) { - throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); - } + if (domainId == null || domainId != Domain.ROOT_DOMAIN) { + throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); + } - if (subdomainAccess == null) { - subdomainAccess = true; - } else if (!subdomainAccess) { - throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); - } + if (subdomainAccess == null) { + subdomainAccess = true; + } else if (!subdomainAccess) { + throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); + } - if (vlanId == null) { - vlanId = Vlan.UNTAGGED; - } else { - if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { - throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + if (vlanId == null) { + vlanId = Vlan.UNTAGGED; + } else { + if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { + throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + } + } + + } else if (zone.getNetworkType() == NetworkType.Advanced) { + if (zone.isSecurityGroupEnabled()) { + if (isolatedPvlan != null) { + throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); + } + // Only Account specific Isolated network with sourceNat service disabled are allowed in security group + // enabled zone + if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { + throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); + } + if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { + throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + } + } + + //don't allow eip/elb networks in Advance zone + if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { + throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); } } - } else if (zone.getNetworkType() == NetworkType.Advanced) { - if (zone.isSecurityGroupEnabled()) { - if (isolatedPvlan != null) { - throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); - } - // Only Account specific Isolated network with sourceNat service disabled are allowed in security group - // enabled zone - if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { - throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); - } - if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { - throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { + _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); + } + + //TODO(VXLAN): Support VNI specified + // VlanId can be specified only when network offering supports it + final boolean vlanSpecified = vlanId != null; + if (vlanSpecified != ntwkOff.isSpecifyVlan()) { + if (vlanSpecified) { + if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + } + } else { + throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); } } - //don't allow eip/elb networks in Advance zone - if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { - throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); - } - } - - if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { - _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); - } - - //TODO(VXLAN): Support VNI specified - // VlanId can be specified only when network offering supports it - final boolean vlanSpecified = vlanId != null; - if (vlanSpecified != ntwkOff.isSpecifyVlan()) { if (vlanSpecified) { - if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); + // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks + URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; + if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + bypassVlanOverlapCheck = true; } - } else { - throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); - } - } - - if (vlanSpecified) { - URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); - // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks - URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; - if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - bypassVlanOverlapCheck = true; - } - //don't allow to specify vlan tag used by physical network for dynamic vlan allocation - if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) - && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " - + zone.getName()); - } - if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && - _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { - throw new InvalidParameterValueException(String.format( - "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", - isolatedPvlan, zone)); - } - if (!UuidUtils.isUuid(vlanId)) { - // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone - if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { - if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - vlanId, zone)); - } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - isolatedPvlan, zone)); - } else { - final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); - //for the network that is created as part of private gateway, - //the vnet is not coming from the data center vnet table, so the list can be empty - if (!dcVnets.isEmpty()) { - final DataCenterVnetVO dcVnet = dcVnets.get(0); - // Fail network creation if specified vlan is dedicated to a different account - if (dcVnet.getAccountGuestVlanMapId() != null) { - final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); - final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); - if (map.getAccountId() != owner.getAccountId()) { - throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); - } - // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool - } else { - final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); - if (maps != null && !maps.isEmpty()) { - final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); - final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); - if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { - throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " - + owner.getAccountName()); + //don't allow to specify vlan tag used by physical network for dynamic vlan allocation + if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) + && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { + throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + + zone.getName()); + } + if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && + _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { + throw new InvalidParameterValueException(String.format( + "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", + isolatedPvlan, zone)); + } + if (!UuidUtils.isUuid(vlanId)) { + // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone + if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { + if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + vlanId, zone)); + } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + isolatedPvlan, zone)); + } else { + final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); + //for the network that is created as part of private gateway, + //the vnet is not coming from the data center vnet table, so the list can be empty + if (!dcVnets.isEmpty()) { + final DataCenterVnetVO dcVnet = dcVnets.get(0); + // Fail network creation if specified vlan is dedicated to a different account + if (dcVnet.getAccountGuestVlanMapId() != null) { + final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); + final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); + if (map.getAccountId() != owner.getAccountId()) { + throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); + } + // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool + } else { + final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); + if (maps != null && !maps.isEmpty()) { + final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); + final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); + if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { + throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " + + owner.getAccountName()); + } } } } } + } else { + // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or + // shared network with same Vlan ID in the zone + if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { + throw new InvalidParameterValueException(String.format( + "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + } } - } else { - // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or - // shared network with same Vlan ID in the zone - if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { + } + + } + + // If networkDomain is not specified, take it from the global configuration + if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { + final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), + Service.Dns); + final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); + if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { + if (networkDomain != null) { + // TBD: NetworkOfferingId and zoneId. Send uuids instead. throw new InvalidParameterValueException(String.format( - "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + "Domain name change is not supported by network offering id=%d in zone %s", + networkOfferingId, zone)); } - } - } - - } - - // If networkDomain is not specified, take it from the global configuration - if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { - final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), - Service.Dns); - final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); - if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { - if (networkDomain != null) { - // TBD: NetworkOfferingId and zoneId. Send uuids instead. - throw new InvalidParameterValueException(String.format( - "Domain name change is not supported by network offering id=%d in zone %s", - networkOfferingId, zone)); - } - } else { - if (networkDomain == null) { - // 1) Get networkDomain from the corresponding account/domain/zone - if (aclType == ACLType.Domain) { - networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); - } else if (aclType == ACLType.Account) { - networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); - } - - // 2) If null, generate networkDomain using domain suffix from the global config variables - if (networkDomain == null) { - networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); - } - } else { - // validate network domain - if (!NetUtils.verifyDomainName(networkDomain)) { - throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " - + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " - + "and the hyphen ('-'); can't start or end with \"-\""); + if (networkDomain == null) { + // 1) Get networkDomain from the corresponding account/domain/zone + if (aclType == ACLType.Domain) { + networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); + } else if (aclType == ACLType.Account) { + networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); + } + + // 2) If null, generate networkDomain using domain suffix from the global config variables + if (networkDomain == null) { + networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); + } + + } else { + // validate network domain + if (!NetUtils.verifyDomainName(networkDomain)) { + throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " + + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " + + "and the hyphen ('-'); can't start or end with \"-\""); + } } } } - } - // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x - // limitation, remove after we introduce support for multiple ip ranges - // with different Cidrs for the same Shared network - final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced - && ntwkOff.getTrafficType() == TrafficType.Guest - && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); - if (cidr == null && ip6Cidr == null && cidrRequired) { - if (ntwkOff.getGuestType() == GuestType.Shared) { - throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); - } else { - throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x + // limitation, remove after we introduce support for multiple ip ranges + // with different Cidrs for the same Shared network + final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced + && ntwkOff.getTrafficType() == TrafficType.Guest + && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); + if (cidr == null && ip6Cidr == null && cidrRequired) { + if (ntwkOff.getGuestType() == GuestType.Shared) { + throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); + } else { + throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + } } - } - checkL2OfferingServices(ntwkOff); + checkL2OfferingServices(ntwkOff); - // No cidr can be specified in Basic zone - if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { - throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); - } + // No cidr can be specified in Basic zone + if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { + throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); + } - // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 - if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && - !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { + // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 + if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && + !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { throw new InvalidParameterValueException("Virtual Guest Cidr " + cidr + " is not RFC 1918 or 6598 compliant"); - } + } final String networkDomainFinal = networkDomain; final String vlanIdFinal = vlanId; @@ -3004,75 +3003,75 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkVO userNetwork = new NetworkVO(); userNetwork.setNetworkDomain(networkDomainFinal); - if (cidr != null && gateway != null) { - userNetwork.setCidr(cidr); - userNetwork.setGateway(gateway); - } + if (cidr != null && gateway != null) { + userNetwork.setCidr(cidr); + userNetwork.setGateway(gateway); + } - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - userNetwork.setIp6Cidr(ip6Cidr); - userNetwork.setIp6Gateway(ip6Gateway); - } + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + userNetwork.setIp6Cidr(ip6Cidr); + userNetwork.setIp6Gateway(ip6Gateway); + } - if (externalId != null) { - userNetwork.setExternalId(externalId); - } + if (externalId != null) { + userNetwork.setExternalId(externalId); + } - if (StringUtils.isNotBlank(routerIp)) { - userNetwork.setRouterIp(routerIp); - } + if (StringUtils.isNotBlank(routerIp)) { + userNetwork.setRouterIp(routerIp); + } - if (StringUtils.isNotBlank(routerIpv6)) { - userNetwork.setRouterIpv6(routerIpv6); - } + if (StringUtils.isNotBlank(routerIpv6)) { + userNetwork.setRouterIpv6(routerIpv6); + } - if (vrIfaceMTUs != null) { - if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { - userNetwork.setPublicMtu(vrIfaceMTUs.first()); + if (vrIfaceMTUs != null) { + if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { + userNetwork.setPublicMtu(vrIfaceMTUs.first()); + } else { + userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); + } + + if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { + userNetwork.setPrivateMtu(vrIfaceMTUs.second()); + } else { + userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); + } } else { userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); - } - - if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { - userNetwork.setPrivateMtu(vrIfaceMTUs.second()); - } else { userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); } - } else { - userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); - userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); - } - if (!GuestType.L2.equals(userNetwork.getGuestType())) { - if (StringUtils.isNotBlank(ip4Dns1)) { - userNetwork.setDns1(ip4Dns1); - } - if (StringUtils.isNotBlank(ip4Dns2)) { - userNetwork.setDns2(ip4Dns2); - } - if (StringUtils.isNotBlank(ip6Dns1)) { - userNetwork.setIp6Dns1(ip6Dns1); - } - if (StringUtils.isNotBlank(ip6Dns2)) { - userNetwork.setIp6Dns2(ip6Dns2); - } - } - - if (vlanIdFinal != null) { - if (isolatedPvlan == null) { - URI uri = null; - if (UuidUtils.isUuid(vlanIdFinal)) { - //Logical router's UUID provided as VLAN_ID - userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field - } else { - uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); + if (!GuestType.L2.equals(userNetwork.getGuestType())) { + if (StringUtils.isNotBlank(ip4Dns1)) { + userNetwork.setDns1(ip4Dns1); } - - if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", - vlanIdFinal, zone)); + if (StringUtils.isNotBlank(ip4Dns2)) { + userNetwork.setDns2(ip4Dns2); } + if (StringUtils.isNotBlank(ip6Dns1)) { + userNetwork.setIp6Dns1(ip6Dns1); + } + if (StringUtils.isNotBlank(ip6Dns2)) { + userNetwork.setIp6Dns2(ip6Dns2); + } + } + + if (vlanIdFinal != null) { + if (isolatedPvlan == null) { + URI uri = null; + if (UuidUtils.isUuid(vlanIdFinal)) { + //Logical router's UUID provided as VLAN_ID + userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field + } else { + uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); + } + + if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", + vlanIdFinal, zone)); + } userNetwork.setBroadcastUri(uri); if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { @@ -4940,6 +4939,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return new ConfigKey[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes, GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled, - TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN}; + TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN, + NETWORK_LB_HAPROXY_IDLE_TIMEOUT}; } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java index 445a59310fb..377b2f91375 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.upgrade; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -96,7 +97,9 @@ public final class DatabaseVersionHierarchy { // we cannot find the version specified, so get the // most recent one immediately before this version if (!contains(fromVersion)) { - return getPath(getRecentVersion(fromVersion), toVersion); + DbUpgrade[] dbUpgrades = getPath(getRecentVersion(fromVersion), toVersion); + return Arrays.stream(dbUpgrades).filter(up -> CloudStackVersion.compare(up.getUpgradedVersion(), fromVersion.toString()) > 0) + .toArray(DbUpgrade[]::new); } final Predicate predicate; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java index 68100e16401..e0aa38a717b 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java @@ -57,8 +57,4 @@ public class Upgrade42020to42030 extends DbUpgradeAbstractImpl implements DbUpgr public InputStream[] getCleanupScripts() { return null; } - - @Override - public void updateSystemVmTemplates(Connection conn) { - } } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml index 1846c3c62a0..2c6869bd81e 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml @@ -73,5 +73,7 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 0e72337ec45..26181d3fce0 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -237,13 +237,11 @@ - - diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql index 3dd6c18f57c..4405250f271 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql @@ -31,7 +31,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" '); -- Create a new group for Usage Server related configurations -INSERT INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); +INSERT IGNORE INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); UPDATE `cloud`.`configuration_subgroup` set `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server'), `precedence` = 1 WHERE `name`='Usage'; UPDATE `cloud`.`configuration` SET `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server') where `subgroup_id` = (SELECT `id` FROM `cloud`.`configuration_subgroup` WHERE `name` = 'Usage'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index 43e0d2c78a9..baa20e9f0ad 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -53,3 +53,9 @@ DELETE FROM `cloud`.`configuration` WHERE name = 'consoleproxy.cmd.port'; -- Drops the unused "backup_interval_type" column of the "cloud.backups" table ALTER TABLE `cloud`.`backups` DROP COLUMN `backup_interval_type`; + +-- Update `user.password.reset.mail.template` configuration value to match new logic +UPDATE `cloud`.`configuration` +SET value = CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team') +WHERE name = 'user.password.reset.mail.template' + AND value IN (CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', 'http://{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team'), CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{domainUrl}}}{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team')); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index b3f3319e13b..6b2c80c99a6 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -118,6 +118,16 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','conserve_mode', 'tin --- Disable/enable NICs CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); +--- Quota tariff/usage mapping +CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_tariff_usage` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `tariff_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the tariff of the Quota usage detail calculated, foreign key to quota_tariff table', + `quota_usage_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the aggregation of Quota usage details, foreign key to quota_usage table', + `quota_used` decimal(20,8) NOT NULL COMMENT 'Amount of quota used', + PRIMARY KEY (`id`), + CONSTRAINT `fk_quota_tariff_usage__tariff_id` FOREIGN KEY (`tariff_id`) REFERENCES `cloud_usage`.`quota_tariff` (`id`), + CONSTRAINT `fk_quota_tariff_usage__quota_usage_id` FOREIGN KEY (`quota_usage_id`) REFERENCES `cloud_usage`.`quota_usage` (`id`)); + -- Add management_server_details table to allow ManagementServer scope configs CREATE TABLE IF NOT EXISTS `management_server_details` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql new file mode 100644 index 00000000000..7ac001384e8 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud_usage`.`quota_usage_view`; + +DROP VIEW IF EXISTS `cloud_usage`.`quota_usage_view`; +CREATE VIEW `cloud_usage`.`quota_usage_view` AS +SELECT qu.id, + qu.usage_item_id, + qu.zone_id, + qu.account_id, + qu.domain_id, + qu.usage_type, + qu.quota_used, + qu.start_date, + qu.end_date, + cu.usage_id AS resource_id, + cu.network_id as network_id, + cu.offering_id as offering_id +FROM `cloud_usage`.`quota_usage` qu +INNER JOIN `cloud_usage`.`cloud_usage` cu ON (cu.id = qu.usage_item_id); diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 27995eb179a..ab64e4698f0 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -44,6 +44,7 @@ import com.cloud.upgrade.dao.Upgrade41120to41130; import com.cloud.upgrade.dao.Upgrade41120to41200; import com.cloud.upgrade.dao.Upgrade41510to41520; import com.cloud.upgrade.dao.Upgrade41610to41700; +import com.cloud.upgrade.dao.Upgrade42010to42100; import com.cloud.upgrade.dao.Upgrade452to453; import com.cloud.upgrade.dao.Upgrade453to460; import com.cloud.upgrade.dao.Upgrade460to461; @@ -380,4 +381,23 @@ public class DatabaseUpgradeCheckerTest { assertFalse("DatabaseUpgradeChecker should not be a standalone component", checker.isStandalone()); } + @Test + public void testCalculateUpgradePath42010to42100() { + + final CloudStackVersion dbVersion = CloudStackVersion.parse("4.20.1.0"); + assertNotNull(dbVersion); + + final CloudStackVersion currentVersion = CloudStackVersion.parse("4.21.0.0"); + assertNotNull(currentVersion); + + final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); + final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); + + assertNotNull(upgrades); + assertEquals(1, upgrades.length); + assertTrue(upgrades[0] instanceof Upgrade42010to42100); + + assertArrayEquals(new String[]{"4.20.1.0", "4.21.0.0"}, upgrades[0].getUpgradableVersionRange()); + assertEquals(currentVersion.toString(), upgrades[0].getUpgradedVersion()); + } } diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 15807eb26d4..3323d5c4d82 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -205,6 +205,12 @@ public class SearchCriteria { } + public void setJoinParametersIfNotNull(String joinName, String conditionName, Object... params) { + if (ArrayUtils.isNotEmpty(params) && (params.length > 1 || params[0] != null)) { + setJoinParameters(joinName, conditionName, params); + } + } + public SearchCriteria getJoin(String joinName) { return _joins.get(joinName).getT(); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index a2fca7b80fd..cd87052e878 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -28,11 +28,12 @@ import java.util.stream.Collectors; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; -import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -66,6 +67,7 @@ import com.cloud.domain.dao.DomainDao; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostTagsDao; +import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.server.ResourceTag; @@ -191,6 +193,9 @@ public class PresetVariableHelper { @Inject ClusterDetailsDao clusterDetailsDao; + @Inject + VpcOfferingDao vpcOfferingDao; + protected boolean backupSnapshotAfterTakingSnapshot = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value(); private List runningAndAllocatedVmUsageTypes = Arrays.asList(UsageTypes.RUNNING_VM, UsageTypes.ALLOCATED_VM); @@ -778,6 +783,19 @@ public class PresetVariableHelper { value.setId(network.getUuid()); value.setName(network.getName()); value.setState(usageRecord.getState()); + + value.setNetworkOffering(getPresetVariableValueNetworkOffering(network.getNetworkOfferingId())); + } + + protected GenericPresetVariable getPresetVariableValueNetworkOffering(Long networkOfferingId) { + NetworkOfferingVO networkOfferingVo = networkOfferingDao.findByIdIncludingRemoved(networkOfferingId); + validateIfObjectIsNull(networkOfferingVo, networkOfferingId, "network offering"); + + GenericPresetVariable networkOffering = new GenericPresetVariable(); + networkOffering.setId(networkOfferingVo.getUuid()); + networkOffering.setName(networkOfferingVo.getName()); + + return networkOffering; } protected void loadPresetVariableValueForVpc(UsageVO usageRecord, Value value) { @@ -793,6 +811,18 @@ public class PresetVariableHelper { value.setId(vpc.getUuid()); value.setName(vpc.getName()); + value.setVpcOffering(getPresetVariableValueVpcOffering(vpc.getVpcOfferingId())); + } + + protected GenericPresetVariable getPresetVariableValueVpcOffering(Long vpcOfferingId) { + VpcOfferingVO vpcOfferingVo = vpcOfferingDao.findByIdIncludingRemoved(vpcOfferingId); + validateIfObjectIsNull(vpcOfferingVo, vpcOfferingId, "vpc offering"); + + GenericPresetVariable vpcOffering = new GenericPresetVariable(); + vpcOffering.setId(vpcOfferingVo.getUuid()); + vpcOffering.setName(vpcOfferingVo.getName()); + + return vpcOffering; } /** diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index ac776d13c57..aff2c040e74 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -96,6 +96,12 @@ public class Value extends GenericPresetVariable { private String state; + @PresetVariableDefinition(description = "Network offering of the network.", supportedTypes = {QuotaTypes.NETWORK}) + private GenericPresetVariable networkOffering; + + @PresetVariableDefinition(description = "VPC offering of the VPC.", supportedTypes = {QuotaTypes.VPC}) + private GenericPresetVariable vpcOffering; + public Host getHost() { return host; } @@ -255,4 +261,20 @@ public class Value extends GenericPresetVariable { public void setState(String state) { this.state = state; } + + public GenericPresetVariable getNetworkOffering() { + return networkOffering; + } + + public void setNetworkOffering(GenericPresetVariable networkOffering) { + this.networkOffering = networkOffering; + } + + public GenericPresetVariable getVpcOffering() { + return vpcOffering; + } + + public void setVpcOffering(GenericPresetVariable vpcOffering) { + this.vpcOffering = vpcOffering; + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java new file mode 100644 index 00000000000..9684ca117b5 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java @@ -0,0 +1,29 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import java.util.List; + +public interface QuotaTariffUsageDao extends GenericDao { + + void persistQuotaTariffUsage(QuotaTariffUsageVO quotaTariffUsage); + + List listQuotaTariffUsages(Long quotaUsageId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java new file mode 100644 index 00000000000..556f552fed6 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Component +public class QuotaTariffUsageDaoImpl extends GenericDaoBase implements QuotaTariffUsageDao { + private SearchBuilder searchQuotaTariffUsages; + + @PostConstruct + public void init() { + searchQuotaTariffUsages = createSearchBuilder(); + searchQuotaTariffUsages.and("quotaUsageId", searchQuotaTariffUsages.entity().getQuotaUsageId(), SearchCriteria.Op.EQ); + searchQuotaTariffUsages.done(); + } + + @Override + public void persistQuotaTariffUsage(final QuotaTariffUsageVO quotaTariffUsage) { + logger.trace("Persisting quota tariff usage [{}].", quotaTariffUsage); + Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(quotaTariffUsage)); + } + + @Override + public List listQuotaTariffUsages(Long quotaUsageId) { + SearchCriteria sc = searchQuotaTariffUsages.create(); + sc.setParameters("quotaUsageId", quotaUsageId); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java new file mode 100644 index 00000000000..126fa11413f --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; + +import java.util.Date; +import java.util.List; + +public interface QuotaUsageJoinDao extends GenericDao { + + List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java new file mode 100644 index 00000000000..b98ea2b3a5d --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.JoinBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.Date; +import java.util.List; + +@Component +public class QuotaUsageJoinDaoImpl extends GenericDaoBase implements QuotaUsageJoinDao { + + private SearchBuilder searchQuotaUsages; + + private SearchBuilder searchQuotaUsagesJoinTariffUsages; + + @Inject + private QuotaTariffUsageDao quotaTariffUsageDao; + + @PostConstruct + public void init() { + searchQuotaUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsages); + searchQuotaUsages.done(); + + SearchBuilder searchQuotaTariffUsages = quotaTariffUsageDao.createSearchBuilder(); + searchQuotaTariffUsages.and("tariffId", searchQuotaTariffUsages.entity().getTariffId(), SearchCriteria.Op.EQ); + searchQuotaUsagesJoinTariffUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsagesJoinTariffUsages); + searchQuotaUsagesJoinTariffUsages.join("searchQuotaTariffUsages", searchQuotaTariffUsages, searchQuotaUsagesJoinTariffUsages.entity().getId(), + searchQuotaTariffUsages.entity().getQuotaUsageId(), JoinBuilder.JoinType.INNER); + searchQuotaUsagesJoinTariffUsages.done(); + } + + private void prepareQuotaUsageSearchBuilder(SearchBuilder searchBuilder) { + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("usageType", searchBuilder.entity().getUsageType(), SearchCriteria.Op.EQ); + searchBuilder.and("resourceId", searchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); + searchBuilder.and("networkId", searchBuilder.entity().getNetworkId(), SearchCriteria.Op.EQ); + searchBuilder.and("offeringId", searchBuilder.entity().getOfferingId(), SearchCriteria.Op.EQ); + searchBuilder.and("startDate", searchBuilder.entity().getStartDate(), SearchCriteria.Op.BETWEEN); + searchBuilder.and("endDate", searchBuilder.entity().getEndDate(), SearchCriteria.Op.BETWEEN); + } + + @Override + public List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId) { + SearchCriteria sc = tariffId == null ? searchQuotaUsages.create() : searchQuotaUsagesJoinTariffUsages.create(); + + sc.setParametersIfNotNull("accountId", accountId); + sc.setParametersIfNotNull("domainId", domainId); + sc.setParametersIfNotNull("usageType", usageType); + sc.setParametersIfNotNull("resourceId", resourceId); + sc.setParametersIfNotNull("networkId", networkId); + sc.setParametersIfNotNull("offeringId", offeringId); + + if (ObjectUtils.allNotNull(startDate, endDate)) { + sc.setParameters("startDate", startDate, endDate); + sc.setParameters("endDate", startDate, endDate); + } + + sc.setJoinParametersIfNotNull("searchQuotaTariffUsages", "tariffId", tariffId); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java new file mode 100644 index 00000000000..4fa9e771713 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java @@ -0,0 +1,86 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +@Entity +@Table(name = "quota_tariff_usage") +public class QuotaTariffUsageVO implements InternalIdentity { + @Id + @Column(name = "id") + private Long id; + + @Column(name = "tariff_id") + private Long tariffId; + + @Column(name = "quota_usage_id") + private Long quotaUsageId; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + public QuotaTariffUsageVO() { + quotaUsed = new BigDecimal(0); + } + + @Override + public long getId() { + return id; + } + + public Long getTariffId() { + return tariffId; + } + + public Long getQuotaUsageId() { + return quotaUsageId; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setId(Long id) { + this.id = id; + } + + public void setTariffId(Long tariffId) { + this.tariffId = tariffId; + } + + public void setQuotaUsageId(Long quotaUsageId) { + this.quotaUsageId = quotaUsageId; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + @Override + public String toString() { + return new ReflectionToStringBuilder(this, ToStringStyle.JSON_STYLE).toString(); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java new file mode 100644 index 00000000000..df9577e23c3 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java @@ -0,0 +1,179 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.math.BigDecimal; +import java.util.Date; + +@Entity +@Table(name = "quota_usage_view") +public class QuotaUsageJoinVO implements InternalIdentity { + + @Id + @Column(name = "id", updatable = false, nullable = false) + private Long id; + + @Column(name = "zone_id") + private Long zoneId = null; + + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "domain_id") + private Long domainId = null; + + @Column(name = "usage_item_id") + private Long usageItemId; + + @Column(name = "usage_type") + private int usageType; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + @Column(name = "start_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date startDate = null; + + @Column(name = "end_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date endDate = null; + + @Column(name = "resource_id") + private Long resourceId = null; + + @Column(name = "network_id") + private Long networkId = null; + + @Column(name = "offering_id") + private Long offeringId = null; + + @Override + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getZoneId() { + return zoneId; + } + + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public Long getDomainId() { + return domainId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public Long getUsageItemId() { + return usageItemId; + } + + public void setUsageItemId(Long usageItemId) { + this.usageItemId = usageItemId; + } + + public int getUsageType() { + return usageType; + } + + public void setUsageType(int usageType) { + this.usageType = usageType; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } + + public Long getNetworkId() { + return networkId; + } + + public void setNetworkId(Long networkId) { + this.networkId = networkId; + } + + public Long getOfferingId() { + return offeringId; + } + + public void setOfferingId(Long offeringId) { + this.offeringId = offeringId; + } + + public QuotaUsageJoinVO () { + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "zoneId", "accountId", "domainId", "usageItemId", "usageType", "quotaUsed", "startDate", + "endDate", "resourceId"); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java new file mode 100644 index 00000000000..92482423100 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java @@ -0,0 +1,62 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.vo; + +import java.util.Date; + +public class QuotaUsageResourceVO { + private String uuid; + private String name; + private Date removed; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public boolean isRemoved() { + return this.removed != null; + } + + public QuotaUsageResourceVO(String uuid, String name, Date removed) { + this.uuid = uuid; + this.name = name; + this.removed = removed; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 304b23b7220..5ca2679c388 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -26,7 +26,9 @@ - + + + diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index 85397503587..fb093a45679 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -31,6 +31,10 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.vpc.VpcOfferingVO; +import com.cloud.network.vpc.VpcVO; +import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; @@ -38,7 +42,9 @@ import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.cloudstack.quota.dao.NetworkDao; import org.apache.cloudstack.quota.dao.VmTemplateDao; +import org.apache.cloudstack.quota.dao.VpcDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -188,6 +194,15 @@ public class PresetVariableHelperTest { @Mock BackupOfferingDao backupOfferingDaoMock; + @Mock + NetworkDao networkDaoMock; + + @Mock + VpcDao vpcDaoMock; + + @Mock + VpcOfferingDao vpcOfferingDaoMock; + List runningAndAllocatedVmUsageTypes = Arrays.asList(UsageTypes.RUNNING_VM, UsageTypes.ALLOCATED_VM); List templateAndIsoUsageTypes = Arrays.asList(UsageTypes.TEMPLATE, UsageTypes.ISO); @@ -223,6 +238,8 @@ public class PresetVariableHelperTest { value.setVmSnapshotType(VMSnapshot.Type.Disk.toString()); value.setComputingResources(getComputingResourcesForTests()); value.setVolumeType(Volume.Type.DATADISK.toString()); + value.setNetworkOffering(getNetworkOfferingForTests()); + value.setVpcOffering(getVpcOfferingForTests()); return value; } @@ -324,6 +341,20 @@ public class PresetVariableHelperTest { return diskOffering; } + private GenericPresetVariable getNetworkOfferingForTests() { + GenericPresetVariable networkOffering = new GenericPresetVariable(); + networkOffering.setId("network_offering_id"); + networkOffering.setName("network_offering_name"); + return networkOffering; + } + + private GenericPresetVariable getVpcOfferingForTests() { + GenericPresetVariable vpcOffering = new GenericPresetVariable(); + vpcOffering.setId("vpc_offering_id"); + vpcOffering.setName("vpc_offering_name"); + return vpcOffering; + } + private void mockMethodValidateIfObjectIsNull() { Mockito.doNothing().when(presetVariableHelperSpy).validateIfObjectIsNull(Mockito.any(), Mockito.anyLong(), Mockito.anyString()); } @@ -1289,4 +1320,100 @@ public class PresetVariableHelperTest { Mockito.when(imageStoreDaoMock.findById(1L)).thenReturn(store); Assert.assertNotNull(presetVariableHelperSpy.getSnapshotImageStoreRef(1L, 1L)); } + + @Test + public void loadPresetVariableValueForNetworkTestRecordIsNotANetworkDoNothing() { + getQuotaTypesForTests(UsageTypes.NETWORK).forEach(type -> { + Mockito.doReturn(type.getKey()).when(usageVoMock).getUsageType(); + presetVariableHelperSpy.loadPresetVariableValueForNetwork(usageVoMock, null); + }); + + Mockito.verifyNoInteractions(networkDaoMock); + } + + @Test + public void loadPresetVariableValueForNetworkTestRecordIsNetworkSetFields() { + Value expected = getValueForTests(); + + NetworkVO networkVoMock = Mockito.mock(NetworkVO.class); + Mockito.doReturn(networkVoMock).when(networkDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + mockMethodValidateIfObjectIsNull(); + + Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); + Mockito.doReturn(expected.getState()).when(usageVoMock).getState(); + Mockito.doReturn(expected.getNetworkOffering()).when(presetVariableHelperSpy).getPresetVariableValueNetworkOffering(Mockito.anyLong()); + + Mockito.doReturn(UsageTypes.NETWORK).when(usageVoMock).getUsageType(); + + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForNetwork(usageVoMock, result); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getState(), result.getState()); + Assert.assertEquals(expected.getNetworkOffering(), result.getNetworkOffering()); + } + + @Test + public void loadPresetVariableValueForVpcTestRecordIsNotAVpcDoNothing() { + getQuotaTypesForTests(UsageTypes.VPC).forEach(type -> { + Mockito.doReturn(type.getKey()).when(usageVoMock).getUsageType(); + presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, null); + }); + + Mockito.verifyNoInteractions(networkDaoMock); + } + + @Test + public void loadPresetVariableValueForVpcTestRecordIsVpcSetFields() { + Value expected = getValueForTests(); + + VpcVO networkVoMock = Mockito.mock(VpcVO.class); + Mockito.doReturn(networkVoMock).when(vpcDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + mockMethodValidateIfObjectIsNull(); + + Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); + Mockito.doReturn(expected.getVpcOffering()).when(presetVariableHelperSpy).getPresetVariableValueVpcOffering(Mockito.anyLong()); + + Mockito.doReturn(UsageTypes.VPC).when(usageVoMock).getUsageType(); + + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, result); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getVpcOffering(), result.getVpcOffering()); + } + + @Test + public void getPresetVariableValueNetworkOfferingTestSetValuesAndReturnObject() { + NetworkOfferingVO networkOfferingVoMock = Mockito.mock(NetworkOfferingVO.class); + Mockito.doReturn(networkOfferingVoMock).when(networkOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + + GenericPresetVariable expected = getGenericPresetVariableForTests(); + Mockito.doReturn(expected.getId()).when(networkOfferingVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkOfferingVoMock).getName(); + + GenericPresetVariable result = presetVariableHelperSpy.getPresetVariableValueNetworkOffering(1L); + + assertPresetVariableIdAndName(expected, result); + } + + @Test + public void getPresetVariableValueVpcOfferingTestSetValuesAndReturnObject() { + VpcOfferingVO vpcOfferingVoMock = Mockito.mock(VpcOfferingVO.class); + Mockito.doReturn(vpcOfferingVoMock).when(vpcOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + + GenericPresetVariable expected = getGenericPresetVariableForTests(); + Mockito.doReturn(expected.getId()).when(vpcOfferingVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(vpcOfferingVoMock).getName(); + + GenericPresetVariable result = presetVariableHelperSpy.getPresetVariableValueVpcOffering(1L); + + assertPresetVariableIdAndName(expected, result); + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index d3bd3868ed1..bfe26a9f425 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; @@ -28,24 +27,25 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementItemResponse; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; -import com.cloud.user.Account; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a quota statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - httpMethod = "GET") +@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a Quota statement for the provided Account, Project, or Domain.", + since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaStatementCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Optional, Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, + description = "Name of the Account for which the Quota statement will be generated. Deprecated, please use accountid instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "ID of the Domain for which the Quota statement will be generated. May be used individually or with account.") private Long domainId; @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End of the period of the Quota statement. " + @@ -56,15 +56,25 @@ public class QuotaStatementCmd extends BaseCmd { ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; - @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type") + @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, + description = "Consider only Quota usage records for the specified usage type in the statement.") private Integer usageType; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "ID of the Account for which the Quota statement will be generated. Can not be specified with projectid.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, + description = "ID of the Project for which the Quota statement will be generated. Can not be specified with accountid.", since = "4.23.0") + private Long projectId; + + @Parameter(name = ApiConstants.SHOW_RESOURCES, type = CommandType.BOOLEAN, description = "List the resources of each Quota type in the period.", since = "4.23.0") + private boolean showResources; + @Inject - private QuotaResponseBuilder _responseBuilder; + QuotaResponseBuilder responseBuilder; public Long getAccountId() { return accountId; @@ -99,43 +109,47 @@ public class QuotaStatementCmd extends BaseCmd { } public Date getEndDate() { - return _responseBuilder.startOfNextDay(endDate == null ? new Date() : new Date(endDate.getTime())); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; + } + + public boolean isShowResources() { + return showResources; + } + + public void setShowResources(boolean showResources) { + this.showResources = showResources; + } + + public Long getProjectId() { + return projectId; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; } - Account activeAccountByName = _accountService.getActiveAccountByName(accountName, domainId); - if (activeAccountByName != null) { - return activeAccountByName.getAccountId(); - } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaUsage(this); - - QuotaStatementResponse response = _responseBuilder.createQuotaStatementResponse(quotaUsage); - response.setStartDate(startDate == null ? null : new Date(startDate.getTime())); - response.setEndDate(endDate == null ? null : new Date(endDate.getTime())); - + QuotaStatementResponse response = responseBuilder.createQuotaStatementResponse(this); + response.setStartDate(startDate); + response.setEndDate(endDate); response.setResponseName(getCommandName()); setResponseObject(response); } - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 177fb00d4b5..bde905c487b 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -32,7 +32,6 @@ import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import java.util.Date; import java.util.List; @@ -49,7 +48,7 @@ public interface QuotaResponseBuilder { boolean isUserAllowedToSeeActivationRules(User user); - QuotaStatementResponse createQuotaStatementResponse(List quotaUsage); + QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd); QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); @@ -57,8 +56,6 @@ public interface QuotaResponseBuilder { QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); - List getQuotaUsage(QuotaStatementCmd cmd); - List getQuotaBalance(QuotaBalanceCmd cmd); QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 2d6ec3255f4..173c0723731 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -21,13 +21,11 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.math.BigDecimal; -import java.math.RoundingMode; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -36,6 +34,7 @@ import java.util.Iterator; import java.util.List; import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -43,12 +42,36 @@ import java.util.stream.Collectors; import javax.inject.Inject; import com.cloud.domain.Domain; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.projects.dao.ProjectDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.SnapshotVO; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; @@ -94,7 +117,8 @@ import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.cloudstack.quota.vo.QuotaUsageResourceVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections4.CollectionUtils; @@ -106,18 +130,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.domain.DomainVO; -import com.cloud.domain.dao.DomainDao; -import com.cloud.event.ActionEvent; -import com.cloud.event.EventTypes; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.UserDao; -import com.cloud.utils.Pair; - @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { protected Logger logger = LogManager.getLogger(getClass()); @@ -140,8 +152,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject - private ProjectDao projectDao; - @Inject private QuotaAccountDao quotaAccountDao; @Inject private DomainDao domainDao; @@ -159,6 +169,21 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; + @Inject + private IPAddressDao ipAddressDao; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkOfferingDao networkOfferingDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private VMInstanceDao vmInstanceDao; + @Inject + private VMTemplateDao vmTemplateDao; + @Inject + private VolumeDao volumeDao; + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; @@ -393,78 +418,212 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } @Override - public QuotaStatementResponse createQuotaStatementResponse(final List quotaUsage) { - if (quotaUsage == null || quotaUsage.isEmpty()) { - throw new InvalidParameterValueException("There is no usage data found for period mentioned."); - } + public QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd) { + Long accountId = getAccountIdForQuotaStatement(cmd); + Long domainId = getDomainIdForQuotaStatement(cmd, accountId); + List quotaUsages = _quotaService.getQuotaUsage(accountId, null, domainId, cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); + + logger.debug("Creating quota statement from [{}] usage records for parameters [{}].", quotaUsages.size(), + ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "accountId", "projectId", "domainId", "startDate", "endDate", "usageType", "showResources")); + createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(quotaUsages, cmd.getUsageType()); + + Map> recordsPerUsageTypes = quotaUsages.stream() + .sorted(Comparator.comparingInt(QuotaUsageJoinVO::getUsageType)) + .collect(Collectors.groupingBy(QuotaUsageJoinVO::getUsageType)); + + List items = new ArrayList<>(); + recordsPerUsageTypes.forEach((key, value) -> items.add(createStatementItem(key, value, cmd.isShowResources()))); QuotaStatementResponse statement = new QuotaStatementResponse(); - - HashMap quotaTariffMap = new HashMap(); - Collection result = QuotaTypes.listQuotaTypes().values(); - - for (QuotaTypes quotaTariff : result) { - quotaTariffMap.put(quotaTariff.getQuotaType(), quotaTariff); - // add dummy record for each usage type - QuotaUsageVO dummy = new QuotaUsageVO(quotaUsage.get(0)); - dummy.setUsageType(quotaTariff.getQuotaType()); - dummy.setQuotaUsed(new BigDecimal(0)); - quotaUsage.add(dummy); - } - - if (logger.isDebugEnabled()) { - logger.debug( - "createQuotaStatementResponse Type=" + quotaUsage.get(0).getUsageType() + " usage=" + quotaUsage.get(0).getQuotaUsed().setScale(2, RoundingMode.HALF_EVEN) - + " rec.id=" + quotaUsage.get(0).getUsageItemId() + " SD=" + quotaUsage.get(0).getStartDate() + " ED=" + quotaUsage.get(0).getEndDate()); - } - - Collections.sort(quotaUsage, new Comparator() { - @Override - public int compare(QuotaUsageVO o1, QuotaUsageVO o2) { - if (o1.getUsageType() == o2.getUsageType()) { - return 0; - } - return o1.getUsageType() < o2.getUsageType() ? -1 : 1; - } - }); - - List items = new ArrayList(); - QuotaStatementItemResponse lineitem; - int type = -1; - BigDecimal usage = new BigDecimal(0); - BigDecimal totalUsage = new BigDecimal(0); - quotaUsage.add(new QuotaUsageVO());// boundary - QuotaUsageVO prev = quotaUsage.get(0); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaStatementResponse record count=" + quotaUsage.size()); - } - for (final QuotaUsageVO quotaRecord : quotaUsage) { - if (type != quotaRecord.getUsageType()) { - if (type != -1) { - lineitem = new QuotaStatementItemResponse(type); - lineitem.setQuotaUsed(usage); - lineitem.setAccountId(prev.getAccountId()); - lineitem.setDomainId(prev.getDomainId()); - lineitem.setUsageUnit(quotaTariffMap.get(type).getQuotaUnit()); - lineitem.setUsageName(quotaTariffMap.get(type).getQuotaName()); - lineitem.setObjectName("quotausage"); - items.add(lineitem); - totalUsage = totalUsage.add(usage); - usage = new BigDecimal(0); - } - type = quotaRecord.getUsageType(); - } - prev = quotaRecord; - usage = usage.add(quotaRecord.getQuotaUsed()); - } - statement.setLineItem(items); - statement.setTotalQuota(totalUsage); + statement.setTotalQuota(items.stream().map(QuotaStatementItemResponse::getQuotaUsed).reduce(BigDecimal.ZERO, BigDecimal::add)); statement.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); statement.setObjectName("statement"); + + if (accountId != null) { + Account account = _accountDao.findByIdIncludingRemoved(accountId); + statement.setAccountId(account.getUuid()); + statement.setAccountName(account.getAccountName()); + domainId = account.getDomainId(); + } + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + statement.setDomainId(domain.getUuid()); + } + return statement; } + protected Long getAccountIdForQuotaStatement(QuotaStatementCmd cmd) { + if (Account.Type.NORMAL.equals(CallContext.current().getCallingAccount().getType())) { + logger.debug("Limiting the Quota statement for the calling Account, as they are a User Account."); + return CallContext.current().getCallingAccountId(); + } + + long accountId = cmd.getEntityOwnerId(); + if (accountId != -1) { + return accountId; + } + + if (cmd.getDomainId() == null) { + logger.debug("Limiting the Quota statement for the calling Account, as 'domainid' was not informed."); + return CallContext.current().getCallingAccountId(); + } + + logger.debug("Allowing admin/domain admin to generate the Quota statement for the provided Domain."); + return null; + } + + protected Long getDomainIdForQuotaStatement(QuotaStatementCmd cmd, Long accountId) { + if (accountId != null) { + logger.debug("Quota statement is already limited to Account [{}].", accountId); + Account account = _accountDao.findByIdIncludingRemoved(accountId); + return account.getDomainId(); + } + + Long domainId = cmd.getDomainId(); + if (domainId != null) { + return domainId; + } + + logger.debug("Limiting the Quota statement for the calling Account's Domain."); + return CallContext.current().getCallingAccount().getDomainId(); + } + + protected void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(List quotaUsages, Integer usageType) { + if (usageType != null) { + logger.debug("As the usage type [{}] was informed as parameter of the API quotaStatement, we will not create dummy records.", usageType); + return; + + } + + for (Integer quotaType : QuotaTypes.listQuotaTypes().keySet()) { + QuotaUsageJoinVO dummy = new QuotaUsageJoinVO(); + dummy.setUsageType(quotaType); + dummy.setQuotaUsed(BigDecimal.ZERO); + quotaUsages.add(dummy); + } + } + + protected QuotaStatementItemResponse createStatementItem(int usageType, List usageRecords, boolean showResources) { + QuotaUsageJoinVO firstRecord = usageRecords.get(0); + int type = firstRecord.getUsageType(); + + QuotaTypes quotaType = QuotaTypes.listQuotaTypes().get(type); + + QuotaStatementItemResponse item = new QuotaStatementItemResponse(type); + item.setQuotaUsed(usageRecords.stream().map(QuotaUsageJoinVO::getQuotaUsed).filter(Objects::nonNull).reduce(BigDecimal.ZERO, BigDecimal::add)); + item.setUsageUnit(quotaType.getQuotaUnit()); + item.setUsageName(quotaType.getQuotaName()); + + setStatementItemResources(item, usageType, usageRecords, showResources); + return item; + } + + protected void setStatementItemResources(QuotaStatementItemResponse statementItem, int usageType, List quotaUsageRecords, boolean showResources) { + if (!showResources) { + return; + } + + List itemDetails = new ArrayList<>(); + + Map quotaUsagesValuesAggregatedById = quotaUsageRecords + .stream() + .filter(quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType) != null) + .collect(Collectors.groupingBy( + quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType), + Collectors.reducing(new BigDecimal(0), QuotaUsageJoinVO::getQuotaUsed, BigDecimal::add) + )); + + for (Map.Entry entry : quotaUsagesValuesAggregatedById.entrySet()) { + QuotaStatementItemResourceResponse detail = new QuotaStatementItemResourceResponse(); + + detail.setQuotaUsed(entry.getValue()); + + QuotaUsageResourceVO resource = getResourceFromIdAndType(entry.getKey(), usageType); + if (resource != null) { + detail.setResourceId(resource.getUuid()); + detail.setDisplayName(resource.getName()); + detail.setRemoved(resource.isRemoved()); + } else { + detail.setDisplayName(""); + + } + itemDetails.add(detail); + } + statementItem.setResources(itemDetails); + } + + protected Long getResourceIdByUsageType(QuotaUsageJoinVO quotaUsageJoinVo, int usageType) { + switch (usageType) { + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + return quotaUsageJoinVo.getNetworkId(); + case QuotaTypes.NETWORK_OFFERING: + return quotaUsageJoinVo.getOfferingId(); + default: + return quotaUsageJoinVo.getResourceId(); + } + } + + protected QuotaUsageResourceVO getResourceFromIdAndType(long resourceId, int usageType) { + switch (usageType) { + case QuotaTypes.ALLOCATED_VM: + case QuotaTypes.RUNNING_VM: + VMInstanceVO vmInstance = vmInstanceDao.findByIdIncludingRemoved(resourceId); + if (vmInstance != null) { + return new QuotaUsageResourceVO(vmInstance.getUuid(), vmInstance.getHostName(), vmInstance.getRemoved()); + } + break; + case QuotaTypes.VOLUME: + case QuotaTypes.VOLUME_SECONDARY: + case QuotaTypes.VM_DISK_BYTES_READ: + case QuotaTypes.VM_DISK_BYTES_WRITE: + case QuotaTypes.VM_DISK_IO_READ: + case QuotaTypes.VM_DISK_IO_WRITE: + VolumeVO volume = volumeDao.findByIdIncludingRemoved(resourceId); + if (volume != null) { + return new QuotaUsageResourceVO(volume.getUuid(), volume.getName(), volume.getRemoved()); + } + break; + case QuotaTypes.VM_SNAPSHOT_ON_PRIMARY: + case QuotaTypes.VM_SNAPSHOT: + case QuotaTypes.SNAPSHOT: + SnapshotVO snapshot = snapshotDao.findByIdIncludingRemoved(resourceId); + if (snapshot != null) { + return new QuotaUsageResourceVO(snapshot.getUuid(), snapshot.getName(), snapshot.getRemoved()); + } + break; + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + NetworkVO network = networkDao.findByIdIncludingRemoved(resourceId); + if (network != null) { + return new QuotaUsageResourceVO(network.getUuid(), network.getName(), network.getRemoved()); + } + break; + case QuotaTypes.TEMPLATE: + case QuotaTypes.ISO: + VMTemplateVO vmTemplate = vmTemplateDao.findByIdIncludingRemoved(resourceId); + if (vmTemplate != null) { + return new QuotaUsageResourceVO(vmTemplate.getUuid(), vmTemplate.getName(), vmTemplate.getRemoved()); + } + break; + case QuotaTypes.NETWORK_OFFERING: + NetworkOfferingVO networkOffering = networkOfferingDao.findByIdIncludingRemoved(resourceId); + if (networkOffering != null) { + return new QuotaUsageResourceVO(networkOffering.getUuid(), networkOffering.getName(), networkOffering.getRemoved()); + } + break; + case QuotaTypes.IP_ADDRESS: + IPAddressVO ipAddress = ipAddressDao.findByIdIncludingRemoved(resourceId); + if (ipAddress != null) { + return new QuotaUsageResourceVO(ipAddress.getUuid(), ipAddress.getName(), ipAddress.getRemoved()); + } + break; + } + return null; + } + @Override public Pair, Integer> listQuotaTariffPlans(final QuotaTariffListCmd cmd) { Date startDate = cmd.getEffectiveDate(); @@ -518,14 +677,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } protected void warnQuotaTariffUpdateDeprecatedFields(QuotaTariffUpdateCmd cmd) { - String warnMessage = "The parameter 's%s' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; + String warnMessage = "The parameter '{}' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; if (cmd.getStartDate() != null) { - logger.warn(String.format(warnMessage,"startdate")); + logger.warn(warnMessage, "startdate"); } if (cmd.getUsageType() != null) { - logger.warn(String.format(warnMessage,"usagetype")); + logger.warn(warnMessage, "usagetype"); } } @@ -712,11 +871,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { return resp; } - @Override - public List getQuotaUsage(QuotaStatementCmd cmd) { - return _quotaService.getQuotaUsage(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); - } - @Override public List getQuotaBalance(QuotaBalanceCmd cmd) { return _quotaService.findQuotaBalanceVO(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getStartDate(), cmd.getEndDate()); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java new file mode 100644 index 00000000000..3e052f73339 --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java @@ -0,0 +1,61 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import java.math.BigDecimal; + +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; + +public class QuotaStatementItemResourceResponse extends BaseResponse { + + @SerializedName(ApiConstants.QUOTA_CONSUMED) + @Param(description = "Quota consumed.") + private BigDecimal quotaUsed; + + @SerializedName(ApiConstants.RESOURCE_ID) + @Param(description = "Resources's ID.") + private String resourceId; + + @SerializedName(ApiConstants.DISPLAY_NAME) + @Param(description = "Resource's display name.") + private String displayName; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "Indicates whether the resource is removed or active.") + private boolean removed; + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setRemoved(boolean removed) { + this.removed = removed; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java index c370d82b3cc..0747c5a9172 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java @@ -17,72 +17,41 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; +import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; public class QuotaStatementItemResponse extends BaseResponse { - @SerializedName("type") - @Param(description = "Usage type") + @SerializedName(ApiConstants.TYPE) + @Param(description = "Usage type.") private int usageType; - @SerializedName("accountid") - @Param(description = "Account id") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain id") - private Long domainId; - - @SerializedName("name") - @Param(description = "Usage type name") + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Usage type.") private String usageName; - @SerializedName("unit") - @Param(description = "Usage unit") + @SerializedName(ApiConstants.UNIT) + @Param(description = "Unit of the Usage type.") private String usageUnit; - @SerializedName("quota") - @Param(description = "Quota consumed") + @SerializedName(ApiConstants.QUOTA) + @Param(description = "Quota consumed.") private BigDecimal quotaUsed; + @SerializedName(ApiConstants.RESOURCES) + @Param(description = "Item's resources.") + private List resources; + public QuotaStatementItemResponse(final int usageType) { this.usageType = usageType; } - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } - - public void setAccountName(String accountName) { - this.accountName = accountName; - } - - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - public String getUsageName() { return usageName; } @@ -112,7 +81,15 @@ public class QuotaStatementItemResponse extends BaseResponse { } public void setQuotaUsed(BigDecimal quotaUsed) { - this.quotaUsed = quotaUsed.setScale(2, RoundingMode.HALF_EVEN); + this.quotaUsed = quotaUsed; + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + this.resources = resources; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java index 0a7ba4dbeb9..81cb1011182 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java @@ -18,56 +18,56 @@ package org.apache.cloudstack.api.response; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import java.util.List; public class QuotaStatementResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; + @SerializedName(ApiConstants.ACCOUNT_ID) + @Param(description = "ID of the Account.") + private String accountId; - @SerializedName("account") - @Param(description = "Account name") + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "Name of the Account.") private String accountName; - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "ID of the Domain.") + private String domainId; - @SerializedName("quotausage") - @Param(description = "List of quota usage under various types", responseObject = QuotaStatementItemResponse.class) + @SerializedName(ApiConstants.QUOTA_USAGE) + @Param(description = "List of Quota usage under various types.", responseObject = QuotaStatementItemResponse.class) private List lineItem; - @SerializedName("totalquota") - @Param(description = "Total quota used during this period") + @SerializedName(ApiConstants.TOTAL_QUOTA) + @Param(description = "Total Quota consumed during this period.") private BigDecimal totalQuota; - @SerializedName("startdate") - @Param(description = "Start date") + @SerializedName(ApiConstants.START_DATE) + @Param(description = "Start date of the Quota statement.") private Date startDate = null; - @SerializedName("enddate") - @Param(description = "End date") + @SerializedName(ApiConstants.END_DATE) + @Param(description = "End date of the Quota statement.") private Date endDate = null; - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Currency of the Quota statement.") private String currency; public QuotaStatementResponse() { super(); } - public Long getAccountId() { + public String getAccountId() { return accountId; } - public void setAccountId(Long accountId) { + public void setAccountId(String accountId) { this.accountId = accountId; } @@ -79,45 +79,36 @@ public class QuotaStatementResponse extends BaseResponse { this.accountName = accountName; } - public Long getDomainId() { + public String getDomainId() { return domainId; } - public void setDomainId(Long domainId) { + public void setDomainId(String domainId) { this.domainId = domainId; } - public List getLineItem() { - return lineItem; - } - public void setLineItem(List lineItem) { this.lineItem = lineItem; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - - public BigDecimal getTotalQuota() { - return totalQuota; + this.endDate = endDate; } public void setTotalQuota(BigDecimal totalQuota) { - this.totalQuota = totalQuota.setScale(2, RoundingMode.HALF_EVEN); + this.totalQuota = totalQuota; } public String getCurrency() { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index a421d0f066a..78acfc11682 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -21,14 +21,14 @@ import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import com.cloud.user.AccountVO; import com.cloud.utils.component.PluggableService; public interface QuotaService extends PluggableService { - List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); + List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index 2d7d623d1d9..a0ba2fbc751 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -53,10 +53,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; -import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.commons.lang3.ObjectUtils; import org.springframework.stereotype.Component; @@ -80,7 +80,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private QuotaAccountDao _quotaAcc; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageJoinDao quotaUsageJoinDao; @Inject private DomainDao _domainDao; @Inject @@ -213,27 +213,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi } @Override - public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { - // if accountId is not specified, use accountName and domainId - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } - } - + public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { if (startDate.after(endDate)) { throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); } @@ -241,7 +221,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi logger.debug("Getting quota records of type [{}] for account [{}] in domain [{}], between [{}] and [{}].", usageType, accountId, domainId, startDate, endDate); - return _quotaUsageDao.findQuotaUsage(accountId, domainId, usageType, startDate, endDate); + return quotaUsageJoinDao.findQuotaUsage(accountId, domainId, usageType, null, null, null, startDate, endDate, null); } @Override diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java index d6f9f747fa8..e67638db632 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java @@ -16,38 +16,29 @@ // under the License. package org.apache.cloudstack.api.command; -import junit.framework.TestCase; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - @RunWith(MockitoJUnitRunner.class) -public class QuotaStatementCmdTest extends TestCase { +public class QuotaStatementCmdTest { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder responseBuilderMock; @Test - public void testQuotaStatementCmd() throws NoSuchFieldException, IllegalAccessException { + public void executeTestVerifyCalls() { QuotaStatementCmd cmd = new QuotaStatementCmd(); cmd.setAccountName("admin"); + cmd.responseBuilder = responseBuilderMock; - Field rbField = QuotaStatementCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + Mockito.doReturn(new QuotaStatementResponse()).when(responseBuilderMock).createQuotaStatementResponse(Mockito.any()); - List quotaUsageVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaUsage(Mockito.eq(cmd))).thenReturn(quotaUsageVOList); - Mockito.when(responseBuilder.createQuotaStatementResponse(Mockito.eq(quotaUsageVOList))).thenReturn(new QuotaStatementResponse()); cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).getQuotaUsage(Mockito.eq(cmd)); + + Mockito.verify(responseBuilderMock).createQuotaStatementResponse(cmd); } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index ea88a106b84..81b4992082d 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; @@ -42,6 +44,7 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; @@ -67,6 +70,7 @@ import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.commons.lang3.time.DateUtils; @@ -914,4 +918,199 @@ public class QuotaResponseBuilderImplTest extends TestCase { Assert.assertTrue(formattedVariables.containsValue("accountname")); Assert.assertTrue(formattedVariables.containsValue("zonename")); } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeDifferentFromNullDoNothing() { + List listUsage = new ArrayList<>(); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, 1); + + Assert.assertTrue(listUsage.isEmpty()); + } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeIsNullAddDummyForAllQuotaTypes() { + List listUsage = new ArrayList<>(); + listUsage.add(new QuotaUsageJoinVO()); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, null); + + Assert.assertEquals(QuotaTypes.listQuotaTypes().size() + 1, listUsage.size()); + + QuotaTypes.listQuotaTypes().entrySet().forEach(entry -> { + Assert.assertTrue(listUsage.stream().anyMatch(usage -> usage.getUsageType() == entry.getKey() && usage.getQuotaUsed().equals(BigDecimal.ZERO))); + }); + } + + private List getQuotaUsagesForTest() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + List quotaUsages = new ArrayList<>(); + + QuotaUsageJoinVO quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(1l); + quotaUsage.setDomainId(2l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(10)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-01")); + quotaUsage.setEndDate(sdf.parse("2022-01-02")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(4l); + quotaUsage.setDomainId(5l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(null); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-03")); + quotaUsage.setEndDate(sdf.parse("2022-01-04")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(6l); + quotaUsage.setDomainId(7l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(5)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-05")); + quotaUsage.setEndDate(sdf.parse("2022-01-06")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + return quotaUsages; + } + + @Test + public void createStatementItemTestReturnItem() { + List quotaUsages = getQuotaUsagesForTest(); + Mockito.doNothing().when(quotaResponseBuilderSpy).setStatementItemResources(Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyBoolean()); + + QuotaStatementItemResponse result = quotaResponseBuilderSpy.createStatementItem(0, quotaUsages, false); + + QuotaUsageJoinVO expected = quotaUsages.get(0); + QuotaTypes quotaTypeExpected = QuotaTypes.listQuotaTypes().get(expected.getUsageType()); + Assert.assertEquals(BigDecimal.valueOf(15), result.getQuotaUsed()); + Assert.assertEquals(quotaTypeExpected.getQuotaUnit(), result.getUsageUnit()); + Assert.assertEquals(quotaTypeExpected.getQuotaName(), result.getUsageName()); + } + + @Test + public void setStatementItemResourcesTestDoNotShowResourcesDoNothing() { + QuotaStatementItemResponse item = new QuotaStatementItemResponse(1); + + quotaResponseBuilderSpy.setStatementItemResources(item, 0, getQuotaUsagesForTest(), false); + + Assert.assertNull(item.getResources()); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountForNormalUser() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsEntityOwnerIdWhenProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(42L).when(cmd).getEntityOwnerId(); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(42L), result); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountWhenCallerIsAdminAndDomainIsNotProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(null).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsAdminAndDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(10L).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertNull(result); + } + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsAccountDomainIdWhenAccountIdIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + AccountVO account = Mockito.mock(AccountVO.class); + + Mockito.doReturn(account).when(accountDaoMock).findByIdIncludingRemoved(55L); + Mockito.doReturn(77L).when(account).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, 55L); + + Assert.assertEquals(Long.valueOf(77L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsProvidedDomainIdWhenAccountIdIsNull() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(99L).when(cmd).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(Long.valueOf(99L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestFallsBackToCallingAccountDomainIdWhenNeitherAccountNorDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + Account account = Mockito.mock(Account.class); + + Mockito.doReturn(null).when(cmd).getDomainId(); + Mockito.doReturn(123L).when(account).getDomainId(); + Mockito.doReturn(account).when(callContextMock).getCallingAccount(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(123L, result.longValue()); + } + } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 259264f3b0e..a0fe63de851 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.joda.time.DateTime; @@ -63,6 +64,8 @@ public class QuotaServiceImplTest extends TestCase { @Mock QuotaBalanceDao quotaBalanceDao; @Mock + QuotaUsageJoinDao quotaUsageJoinDaoMock; + @Mock QuotaResponseBuilder respBldr; @Mock private AccountVO accountVoMock; @@ -85,9 +88,9 @@ public class QuotaServiceImplTest extends TestCase { quotaAccountDaoField.setAccessible(true); quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); - Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); + Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("quotaUsageJoinDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageJoinDaoMock); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); @@ -142,7 +145,8 @@ public class QuotaServiceImplTest extends TestCase { final Date endDate = new Date(); quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); - Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); + Mockito.verify(quotaUsageJoinDaoMock, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); } @Test diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 745fcfcda6c..c44620d557d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -18,6 +18,9 @@ package com.cloud.hypervisor.kvm.resource; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_VDDK_LIB_DIR; +import static com.cloud.host.Host.HOST_VDDK_SUPPORT; +import static com.cloud.host.Host.HOST_VDDK_VERSION; import static com.cloud.host.Host.HOST_VIRTV2V_VERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import static org.apache.cloudstack.utils.linux.KVMHostInfo.isHostS390x; @@ -365,6 +368,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "rpm -qa | grep -i virtio-win"; public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win"; public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit"; + public static final String VDDK_AUTODETECT_PATH_CMD = "find / -type d -name 'vmware-vix-disklib-distrib' 2>/dev/null | head -n 1"; public static final int LIBVIRT_CGROUP_CPU_SHARES_MIN = 2; public static final int LIBVIRT_CGROUP_CPU_SHARES_MAX = 262144; @@ -903,10 +907,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private boolean convertInstanceVerboseMode = false; private Map convertInstanceEnv = null; + private String vddkLibDir = null; + private static final String libguestfsBackend = "direct"; protected boolean dpdkSupport = false; protected String dpdkOvsPath; protected String directDownloadTemporaryDownloadPath; protected String cachePath; + private String vddkTransports = null; + private String vddkThumbprint = null; + private String vddkVersion = null; + private String detectedPasswordFileOption = null; protected String javaTempDir = System.getProperty("java.io.tmpdir"); private String getEndIpFromStartIp(final String startIp, final int numIps) { @@ -971,6 +981,26 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return convertInstanceEnv; } + public String getVddkLibDir() { + return vddkLibDir; + } + + public String getLibguestfsBackend() { + return libguestfsBackend; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public String getVddkVersion() { + return vddkVersion; + } + /** * Defines resource's public and private network interface according to what is configured in agent.properties. */ @@ -1180,6 +1210,37 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv setConvertInstanceEnv(convertEnvTmpDir, convertEnvVirtv2vTmpDir); + vddkLibDir = StringUtils.trimToNull(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_LIB_DIR)); + if (StringUtils.isNotBlank(vddkLibDir) && !isVddkLibDirValid(vddkLibDir)) { + LOGGER.warn("Configured VDDK library dir [{}] is invalid (missing lib64/libvixDiskLib.so), attempting auto-detection", vddkLibDir); + vddkLibDir = null; + } + if (StringUtils.isBlank(vddkLibDir)) { + vddkLibDir = detectVddkLibDir(); + } + if (StringUtils.isNotBlank(vddkLibDir)) { + LOGGER.info("Detected VDDK library dir: {}", vddkLibDir); + } else { + LOGGER.warn("Could not detect a valid VDDK library dir; VDDK conversion will be unavailable"); + } + + vddkVersion = detectVddkVersion(); + if (StringUtils.isNotBlank(vddkVersion)) { + LOGGER.info("Detected nbdkit VDDK plugin version: {}", vddkVersion); + } + + vddkTransports = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_TRANSPORTS)); + vddkThumbprint = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_THUMBPRINT)); + + detectedPasswordFileOption = detectPasswordFileOption(); + if (StringUtils.isNotBlank(detectedPasswordFileOption)) { + LOGGER.info("Detected virt-v2v password option: {}", detectedPasswordFileOption); + } else { + LOGGER.warn("Could not detect virt-v2v password option, VDDK conversions may fail"); + } + pool = (String)params.get("pool"); if (pool == null) { pool = "/root"; @@ -4251,6 +4312,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cmd.setHostTags(getHostTags()); boolean instanceConversionSupported = hostSupportsInstanceConversion(); cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); + cmd.getHostDetails().put(HOST_VDDK_SUPPORT, String.valueOf(hostSupportsVddk())); + if (StringUtils.isNotBlank(vddkLibDir)) { + cmd.getHostDetails().put(HOST_VDDK_LIB_DIR, vddkLibDir); + } + if (StringUtils.isNotBlank(vddkVersion)) { + cmd.getHostDetails().put(HOST_VDDK_VERSION, vddkVersion); + } if (instanceConversionSupported) { cmd.getHostDetails().put(HOST_VIRTV2V_VERSION, getHostVirtV2vVersion()); } @@ -5990,6 +6058,66 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + public boolean hostSupportsVddk() { + return hostSupportsVddk(null); + } + + public boolean hostSupportsVddk(String overriddenVddkLibDir) { + String effectiveVddkLibDir = StringUtils.trimToNull(overriddenVddkLibDir); + if (StringUtils.isBlank(effectiveVddkLibDir)) { + effectiveVddkLibDir = StringUtils.trimToNull(vddkLibDir); + } + if (StringUtils.isBlank(effectiveVddkLibDir) || !isVddkLibDirValid(effectiveVddkLibDir)) { + effectiveVddkLibDir = detectVddkLibDir(); + } + return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(detectVddkVersion()); + } + + protected boolean isVddkLibDirValid(String path) { + if (StringUtils.isBlank(path)) { + return false; + } + File libDir = new File(path, "lib64"); + if (!libDir.isDirectory()) { + return false; + } + File[] libs = libDir.listFiles((dir, name) -> name.startsWith("libvixDiskLib.so")); + return libs != null && libs.length > 0; + } + + protected String detectVddkLibDir() { + String detectedPath = StringUtils.trimToNull(Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD)); + if (StringUtils.isNotBlank(detectedPath) && isVddkLibDirValid(detectedPath)) { + return detectedPath; + } + return null; + } + + protected String detectVddkVersion() { + try { + ProcessBuilder pb = new ProcessBuilder("nbdkit", "vddk", "--version"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (StringUtils.isBlank(output)) { + return null; + } + + for (String line : output.split("\\R")) { + String trimmed = StringUtils.trimToEmpty(line); + if (trimmed.startsWith("vddk ")) { + return StringUtils.trimToNull(trimmed.substring("vddk ".length())); + } + } + return null; + } catch (Exception e) { + LOGGER.error("Failed to detect vddk version: {}", e.getMessage()); + return null; + } + } + public boolean hostSupportsWindowsGuestConversion() { if (isUbuntuOrDebianHost()) { int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); @@ -6004,6 +6132,40 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + /** + * Detect which password option virt-v2v supports by examining its --help output + * @return "-ip" if supported (virt-v2v >= 2.8.1), "--password-file" if older version, or null if detection fails + */ + protected String detectPasswordFileOption() { + try { + ProcessBuilder pb = new ProcessBuilder("virt-v2v", "--help"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (output.contains("-ip ")) { + return "-ip"; + } else if (output.contains("--password-file")) { + return "--password-file"; + } else { + LOGGER.error("virt-v2v does not support -ip or --password-file"); + return null; + } + } catch (Exception e) { + LOGGER.error("Failed to detect virt-v2v password option: {}", e.getMessage()); + return null; + } + } + + /** + * Get the detected password file option for virt-v2v + * @return the password option ("-ip" or "--password-file") or null if not detected + */ + public String getDetectedPasswordFileOption() { + return detectedPasswordFileOption; + } + public String getHostVirtV2vVersion() { if (!hostSupportsInstanceConversion()) { return ""; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java index b94b4830003..de9341715f0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java @@ -30,7 +30,15 @@ public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper supportedInstanceConvertSourceHypervisors = List.of(Hypervisor.HypervisorType.VMware); + private static final Pattern SHA1_FINGERPRINT_PATTERN = Pattern.compile("(?i)(?:SHA1\\s+)?Fingerprint\\s*=\\s*([0-9A-F:]+)"); @Override public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { @@ -61,7 +69,8 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { - noOfThreads = 0; - } - ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); - temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); - if (!ovfExported) { - String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); - logger.error(String.format("(%s) %s", originalVMName, err)); - return new Answer(cmd, false, err); - } - sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); - } else { - ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - } - - logger.info(String.format("(%s) Attempting to convert the OVF %s of the instance %s from %s to KVM", - originalVMName, ovfTemplateDirOnConversionLocation, sourceInstanceName, sourceHypervisorType)); - final String temporaryConvertUuid = UUID.randomUUID().toString(); boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); boolean cleanupSecondaryStorage = false; + boolean ovfExported = false; + String ovfTemplateDirOnConversionLocation = null; + try { - boolean result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, - timeout, verboseModeEnabled, extraParams, serverResource); + boolean result; + if (useVddk) { + logger.info("({}) Using VDDK-based conversion (direct from VMware)", originalVMName); + String vddkLibDir = resolveVddkSetting(cmd.getVddkLibDir(), serverResource.getVddkLibDir()); + if (StringUtils.isBlank(vddkLibDir)) { + String err = String.format("VDDK lib dir is not configured on the host. " + + "Set '%s' in agent.properties or in details parameter of the import api call to use VDDK-based conversion.", "vddk.lib.dir"); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + String vddkTransports = resolveVddkSetting(cmd.getVddkTransports(), serverResource.getVddkTransports()); + String configuredVddkThumbprint = resolveVddkSetting(cmd.getVddkThumbprint(), serverResource.getVddkThumbprint()); + String passwordOption = serverResource.getDetectedPasswordFileOption(); + result = performInstanceConversionUsingVddk(sourceInstance, originalVMName, temporaryConvertPath, + vddkLibDir, serverResource.getLibguestfsBackend(), vddkTransports, configuredVddkThumbprint, + timeout, verboseModeEnabled, extraParams, temporaryConvertUuid, passwordOption); + } else { + logger.info("({}) Using OVF-based conversion (export + local convert)", originalVMName); + String sourceOVFDirPath; + if (cmd.getExportOvfToConversionLocation()) { + String exportInstanceOVAUrl = getExportInstanceOVAUrl(sourceInstance, originalVMName); + + if (StringUtils.isBlank(exportInstanceOVAUrl)) { + String err = String.format("Couldn't export OVA for the VM %s, due to empty url", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + + int noOfThreads = cmd.getThreadsCountToExportOvf(); + if (noOfThreads > 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { + noOfThreads = 0; + } + ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); + temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); + + if (!ovfExported) { + String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); + } else { + ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + } + + result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, + timeout, verboseModeEnabled, extraParams, serverResource); + } + if (!result) { - String err = String.format( - "The virt-v2v conversion for the OVF %s failed. Please check the agent logs " + - "for the virt-v2v output. Please try on a different kvm host which " + - "has a different virt-v2v version.", - ovfTemplateDirOnConversionLocation); - logger.error(String.format("(%s) %s", originalVMName, err)); + String err = String.format("Instance conversion failed for VM %s. Please check virt-v2v logs.", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); return new Answer(cmd, false, err); } return new ConvertInstanceAnswer(cmd, temporaryConvertUuid); } catch (Exception e) { - String error = String.format("Error converting instance %s from %s, due to: %s", - sourceInstanceName, sourceHypervisorType, e.getMessage()); - logger.error(String.format("(%s) %s", originalVMName, error), e); + String error = String.format("Error converting instance %s from %s, due to: %s", sourceInstanceName, sourceHypervisorType, e.getMessage()); + logger.error("({}) {}", originalVMName, error, e); cleanupSecondaryStorage = true; return new Answer(cmd, false, error); } finally { @@ -275,4 +298,198 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper/dev/null | " + + "openssl x509 -fingerprint -sha1 -noout", endpoint); + + Script script = new Script("/bin/bash", timeout, logger); + script.add("-c"); + script.add(command); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + script.execute(parser); + + String output = parser.getLines(); + if (script.getExitValue() != 0) { + logger.error("({}) Failed to fetch vCenter thumbprint for {}", originalVMName, vcenterHost); + return null; + } + + String thumbprint = extractSha1Fingerprint(output); + if (StringUtils.isBlank(thumbprint)) { + logger.error("({}) Failed to parse vCenter thumbprint from output for {}", originalVMName, vcenterHost); + return null; + } + return thumbprint; + } + + private String extractSha1Fingerprint(String output) { + String parsedOutput = StringUtils.trimToEmpty(output); + if (StringUtils.isBlank(parsedOutput)) { + return null; + } + + for (String line : parsedOutput.split("\\R")) { + String trimmedLine = StringUtils.trimToEmpty(line); + if (StringUtils.isBlank(trimmedLine)) { + continue; + } + + Matcher matcher = SHA1_FINGERPRINT_PATTERN.matcher(trimmedLine); + if (matcher.find()) { + return matcher.group(1).toUpperCase(Locale.ROOT); + } + + // Fallback for raw fingerprint-only output. + if (trimmedLine.matches("(?i)[0-9a-f]{2}(:[0-9a-f]{2})+")) { + return trimmedLine.toUpperCase(Locale.ROOT); + } + } + return null; + } + + /** + * Build vpx:// URL for virt-v2v + * + * Format: + * vpx://user@vcenter/DC/cluster/host?no_verify=1 + */ + private String buildVpxUrl(RemoteInstanceTO vmwareInstance) { + + String vmName = vmwareInstance.getInstanceName(); + String vcenter = vmwareInstance.getVcenterHost(); + String username = vmwareInstance.getVcenterUsername(); + String datacenter = vmwareInstance.getDatacenterName(); + String cluster = vmwareInstance.getClusterName(); + String host = vmwareInstance.getHostName(); + + String encodedUsername = encodeUsername(username); + + StringBuilder url = new StringBuilder(); + url.append("vpx://") + .append(encodedUsername) + .append("@") + .append(vcenter) + .append("/") + .append(datacenter); + + if (StringUtils.isNotBlank(cluster)) { + url.append("/").append(cluster); + } + + if (StringUtils.isNotBlank(host)) { + url.append("/").append(host); + } + + url.append("?no_verify=1"); + + logger.info("({}) Using VPX URL: {}", vmName, url); + return url.toString(); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 43607edc53a..f54918bbc22 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -259,6 +259,12 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper 0 && sleeptime > migrateWait * 1000) { DomainState state = null; try { @@ -306,8 +310,6 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper 0 && sleeptime > migratePauseAfter) { DomainState state = null; try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index e74923b281f..5a7d6d2c203 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -34,6 +34,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; @ResourceWrapper(handles = ReadyCommand.class) public final class LibvirtReadyCommandWrapper extends CommandWrapper { @@ -50,6 +51,9 @@ public final class LibvirtReadyCommandWrapper extends CommandWrapper filesMock = Mockito.mockStatic(Files.class); + MockedConstruction