diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 777bd4bf5cd..013c2e5bf2a 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -460,3 +460,16 @@ iscsi.session.cleanup.enabled=false # Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. # incremental.snapshot.retry.rebase.wait=60 + +# Path to the VDDK library directory for VMware to KVM conversion via VDDK, +# passed to virt-v2v as -io vddk-libdir= +#vddk.lib.dir= + +# Ordered VDDK transport preference for VMware to KVM conversion via VDDK, passed as +# -io vddk-transports= to virt-v2v. Example: nbd:nbdssl +#vddk.transports= + +# Optional vCenter SHA1 thumbprint for VMware to KVM conversion via VDDK, passed as +# -io vddk-thumbprint=. If unset, CloudStack computes it on the KVM host via openssl. +#vddk.thumbprint= + diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index b0926b23a1e..e69a7efdc9c 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -808,6 +808,30 @@ public class AgentProperties{ */ public static final Property CONVERT_ENV_VIRTV2V_TMPDIR = new Property<>("convert.instance.env.virtv2v.tmpdir", null, String.class); + /** + * Path to the VDDK library directory on the KVM conversion host, used when converting VMs from VMware to KVM via VDDK. + * This directory is passed to virt-v2v as -io vddk-libdir=<path>. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_LIB_DIR = new Property<>("vddk.lib.dir", null, String.class); + + /** + * Ordered list of VDDK transports for virt-v2v, passed as -io vddk-transports=<value>. + * Example: nbd:nbdssl. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_TRANSPORTS = new Property<>("vddk.transports", null, String.class); + + /** + * vCenter TLS certificate thumbprint used by virt-v2v VDDK mode, passed as -io vddk-thumbprint=<value>. + * If unset, the KVM host computes it at runtime from the vCenter endpoint. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_THUMBPRINT = new Property<>("vddk.thumbprint", null, String.class); + /** * BGP controll CIDR * Data type: String.
diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java index 18737c584b3..7daeb964917 100644 --- a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -36,13 +36,17 @@ public class RemoteInstanceTO implements Serializable { private String vcenterPassword; private String vcenterHost; private String datacenterName; + private String clusterName; + private String hostName; public RemoteInstanceTO() { } - public RemoteInstanceTO(String instanceName) { + public RemoteInstanceTO(String instanceName, String clusterName, String hostName) { this.hypervisorType = Hypervisor.HypervisorType.VMware; this.instanceName = instanceName; + this.clusterName = clusterName; + this.hostName = hostName; } public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) { @@ -55,6 +59,12 @@ public class RemoteInstanceTO implements Serializable { this.datacenterName = datacenterName; } + public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName, String clusterName, String hostName) { + this(instanceName, instancePath, vcenterHost, vcenterUsername, vcenterPassword, datacenterName); + this.clusterName = clusterName; + this.hostName = hostName; + } + public Hypervisor.HypervisorType getHypervisorType() { return this.hypervisorType; } @@ -82,4 +92,12 @@ public class RemoteInstanceTO implements Serializable { public String getDatacenterName() { return datacenterName; } + + public String getClusterName() { + return clusterName; + } + + public String getHostName() { + return hostName; + } } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 9c011bac319..b5234820151 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -57,6 +57,9 @@ public interface Host extends StateObject, Identity, Partition, HAResour String HOST_UEFI_ENABLE = "host.uefi.enable"; String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + String HOST_VDDK_SUPPORT = "host.vddk.support"; + String HOST_VDDK_LIB_DIR = "vddk.lib.dir"; + String HOST_VDDK_VERSION = "host.vddk.version"; String HOST_OVFTOOL_VERSION = "host.ovftool.version"; String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; String HOST_SSH_PORT = "host.ssh.port"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 9ec85734233..7eae16a2a37 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -526,7 +526,6 @@ public class ApiConstants { public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; - public static final String USER_SECRET_KEY = "usersecretkey"; public static final String SEARCH_BASE = "searchbase"; public static final String SECONDARY_IP = "secondaryip"; public static final String SECURITY_GROUP_IDS = "securitygroupids"; @@ -630,6 +629,8 @@ public class ApiConstants { public static final String USERNAME = "username"; public static final String USER_CONFIGURABLE = "userconfigurable"; public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; + public static final String USER_SECRET_KEY = "usersecretkey"; + public static final String USE_VDDK = "usevddk"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver"; public static final String UPDATE_IN_SEQUENCE = "updateinsequence"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 50ccfbd69c5..db7dcc3fb44 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -179,6 +179,14 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "(only for importing VMs from VMware to KVM) optional - the ID of the guest OS for the imported VM.") private Long guestOsId; + @Parameter(name = ApiConstants.USE_VDDK, + type = CommandType.BOOLEAN, + since = "4.22.1", + description = "(only for importing VMs from VMware to KVM) optional - if true, uses VDDK on the KVM conversion host for converting the VM. " + + "This parameter is mutually exclusive with " + ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES + ".") + private Boolean useVddk; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -255,6 +263,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return storagePoolId; } + public boolean getUseVddk() { + return BooleanUtils.toBooleanDefaultIfNull(useVddk, true); + } + public String getTmpPath() { return tmpPath; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java index 4a9a0569c3b..abbb1760f9d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.bucket; import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.user.Account; @@ -82,7 +83,7 @@ public class DeleteBucketCmd extends BaseCmd { } @Override - public void execute() throws ConcurrentOperationException { + public void execute() throws ConcurrentOperationException, ResourceAllocationException { CallContext.current().setEventDetails("Bucket ID: " + getResourceUuid(ApiConstants.ID)); boolean result = _bucketService.deleteBucket(id, CallContext.current().getCallingAccount()); SuccessResponse response = new SuccessResponse(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 6c0121a3e4d..0da11bbd651 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -235,7 +235,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer * @param forced Indicates if backup will be force removed or not * @return returns operation success */ - boolean deleteBackup(final Long backupId, final Boolean forced); + boolean deleteBackup(final Long backupId, final Boolean forced) throws ResourceAllocationException; void validateBackupForZone(Long zoneId); diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java index e27ef308d7f..8c164133db8 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -95,7 +95,7 @@ public interface BucketApiService { */ Bucket createBucket(CreateBucketCmd cmd); - boolean deleteBucket(long bucketId, Account caller); + boolean deleteBucket(long bucketId, Account caller) throws ResourceAllocationException; boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException; diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java index fc066e5c589..c46fb697a3c 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java @@ -18,6 +18,8 @@ package com.cloud.agent.api; public class CheckConvertInstanceCommand extends Command { boolean checkWindowsGuestConversionSupport = false; + boolean useVddk = false; + String vddkLibDir; public CheckConvertInstanceCommand() { } @@ -26,6 +28,11 @@ public class CheckConvertInstanceCommand extends Command { this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; } + public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport, boolean useVddk) { + this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; + this.useVddk = useVddk; + } + @Override public boolean executeInSequence() { return false; @@ -34,4 +41,20 @@ public class CheckConvertInstanceCommand extends Command { public boolean getCheckWindowsGuestConversionSupport() { return checkWindowsGuestConversionSupport; } + + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index 24336747ccf..38e0dca7736 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -31,6 +31,10 @@ public class ConvertInstanceCommand extends Command { private boolean exportOvfToConversionLocation; private int threadsCountToExportOvf = 0; private String extraParams; + private boolean useVddk; + private String vddkLibDir; + private String vddkTransports; + private String vddkThumbprint; public ConvertInstanceCommand() { } @@ -90,6 +94,38 @@ public class ConvertInstanceCommand extends Command { this.extraParams = extraParams; } + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public void setVddkTransports(String vddkTransports) { + this.vddkTransports = vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public void setVddkThumbprint(String vddkThumbprint) { + this.vddkThumbprint = vddkThumbprint; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java index ed337885bee..21c4e7b97d0 100644 --- a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java +++ b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java @@ -24,6 +24,8 @@ import com.cloud.resource.ResourceState; public class PropagateResourceEventCommand extends Command { long hostId; ResourceState.Event event; + boolean forced; + boolean forceDeleteStorage; protected PropagateResourceEventCommand() { @@ -34,6 +36,13 @@ public class PropagateResourceEventCommand extends Command { this.event = event; } + public PropagateResourceEventCommand(long hostId, ResourceState.Event event, boolean forced, boolean forceDeleteStorage) { + this.hostId = hostId; + this.event = event; + this.forced = forced; + this.forceDeleteStorage = forceDeleteStorage; + } + public long getHostId() { return hostId; } @@ -42,6 +51,14 @@ public class PropagateResourceEventCommand extends Command { return event; } + public boolean isForced() { + return forced; + } + + public boolean isForceDeleteStorage() { + return forceDeleteStorage; + } + @Override public boolean executeInSequence() { // TODO Auto-generated method stub diff --git a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java index d8cc74817d7..96d73e11990 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java @@ -36,6 +36,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { public String lbStatsAuth = "admin1:AdMiN123"; public String lbStatsUri = "/admin?stats"; public String maxconn = ""; + public Long idleTimeout = 50000L; /* 0=infinite, >0 = timeout in milliseconds */ public String lbProtocol; public boolean keepAliveEnabled = false; NicTO nic; @@ -50,7 +51,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { } public LoadBalancerConfigCommand(LoadBalancerTO[] loadBalancers, String publicIp, String guestIp, String privateIp, NicTO nic, Long vpcId, String maxconn, - boolean keepAliveEnabled) { + boolean keepAliveEnabled, Long idleTimeout) { this.loadBalancers = loadBalancers; this.lbStatsPublicIP = publicIp; this.lbStatsPrivateIP = privateIp; @@ -59,6 +60,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { this.vpcId = vpcId; this.maxconn = maxconn; this.keepAliveEnabled = keepAliveEnabled; + this.idleTimeout = idleTimeout; } public NicTO getNic() { diff --git a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java index 128652fc64f..7d544c2e49c 100644 --- a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java +++ b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java @@ -635,6 +635,19 @@ public class HAProxyConfigurator implements LoadBalancerConfigurator { if (lbCmd.keepAliveEnabled) { dSection.set(7, "\tno option httpclose"); } + if (lbCmd.idleTimeout > 0) { + dSection.set(9, "\ttimeout client " + Long.toString(lbCmd.idleTimeout)); + dSection.set(10, "\ttimeout server " + Long.toString(lbCmd.idleTimeout)); + } else if (lbCmd.idleTimeout == 0) { + // .remove() is not allowed, only .set() operations are allowed as the list + // is a fixed size. So lets just mark the entry as blank. + dSection.set(9, ""); + dSection.set(10, ""); + } else { + // Negative idleTimeout values are considered invalid; retain the + // default HAProxy timeout values from defaultsSection for predictability. + logger.warn("Negative idleTimeout ({}) configured; retaining default HAProxy timeouts.", lbCmd.idleTimeout); + } if (logger.isDebugEnabled()) { for (final String s : dSection) { diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java index 6d4c6234c42..4ddadab999a 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java @@ -235,7 +235,7 @@ public class ConfigHelperTest { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 0L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java index 4196587cc3f..ed819bb7c68 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java @@ -779,7 +779,7 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer { final LoadBalancerTO[] arrayLbs = new LoadBalancerTO[lbs.size()]; lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; @@ -795,7 +795,7 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); nic.setIp("10.1.10.2"); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; diff --git a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java index 72361c2880e..073f976719b 100644 --- a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java +++ b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java @@ -79,13 +79,14 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("keepalive disabled should result in 'option httpclose' in the resulting haproxy config", result.contains("\toption httpclose")); - cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true); + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); result = genConfig(hpg, cmd); assertTrue("keepalive enabled should result in 'no option httpclose' in the resulting haproxy config", result.contains("\tno option httpclose")); + // TODO // create lb command // setup tests for @@ -93,6 +94,27 @@ public class HAProxyConfiguratorTest { // httpmode } + /** + * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. + */ + @Test + public void testGenerateConfigurationLoadBalancerIdleTimeoutConfigCommand() { + LoadBalancerTO lb = new LoadBalancerTO("1", "10.2.0.1", 80, "http", "bla", false, false, false, null); + LoadBalancerTO[] lba = new LoadBalancerTO[1]; + lba[0] = lb; + HAProxyConfigurator hpg = new HAProxyConfigurator(); + + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); + String result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 0 should not generate 'timeout server' in the resulting haproxy config", !result.contains("\ttimeout server")); + assertTrue("idleTimeout of 0 should not generate 'timeout client' in the resulting haproxy config", !result.contains("\ttimeout client")); + + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 1234L); + result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 1234 should result in 'timeout server 1234' in the resulting haproxy config", result.contains("\ttimeout server 1234")); + assertTrue("idleTimeout of 1234 should result in 'timeout client 1234' in the resulting haproxy config", result.contains("\ttimeout client 1234")); + } + /** * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. */ @@ -106,7 +128,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("'send-proxy' should result if protocol is 'tcp-proxy'", result.contains("send-proxy")); } @@ -118,7 +140,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("acl network_allowed src 1.1.1.1 2.2.2.2/24 \n\ttcp-request connection reject if !network_allowed")); } @@ -131,7 +153,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("bind 10.2.0.1:443 ssl crt /etc/cloudstack/ssl/10_2_0_1-443.pem")); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 615320e1788..b7b548fb940 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -122,6 +122,14 @@ public interface NetworkOrchestrationService { "Load Balancer(haproxy) maximum number of concurrent connections(global max)", true, Scope.Global); + ConfigKey NETWORK_LB_HAPROXY_IDLE_TIMEOUT = new ConfigKey<>( + "Network", + Long.class, + "network.loadbalancer.haproxy.idle.timeout", + "50000", + "Load Balancer(haproxy) idle timeout in milliseconds. Use 0 for infinite.", + true, + Scope.Global); List setupNetwork(Account owner, NetworkOffering offering, DeploymentPlan plan, String name, String displayText, boolean isDefault) throws ConcurrentOperationException; diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index e724f5d081b..4767e86e8ab 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -122,6 +122,8 @@ public interface ResourceManager extends ResourceService, Configurable { public boolean executeUserRequest(long hostId, ResourceState.Event event) throws AgentUnavailableException; + boolean executeUserRequest(long hostId, ResourceState.Event event, boolean isForced, boolean isForceDeleteStorage) throws AgentUnavailableException; + boolean resourceStateTransitTo(Host host, Event event, long msId) throws NoTransitionException; boolean umanageHost(long hostId); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 527c5259376..1215829d92f 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -805,8 +805,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); + String vddkSupport = detailsMap.get(Host.HOST_VDDK_SUPPORT); + String vddkLibDir = detailsMap.get(Host.HOST_VDDK_LIB_DIR); + String vddkVersion = detailsMap.get(Host.HOST_VDDK_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion)) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) { _hostDao.loadDetails(host); boolean updateNeeded = false; if (StringUtils.isNotBlank(uefiEnabled) && !uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { @@ -821,6 +824,26 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl host.getDetails().put(Host.HOST_OVFTOOL_VERSION, ovftoolVersion); updateNeeded = true; } + if (StringUtils.isNotBlank(vddkSupport) && !vddkSupport.equals(host.getDetails().get(Host.HOST_VDDK_SUPPORT))) { + host.getDetails().put(Host.HOST_VDDK_SUPPORT, vddkSupport); + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkLibDir).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_LIB_DIR)))) { + if (StringUtils.isBlank(vddkLibDir)) { + host.getDetails().remove(Host.HOST_VDDK_LIB_DIR); + } else { + host.getDetails().put(Host.HOST_VDDK_LIB_DIR, vddkLibDir); + } + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkVersion).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_VERSION)))) { + if (StringUtils.isBlank(vddkVersion)) { + host.getDetails().remove(Host.HOST_VDDK_VERSION); + } else { + host.getDetails().put(Host.HOST_VDDK_VERSION, vddkVersion); + } + updateNeeded = true; + } if (updateNeeded) { _hostDao.saveDetails(host); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index cfa0949883f..38a198b7304 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -1306,11 +1306,20 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust boolean result; try { - result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent()); + result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent(), cmd.isForced(), cmd.isForceDeleteStorage()); logger.debug("Result is {}", result); } catch (final AgentUnavailableException ex) { logger.warn("Agent is unavailable", ex); return null; + } catch (final RuntimeException ex) { + logger.error(String.format("Failed to execute propagated event %s for host %d", cmd.getEvent().name(), cmd.getHostId()), ex); + final Answer[] answers = new Answer[1]; + String details = ex.getMessage(); + if (details == null || details.isEmpty()) { + details = ex.toString(); + } + answers[0] = new Answer(cmd, false, details); + return _gson.toJson(answers); } final Answer[] answers = new Answer[1]; diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 2ae228adba9..7d455e7d6dc 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -2776,219 +2776,218 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra boolean ipv6 = false; try (CheckedReservation networkReservation = new CheckedReservation(owner, domainId, Resource.ResourceType.network, null, null, 1L, reservationDao, _resourceLimitMgr)) { - - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - ipv6 = true; - } - // Validate zone - if (zone.getNetworkType() == NetworkType.Basic) { - // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true - if (aclType == null || aclType != ACLType.Domain) { - throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + ipv6 = true; } + // Validate zone + if (zone.getNetworkType() == NetworkType.Basic) { + // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true + if (aclType == null || aclType != ACLType.Domain) { + throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); + } - // Only one guest network is supported in Basic zone - final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); - if (!guestNetworks.isEmpty()) { - throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); - } + // Only one guest network is supported in Basic zone + final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); + if (!guestNetworks.isEmpty()) { + throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); + } - // if zone is basic, only Shared network offerings w/o source nat service are allowed - if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { - throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " - + Service.SourceNat.getName() + " service are allowed"); - } + // if zone is basic, only Shared network offerings w/o source nat service are allowed + if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { + throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " + + Service.SourceNat.getName() + " service are allowed"); + } - if (domainId == null || domainId != Domain.ROOT_DOMAIN) { - throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); - } + if (domainId == null || domainId != Domain.ROOT_DOMAIN) { + throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); + } - if (subdomainAccess == null) { - subdomainAccess = true; - } else if (!subdomainAccess) { - throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); - } + if (subdomainAccess == null) { + subdomainAccess = true; + } else if (!subdomainAccess) { + throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); + } - if (vlanId == null) { - vlanId = Vlan.UNTAGGED; - } else { - if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { - throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + if (vlanId == null) { + vlanId = Vlan.UNTAGGED; + } else { + if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { + throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + } + } + + } else if (zone.getNetworkType() == NetworkType.Advanced) { + if (zone.isSecurityGroupEnabled()) { + if (isolatedPvlan != null) { + throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); + } + // Only Account specific Isolated network with sourceNat service disabled are allowed in security group + // enabled zone + if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { + throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); + } + if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { + throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + } + } + + //don't allow eip/elb networks in Advance zone + if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { + throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); } } - } else if (zone.getNetworkType() == NetworkType.Advanced) { - if (zone.isSecurityGroupEnabled()) { - if (isolatedPvlan != null) { - throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); - } - // Only Account specific Isolated network with sourceNat service disabled are allowed in security group - // enabled zone - if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { - throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); - } - if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { - throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { + _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); + } + + //TODO(VXLAN): Support VNI specified + // VlanId can be specified only when network offering supports it + final boolean vlanSpecified = vlanId != null; + if (vlanSpecified != ntwkOff.isSpecifyVlan()) { + if (vlanSpecified) { + if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + } + } else { + throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); } } - //don't allow eip/elb networks in Advance zone - if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { - throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); - } - } - - if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { - _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); - } - - //TODO(VXLAN): Support VNI specified - // VlanId can be specified only when network offering supports it - final boolean vlanSpecified = vlanId != null; - if (vlanSpecified != ntwkOff.isSpecifyVlan()) { if (vlanSpecified) { - if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); + // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks + URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; + if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + bypassVlanOverlapCheck = true; } - } else { - throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); - } - } - - if (vlanSpecified) { - URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); - // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks - URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; - if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - bypassVlanOverlapCheck = true; - } - //don't allow to specify vlan tag used by physical network for dynamic vlan allocation - if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) - && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " - + zone.getName()); - } - if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && - _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { - throw new InvalidParameterValueException(String.format( - "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", - isolatedPvlan, zone)); - } - if (!UuidUtils.isUuid(vlanId)) { - // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone - if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { - if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - vlanId, zone)); - } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - isolatedPvlan, zone)); - } else { - final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); - //for the network that is created as part of private gateway, - //the vnet is not coming from the data center vnet table, so the list can be empty - if (!dcVnets.isEmpty()) { - final DataCenterVnetVO dcVnet = dcVnets.get(0); - // Fail network creation if specified vlan is dedicated to a different account - if (dcVnet.getAccountGuestVlanMapId() != null) { - final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); - final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); - if (map.getAccountId() != owner.getAccountId()) { - throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); - } - // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool - } else { - final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); - if (maps != null && !maps.isEmpty()) { - final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); - final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); - if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { - throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " - + owner.getAccountName()); + //don't allow to specify vlan tag used by physical network for dynamic vlan allocation + if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) + && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { + throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + + zone.getName()); + } + if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && + _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { + throw new InvalidParameterValueException(String.format( + "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", + isolatedPvlan, zone)); + } + if (!UuidUtils.isUuid(vlanId)) { + // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone + if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { + if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + vlanId, zone)); + } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + isolatedPvlan, zone)); + } else { + final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); + //for the network that is created as part of private gateway, + //the vnet is not coming from the data center vnet table, so the list can be empty + if (!dcVnets.isEmpty()) { + final DataCenterVnetVO dcVnet = dcVnets.get(0); + // Fail network creation if specified vlan is dedicated to a different account + if (dcVnet.getAccountGuestVlanMapId() != null) { + final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); + final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); + if (map.getAccountId() != owner.getAccountId()) { + throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); + } + // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool + } else { + final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); + if (maps != null && !maps.isEmpty()) { + final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); + final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); + if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { + throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " + + owner.getAccountName()); + } } } } } + } else { + // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or + // shared network with same Vlan ID in the zone + if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { + throw new InvalidParameterValueException(String.format( + "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + } } - } else { - // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or - // shared network with same Vlan ID in the zone - if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { + } + + } + + // If networkDomain is not specified, take it from the global configuration + if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { + final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), + Service.Dns); + final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); + if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { + if (networkDomain != null) { + // TBD: NetworkOfferingId and zoneId. Send uuids instead. throw new InvalidParameterValueException(String.format( - "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + "Domain name change is not supported by network offering id=%d in zone %s", + networkOfferingId, zone)); } - } - } - - } - - // If networkDomain is not specified, take it from the global configuration - if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { - final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), - Service.Dns); - final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); - if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { - if (networkDomain != null) { - // TBD: NetworkOfferingId and zoneId. Send uuids instead. - throw new InvalidParameterValueException(String.format( - "Domain name change is not supported by network offering id=%d in zone %s", - networkOfferingId, zone)); - } - } else { - if (networkDomain == null) { - // 1) Get networkDomain from the corresponding account/domain/zone - if (aclType == ACLType.Domain) { - networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); - } else if (aclType == ACLType.Account) { - networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); - } - - // 2) If null, generate networkDomain using domain suffix from the global config variables - if (networkDomain == null) { - networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); - } - } else { - // validate network domain - if (!NetUtils.verifyDomainName(networkDomain)) { - throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " - + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " - + "and the hyphen ('-'); can't start or end with \"-\""); + if (networkDomain == null) { + // 1) Get networkDomain from the corresponding account/domain/zone + if (aclType == ACLType.Domain) { + networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); + } else if (aclType == ACLType.Account) { + networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); + } + + // 2) If null, generate networkDomain using domain suffix from the global config variables + if (networkDomain == null) { + networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); + } + + } else { + // validate network domain + if (!NetUtils.verifyDomainName(networkDomain)) { + throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " + + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " + + "and the hyphen ('-'); can't start or end with \"-\""); + } } } } - } - // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x - // limitation, remove after we introduce support for multiple ip ranges - // with different Cidrs for the same Shared network - final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced - && ntwkOff.getTrafficType() == TrafficType.Guest - && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); - if (cidr == null && ip6Cidr == null && cidrRequired) { - if (ntwkOff.getGuestType() == GuestType.Shared) { - throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); - } else { - throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x + // limitation, remove after we introduce support for multiple ip ranges + // with different Cidrs for the same Shared network + final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced + && ntwkOff.getTrafficType() == TrafficType.Guest + && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); + if (cidr == null && ip6Cidr == null && cidrRequired) { + if (ntwkOff.getGuestType() == GuestType.Shared) { + throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); + } else { + throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + } } - } - checkL2OfferingServices(ntwkOff); + checkL2OfferingServices(ntwkOff); - // No cidr can be specified in Basic zone - if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { - throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); - } + // No cidr can be specified in Basic zone + if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { + throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); + } - // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 - if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && - !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { + // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 + if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && + !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { throw new InvalidParameterValueException("Virtual Guest Cidr " + cidr + " is not RFC 1918 or 6598 compliant"); - } + } final String networkDomainFinal = networkDomain; final String vlanIdFinal = vlanId; @@ -3004,75 +3003,75 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkVO userNetwork = new NetworkVO(); userNetwork.setNetworkDomain(networkDomainFinal); - if (cidr != null && gateway != null) { - userNetwork.setCidr(cidr); - userNetwork.setGateway(gateway); - } + if (cidr != null && gateway != null) { + userNetwork.setCidr(cidr); + userNetwork.setGateway(gateway); + } - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - userNetwork.setIp6Cidr(ip6Cidr); - userNetwork.setIp6Gateway(ip6Gateway); - } + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + userNetwork.setIp6Cidr(ip6Cidr); + userNetwork.setIp6Gateway(ip6Gateway); + } - if (externalId != null) { - userNetwork.setExternalId(externalId); - } + if (externalId != null) { + userNetwork.setExternalId(externalId); + } - if (StringUtils.isNotBlank(routerIp)) { - userNetwork.setRouterIp(routerIp); - } + if (StringUtils.isNotBlank(routerIp)) { + userNetwork.setRouterIp(routerIp); + } - if (StringUtils.isNotBlank(routerIpv6)) { - userNetwork.setRouterIpv6(routerIpv6); - } + if (StringUtils.isNotBlank(routerIpv6)) { + userNetwork.setRouterIpv6(routerIpv6); + } - if (vrIfaceMTUs != null) { - if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { - userNetwork.setPublicMtu(vrIfaceMTUs.first()); + if (vrIfaceMTUs != null) { + if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { + userNetwork.setPublicMtu(vrIfaceMTUs.first()); + } else { + userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); + } + + if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { + userNetwork.setPrivateMtu(vrIfaceMTUs.second()); + } else { + userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); + } } else { userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); - } - - if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { - userNetwork.setPrivateMtu(vrIfaceMTUs.second()); - } else { userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); } - } else { - userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); - userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); - } - if (!GuestType.L2.equals(userNetwork.getGuestType())) { - if (StringUtils.isNotBlank(ip4Dns1)) { - userNetwork.setDns1(ip4Dns1); - } - if (StringUtils.isNotBlank(ip4Dns2)) { - userNetwork.setDns2(ip4Dns2); - } - if (StringUtils.isNotBlank(ip6Dns1)) { - userNetwork.setIp6Dns1(ip6Dns1); - } - if (StringUtils.isNotBlank(ip6Dns2)) { - userNetwork.setIp6Dns2(ip6Dns2); - } - } - - if (vlanIdFinal != null) { - if (isolatedPvlan == null) { - URI uri = null; - if (UuidUtils.isUuid(vlanIdFinal)) { - //Logical router's UUID provided as VLAN_ID - userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field - } else { - uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); + if (!GuestType.L2.equals(userNetwork.getGuestType())) { + if (StringUtils.isNotBlank(ip4Dns1)) { + userNetwork.setDns1(ip4Dns1); } - - if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", - vlanIdFinal, zone)); + if (StringUtils.isNotBlank(ip4Dns2)) { + userNetwork.setDns2(ip4Dns2); } + if (StringUtils.isNotBlank(ip6Dns1)) { + userNetwork.setIp6Dns1(ip6Dns1); + } + if (StringUtils.isNotBlank(ip6Dns2)) { + userNetwork.setIp6Dns2(ip6Dns2); + } + } + + if (vlanIdFinal != null) { + if (isolatedPvlan == null) { + URI uri = null; + if (UuidUtils.isUuid(vlanIdFinal)) { + //Logical router's UUID provided as VLAN_ID + userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field + } else { + uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); + } + + if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", + vlanIdFinal, zone)); + } userNetwork.setBroadcastUri(uri); if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { @@ -4940,6 +4939,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return new ConfigKey[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes, GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled, - TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN}; + TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN, + NETWORK_LB_HAPROXY_IDLE_TIMEOUT}; } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java index 445a59310fb..377b2f91375 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.upgrade; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -96,7 +97,9 @@ public final class DatabaseVersionHierarchy { // we cannot find the version specified, so get the // most recent one immediately before this version if (!contains(fromVersion)) { - return getPath(getRecentVersion(fromVersion), toVersion); + DbUpgrade[] dbUpgrades = getPath(getRecentVersion(fromVersion), toVersion); + return Arrays.stream(dbUpgrades).filter(up -> CloudStackVersion.compare(up.getUpgradedVersion(), fromVersion.toString()) > 0) + .toArray(DbUpgrade[]::new); } final Predicate predicate; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java index 68100e16401..e0aa38a717b 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java @@ -57,8 +57,4 @@ public class Upgrade42020to42030 extends DbUpgradeAbstractImpl implements DbUpgr public InputStream[] getCleanupScripts() { return null; } - - @Override - public void updateSystemVmTemplates(Connection conn) { - } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql index 3dd6c18f57c..4405250f271 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql @@ -31,7 +31,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" '); -- Create a new group for Usage Server related configurations -INSERT INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); +INSERT IGNORE INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); UPDATE `cloud`.`configuration_subgroup` set `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server'), `precedence` = 1 WHERE `name`='Usage'; UPDATE `cloud`.`configuration` SET `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server') where `subgroup_id` = (SELECT `id` FROM `cloud`.`configuration_subgroup` WHERE `name` = 'Usage'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index 43e0d2c78a9..0300e555463 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -53,3 +53,10 @@ DELETE FROM `cloud`.`configuration` WHERE name = 'consoleproxy.cmd.port'; -- Drops the unused "backup_interval_type" column of the "cloud.backups" table ALTER TABLE `cloud`.`backups` DROP COLUMN `backup_interval_type`; + +-- Update `user.password.reset.mail.template` configuration value to match new logic +UPDATE `cloud`.`configuration` +SET value = CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team') +WHERE name = 'user.password.reset.mail.template' + AND value IN (CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', 'http://{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team'), CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{domainUrl}}}{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team')); + diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 27995eb179a..ab64e4698f0 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -44,6 +44,7 @@ import com.cloud.upgrade.dao.Upgrade41120to41130; import com.cloud.upgrade.dao.Upgrade41120to41200; import com.cloud.upgrade.dao.Upgrade41510to41520; import com.cloud.upgrade.dao.Upgrade41610to41700; +import com.cloud.upgrade.dao.Upgrade42010to42100; import com.cloud.upgrade.dao.Upgrade452to453; import com.cloud.upgrade.dao.Upgrade453to460; import com.cloud.upgrade.dao.Upgrade460to461; @@ -380,4 +381,23 @@ public class DatabaseUpgradeCheckerTest { assertFalse("DatabaseUpgradeChecker should not be a standalone component", checker.isStandalone()); } + @Test + public void testCalculateUpgradePath42010to42100() { + + final CloudStackVersion dbVersion = CloudStackVersion.parse("4.20.1.0"); + assertNotNull(dbVersion); + + final CloudStackVersion currentVersion = CloudStackVersion.parse("4.21.0.0"); + assertNotNull(currentVersion); + + final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); + final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); + + assertNotNull(upgrades); + assertEquals(1, upgrades.length); + assertTrue(upgrades[0] instanceof Upgrade42010to42100); + + assertArrayEquals(new String[]{"4.20.1.0", "4.21.0.0"}, upgrades[0].getUpgradableVersionRange()); + assertEquals(currentVersion.toString(), upgrades[0].getUpgradedVersion()); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 67ea7a1d0dc..0f0f80c0c17 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -18,6 +18,9 @@ package com.cloud.hypervisor.kvm.resource; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_VDDK_LIB_DIR; +import static com.cloud.host.Host.HOST_VDDK_SUPPORT; +import static com.cloud.host.Host.HOST_VDDK_VERSION; import static com.cloud.host.Host.HOST_VIRTV2V_VERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import static org.apache.cloudstack.utils.linux.KVMHostInfo.isHostS390x; @@ -365,6 +368,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public static final String WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "rpm -qa | grep -i virtio-win"; public static final String UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD = "dpkg -l virtio-win"; public static final String UBUNTU_NBDKIT_PKG_CHECK_CMD = "dpkg -l nbdkit"; + public static final String VDDK_AUTODETECT_PATH_CMD = "find / -type d -name 'vmware-vix-disklib-distrib' 2>/dev/null | head -n 1"; public static final int LIBVIRT_CGROUP_CPU_SHARES_MIN = 2; public static final int LIBVIRT_CGROUP_CPU_SHARES_MAX = 262144; @@ -885,10 +889,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv private boolean convertInstanceVerboseMode = false; private Map convertInstanceEnv = null; + private String vddkLibDir = null; + private static final String libguestfsBackend = "direct"; protected boolean dpdkSupport = false; protected String dpdkOvsPath; protected String directDownloadTemporaryDownloadPath; protected String cachePath; + private String vddkTransports = null; + private String vddkThumbprint = null; + private String vddkVersion = null; + private String detectedPasswordFileOption = null; protected String javaTempDir = System.getProperty("java.io.tmpdir"); private String getEndIpFromStartIp(final String startIp, final int numIps) { @@ -953,6 +963,26 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return convertInstanceEnv; } + public String getVddkLibDir() { + return vddkLibDir; + } + + public String getLibguestfsBackend() { + return libguestfsBackend; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public String getVddkVersion() { + return vddkVersion; + } + /** * Defines resource's public and private network interface according to what is configured in agent.properties. */ @@ -1153,6 +1183,37 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv setConvertInstanceEnv(convertEnvTmpDir, convertEnvVirtv2vTmpDir); + vddkLibDir = StringUtils.trimToNull(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_LIB_DIR)); + if (StringUtils.isNotBlank(vddkLibDir) && !isVddkLibDirValid(vddkLibDir)) { + LOGGER.warn("Configured VDDK library dir [{}] is invalid (missing lib64/libvixDiskLib.so), attempting auto-detection", vddkLibDir); + vddkLibDir = null; + } + if (StringUtils.isBlank(vddkLibDir)) { + vddkLibDir = detectVddkLibDir(); + } + if (StringUtils.isNotBlank(vddkLibDir)) { + LOGGER.info("Detected VDDK library dir: {}", vddkLibDir); + } else { + LOGGER.warn("Could not detect a valid VDDK library dir; VDDK conversion will be unavailable"); + } + + vddkVersion = detectVddkVersion(); + if (StringUtils.isNotBlank(vddkVersion)) { + LOGGER.info("Detected nbdkit VDDK plugin version: {}", vddkVersion); + } + + vddkTransports = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_TRANSPORTS)); + vddkThumbprint = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_THUMBPRINT)); + + detectedPasswordFileOption = detectPasswordFileOption(); + if (StringUtils.isNotBlank(detectedPasswordFileOption)) { + LOGGER.info("Detected virt-v2v password option: {}", detectedPasswordFileOption); + } else { + LOGGER.warn("Could not detect virt-v2v password option, VDDK conversions may fail"); + } + pool = (String)params.get("pool"); if (pool == null) { pool = "/root"; @@ -4224,6 +4285,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cmd.setHostTags(getHostTags()); boolean instanceConversionSupported = hostSupportsInstanceConversion(); cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); + cmd.getHostDetails().put(HOST_VDDK_SUPPORT, String.valueOf(hostSupportsVddk())); + if (StringUtils.isNotBlank(vddkLibDir)) { + cmd.getHostDetails().put(HOST_VDDK_LIB_DIR, vddkLibDir); + } + if (StringUtils.isNotBlank(vddkVersion)) { + cmd.getHostDetails().put(HOST_VDDK_VERSION, vddkVersion); + } if (instanceConversionSupported) { cmd.getHostDetails().put(HOST_VIRTV2V_VERSION, getHostVirtV2vVersion()); } @@ -5945,6 +6013,66 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + public boolean hostSupportsVddk() { + return hostSupportsVddk(null); + } + + public boolean hostSupportsVddk(String overriddenVddkLibDir) { + String effectiveVddkLibDir = StringUtils.trimToNull(overriddenVddkLibDir); + if (StringUtils.isBlank(effectiveVddkLibDir)) { + effectiveVddkLibDir = StringUtils.trimToNull(vddkLibDir); + } + if (StringUtils.isBlank(effectiveVddkLibDir) || !isVddkLibDirValid(effectiveVddkLibDir)) { + effectiveVddkLibDir = detectVddkLibDir(); + } + return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(detectVddkVersion()); + } + + protected boolean isVddkLibDirValid(String path) { + if (StringUtils.isBlank(path)) { + return false; + } + File libDir = new File(path, "lib64"); + if (!libDir.isDirectory()) { + return false; + } + File[] libs = libDir.listFiles((dir, name) -> name.startsWith("libvixDiskLib.so")); + return libs != null && libs.length > 0; + } + + protected String detectVddkLibDir() { + String detectedPath = StringUtils.trimToNull(Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD)); + if (StringUtils.isNotBlank(detectedPath) && isVddkLibDirValid(detectedPath)) { + return detectedPath; + } + return null; + } + + protected String detectVddkVersion() { + try { + ProcessBuilder pb = new ProcessBuilder("nbdkit", "vddk", "--version"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (StringUtils.isBlank(output)) { + return null; + } + + for (String line : output.split("\\R")) { + String trimmed = StringUtils.trimToEmpty(line); + if (trimmed.startsWith("vddk ")) { + return StringUtils.trimToNull(trimmed.substring("vddk ".length())); + } + } + return null; + } catch (Exception e) { + LOGGER.error("Failed to detect vddk version: {}", e.getMessage()); + return null; + } + } + public boolean hostSupportsWindowsGuestConversion() { if (isUbuntuOrDebianHost()) { int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); @@ -5959,6 +6087,40 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + /** + * Detect which password option virt-v2v supports by examining its --help output + * @return "-ip" if supported (virt-v2v >= 2.8.1), "--password-file" if older version, or null if detection fails + */ + protected String detectPasswordFileOption() { + try { + ProcessBuilder pb = new ProcessBuilder("virt-v2v", "--help"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (output.contains("-ip ")) { + return "-ip"; + } else if (output.contains("--password-file")) { + return "--password-file"; + } else { + LOGGER.error("virt-v2v does not support -ip or --password-file"); + return null; + } + } catch (Exception e) { + LOGGER.error("Failed to detect virt-v2v password option: {}", e.getMessage()); + return null; + } + } + + /** + * Get the detected password file option for virt-v2v + * @return the password option ("-ip" or "--password-file") or null if not detected + */ + public String getDetectedPasswordFileOption() { + return detectedPasswordFileOption; + } + public String getHostVirtV2vVersion() { if (!hostSupportsInstanceConversion()) { return ""; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java index b94b4830003..de9341715f0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java @@ -30,7 +30,15 @@ public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper supportedInstanceConvertSourceHypervisors = List.of(Hypervisor.HypervisorType.VMware); + private static final Pattern SHA1_FINGERPRINT_PATTERN = Pattern.compile("(?i)(?:SHA1\\s+)?Fingerprint\\s*=\\s*([0-9A-F:]+)"); @Override public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { @@ -61,7 +69,8 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { - noOfThreads = 0; - } - ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); - temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); - if (!ovfExported) { - String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); - logger.error(String.format("(%s) %s", originalVMName, err)); - return new Answer(cmd, false, err); - } - sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); - } else { - ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - } - - logger.info(String.format("(%s) Attempting to convert the OVF %s of the instance %s from %s to KVM", - originalVMName, ovfTemplateDirOnConversionLocation, sourceInstanceName, sourceHypervisorType)); - final String temporaryConvertUuid = UUID.randomUUID().toString(); boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); boolean cleanupSecondaryStorage = false; + boolean ovfExported = false; + String ovfTemplateDirOnConversionLocation = null; + try { - boolean result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, - timeout, verboseModeEnabled, extraParams, serverResource); + boolean result; + if (useVddk) { + logger.info("({}) Using VDDK-based conversion (direct from VMware)", originalVMName); + String vddkLibDir = resolveVddkSetting(cmd.getVddkLibDir(), serverResource.getVddkLibDir()); + if (StringUtils.isBlank(vddkLibDir)) { + String err = String.format("VDDK lib dir is not configured on the host. " + + "Set '%s' in agent.properties or in details parameter of the import api calll to use VDDK-based conversion.", "vddk.lib.dir"); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + String vddkTransports = resolveVddkSetting(cmd.getVddkTransports(), serverResource.getVddkTransports()); + String configuredVddkThumbprint = resolveVddkSetting(cmd.getVddkThumbprint(), serverResource.getVddkThumbprint()); + String passwordOption = serverResource.getDetectedPasswordFileOption(); + result = performInstanceConversionUsingVddk(sourceInstance, originalVMName, temporaryConvertPath, + vddkLibDir, serverResource.getLibguestfsBackend(), vddkTransports, configuredVddkThumbprint, + timeout, verboseModeEnabled, extraParams, temporaryConvertUuid, passwordOption); + } else { + logger.info("({}) Using OVF-based conversion (export + local convert)", originalVMName); + String sourceOVFDirPath; + if (cmd.getExportOvfToConversionLocation()) { + String exportInstanceOVAUrl = getExportInstanceOVAUrl(sourceInstance, originalVMName); + + if (StringUtils.isBlank(exportInstanceOVAUrl)) { + String err = String.format("Couldn't export OVA for the VM %s, due to empty url", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + + int noOfThreads = cmd.getThreadsCountToExportOvf(); + if (noOfThreads > 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { + noOfThreads = 0; + } + ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); + temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); + + if (!ovfExported) { + String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); + } else { + ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + } + + result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, + timeout, verboseModeEnabled, extraParams, serverResource); + } + if (!result) { - String err = String.format( - "The virt-v2v conversion for the OVF %s failed. Please check the agent logs " + - "for the virt-v2v output. Please try on a different kvm host which " + - "has a different virt-v2v version.", - ovfTemplateDirOnConversionLocation); - logger.error(String.format("(%s) %s", originalVMName, err)); + String err = String.format("Instance conversion failed for VM %s. Please check virt-v2v logs.", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); return new Answer(cmd, false, err); } return new ConvertInstanceAnswer(cmd, temporaryConvertUuid); } catch (Exception e) { - String error = String.format("Error converting instance %s from %s, due to: %s", - sourceInstanceName, sourceHypervisorType, e.getMessage()); - logger.error(String.format("(%s) %s", originalVMName, error), e); + String error = String.format("Error converting instance %s from %s, due to: %s", sourceInstanceName, sourceHypervisorType, e.getMessage()); + logger.error("({}) {}", originalVMName, error, e); cleanupSecondaryStorage = true; return new Answer(cmd, false, error); } finally { @@ -275,4 +298,198 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper/dev/null | " + + "openssl x509 -fingerprint -sha1 -noout", endpoint); + + Script script = new Script("/bin/bash", timeout, logger); + script.add("-c"); + script.add(command); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + script.execute(parser); + + String output = parser.getLines(); + if (script.getExitValue() != 0) { + logger.error("({}) Failed to fetch vCenter thumbprint for {}", originalVMName, vcenterHost); + return null; + } + + String thumbprint = extractSha1Fingerprint(output); + if (StringUtils.isBlank(thumbprint)) { + logger.error("({}) Failed to parse vCenter thumbprint from output for {}", originalVMName, vcenterHost); + return null; + } + return thumbprint; + } + + private String extractSha1Fingerprint(String output) { + String parsedOutput = StringUtils.trimToEmpty(output); + if (StringUtils.isBlank(parsedOutput)) { + return null; + } + + for (String line : parsedOutput.split("\\R")) { + String trimmedLine = StringUtils.trimToEmpty(line); + if (StringUtils.isBlank(trimmedLine)) { + continue; + } + + Matcher matcher = SHA1_FINGERPRINT_PATTERN.matcher(trimmedLine); + if (matcher.find()) { + return matcher.group(1).toUpperCase(Locale.ROOT); + } + + // Fallback for raw fingerprint-only output. + if (trimmedLine.matches("(?i)[0-9a-f]{2}(:[0-9a-f]{2})+")) { + return trimmedLine.toUpperCase(Locale.ROOT); + } + } + return null; + } + + /** + * Build vpx:// URL for virt-v2v + * + * Format: + * vpx://user@vcenter/DC/cluster/host?no_verify=1 + */ + private String buildVpxUrl(RemoteInstanceTO vmwareInstance) { + + String vmName = vmwareInstance.getInstanceName(); + String vcenter = vmwareInstance.getVcenterHost(); + String username = vmwareInstance.getVcenterUsername(); + String datacenter = vmwareInstance.getDatacenterName(); + String cluster = vmwareInstance.getClusterName(); + String host = vmwareInstance.getHostName(); + + String encodedUsername = encodeUsername(username); + + StringBuilder url = new StringBuilder(); + url.append("vpx://") + .append(encodedUsername) + .append("@") + .append(vcenter) + .append("/") + .append(datacenter); + + if (StringUtils.isNotBlank(cluster)) { + url.append("/").append(cluster); + } + + if (StringUtils.isNotBlank(host)) { + url.append("/").append(host); + } + + url.append("?no_verify=1"); + + logger.info("({}) Using VPX URL: {}", vmName, url); + return url.toString(); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index e74923b281f..5a7d6d2c203 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -34,6 +34,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; @ResourceWrapper(handles = ReadyCommand.class) public final class LibvirtReadyCommandWrapper extends CommandWrapper { @@ -50,6 +51,9 @@ public final class LibvirtReadyCommandWrapper extends CommandWrapper filesMock = Mockito.mockStatic(Files.class); + MockedConstruction