From 8221be3a8ce6177545478720fc2e776eb0737242 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Thu, 13 Jun 2024 11:29:51 +0530 Subject: [PATCH 01/17] Fix marvin package version while building packages (#9230) --- tools/marvin/mvn-setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/marvin/mvn-setup.py b/tools/marvin/mvn-setup.py index 61bd2219238..cabcf0dc659 100755 --- a/tools/marvin/mvn-setup.py +++ b/tools/marvin/mvn-setup.py @@ -34,6 +34,8 @@ def replaceVersion(fname, version): with open(fname, 'r') as f: content = f.read() needle = '\nVERSION\s*=\s*[\'"][^\'"]*[\'"]' + # Ensure the version is PEP440 compliant + version = version.replace('-', '+', 1) replacement = '\nVERSION = "%s"' % version content = re.sub(needle, replacement, content, 1) with open(fname, 'w') as f: From 78ace3a750c0285884b52646d4926052023ebd00 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 13 Jun 2024 11:30:33 +0530 Subject: [PATCH 02/17] saml: introduce saml2.check.signature (#9219) Adminstrators should ensure that IDP configuration has signing certificate for the actual signature check to be performed. In addition to this, this change introduces a new global setting `saml2.check.signature` which can deliberately fail a SAML login attempt when the SAML response has missing signature. Signed-off-by: Rohit Yadav --- .../SAML2LoginAPIAuthenticatorCmd.java | 15 +++++++-- .../cloudstack/saml/SAML2AuthManager.java | 33 ++++++++++--------- .../cloudstack/saml/SAML2AuthManagerImpl.java | 2 +- .../SAML2LoginAPIAuthenticatorCmdTest.java | 24 ++++++++++++++ 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java index 6bb3e788a95..fd4da7a59b2 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java @@ -144,6 +144,14 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent return responseObject; } + protected void checkAndFailOnMissingSAMLSignature(Signature signature) { + if (signature == null && SAML2AuthManager.SAMLCheckSignature.value()) { + s_logger.error("Failing SAML login due to missing signature in the SAML response and signature check is enforced. " + + "Please check and ensure the IDP configuration has signing certificate or relax the saml2.check.signature setting."); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Signature is missing from the SAML Response. Please contact the Administrator"); + } + } + @Override public String authenticate(final String command, final Map params, final HttpSession session, final InetAddress remoteAddress, final String responseType, final StringBuilder auditTrailSb, final HttpServletRequest req, final HttpServletResponse resp) throws ServerApiException { try { @@ -225,6 +233,7 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent session.setAttribute(SAMLPluginConstants.SAML_IDPID, issuer.getValue()); Signature sig = processedSAMLResponse.getSignature(); + checkAndFailOnMissingSAMLSignature(sig); if (idpMetadata.getSigningCertificate() != null && sig != null) { BasicX509Credential credential = new BasicX509Credential(); credential.setEntityCertificate(idpMetadata.getSigningCertificate()); @@ -238,9 +247,8 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent params, responseType)); } } - if (username == null) { - username = SAMLUtils.getValueFromAssertions(processedSAMLResponse.getAssertions(), SAML2AuthManager.SAMLUserAttributeName.value()); - } + + username = SAMLUtils.getValueFromAssertions(processedSAMLResponse.getAssertions(), SAML2AuthManager.SAMLUserAttributeName.value()); for (Assertion assertion: processedSAMLResponse.getAssertions()) { if (assertion!= null && assertion.getSubject() != null && assertion.getSubject().getNameID() != null) { @@ -272,6 +280,7 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent continue; } Signature encSig = assertion.getSignature(); + checkAndFailOnMissingSAMLSignature(encSig); if (idpMetadata.getSigningCertificate() != null && encSig != null) { BasicX509Credential sigCredential = new BasicX509Credential(); sigCredential.setEntityCertificate(idpMetadata.getSigningCertificate()); diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java index e52a7e32695..a5dae36581c 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java @@ -25,51 +25,54 @@ import java.util.Collection; public interface SAML2AuthManager extends PluggableAPIAuthenticator, PluggableService { - public static final ConfigKey SAMLIsPluginEnabled = new ConfigKey("Advanced", Boolean.class, "saml2.enabled", "false", + ConfigKey SAMLIsPluginEnabled = new ConfigKey("Advanced", Boolean.class, "saml2.enabled", "false", "Indicates whether SAML SSO plugin is enabled or not", true); - public static final ConfigKey SAMLServiceProviderID = new ConfigKey("Advanced", String.class, "saml2.sp.id", "org.apache.cloudstack", + ConfigKey SAMLServiceProviderID = new ConfigKey("Advanced", String.class, "saml2.sp.id", "org.apache.cloudstack", "SAML2 Service Provider Identifier String", true); - public static final ConfigKey SAMLServiceProviderContactPersonName = new ConfigKey("Advanced", String.class, "saml2.sp.contact.person", "CloudStack Developers", + ConfigKey SAMLServiceProviderContactPersonName = new ConfigKey("Advanced", String.class, "saml2.sp.contact.person", "CloudStack Developers", "SAML2 Service Provider Contact Person Name", true); - public static final ConfigKey SAMLServiceProviderContactEmail = new ConfigKey("Advanced", String.class, "saml2.sp.contact.email", "dev@cloudstack.apache.org", + ConfigKey SAMLServiceProviderContactEmail = new ConfigKey("Advanced", String.class, "saml2.sp.contact.email", "dev@cloudstack.apache.org", "SAML2 Service Provider Contact Email Address", true); - public static final ConfigKey SAMLServiceProviderOrgName = new ConfigKey("Advanced", String.class, "saml2.sp.org.name", "Apache CloudStack", + ConfigKey SAMLServiceProviderOrgName = new ConfigKey("Advanced", String.class, "saml2.sp.org.name", "Apache CloudStack", "SAML2 Service Provider Organization Name", true); - public static final ConfigKey SAMLServiceProviderOrgUrl = new ConfigKey("Advanced", String.class, "saml2.sp.org.url", "http://cloudstack.apache.org", + ConfigKey SAMLServiceProviderOrgUrl = new ConfigKey("Advanced", String.class, "saml2.sp.org.url", "http://cloudstack.apache.org", "SAML2 Service Provider Organization URL", true); - public static final ConfigKey SAMLServiceProviderSingleSignOnURL = new ConfigKey("Advanced", String.class, "saml2.sp.sso.url", "http://localhost:8080/client/api?command=samlSso", + ConfigKey SAMLServiceProviderSingleSignOnURL = new ConfigKey("Advanced", String.class, "saml2.sp.sso.url", "http://localhost:8080/client/api?command=samlSso", "SAML2 CloudStack Service Provider Single Sign On URL", true); - public static final ConfigKey SAMLServiceProviderSingleLogOutURL = new ConfigKey("Advanced", String.class, "saml2.sp.slo.url", "http://localhost:8080/client/", + ConfigKey SAMLServiceProviderSingleLogOutURL = new ConfigKey("Advanced", String.class, "saml2.sp.slo.url", "http://localhost:8080/client/", "SAML2 CloudStack Service Provider Single Log Out URL", true); - public static final ConfigKey SAMLCloudStackRedirectionUrl = new ConfigKey("Advanced", String.class, "saml2.redirect.url", "http://localhost:8080/client", + ConfigKey SAMLCloudStackRedirectionUrl = new ConfigKey("Advanced", String.class, "saml2.redirect.url", "http://localhost:8080/client", "The CloudStack UI url the SSO should redirected to when successful", true); - public static final ConfigKey SAMLUserAttributeName = new ConfigKey("Advanced", String.class, "saml2.user.attribute", "uid", + ConfigKey SAMLUserAttributeName = new ConfigKey("Advanced", String.class, "saml2.user.attribute", "uid", "Attribute name to be looked for in SAML response that will contain the username", true); - public static final ConfigKey SAMLIdentityProviderMetadataURL = new ConfigKey("Advanced", String.class, "saml2.idp.metadata.url", "https://openidp.feide.no/simplesaml/saml2/idp/metadata.php", + ConfigKey SAMLIdentityProviderMetadataURL = new ConfigKey("Advanced", String.class, "saml2.idp.metadata.url", "https://openidp.feide.no/simplesaml/saml2/idp/metadata.php", "SAML2 Identity Provider Metadata XML Url", true); - public static final ConfigKey SAMLDefaultIdentityProviderId = new ConfigKey("Advanced", String.class, "saml2.default.idpid", "https://openidp.feide.no", + ConfigKey SAMLDefaultIdentityProviderId = new ConfigKey("Advanced", String.class, "saml2.default.idpid", "https://openidp.feide.no", "The default IdP entity ID to use only in case of multiple IdPs", true); - public static final ConfigKey SAMLSignatureAlgorithm = new ConfigKey<>(String.class, "saml2.sigalg", "Advanced", "SHA1", + ConfigKey SAMLSignatureAlgorithm = new ConfigKey<>(String.class, "saml2.sigalg", "Advanced", "SHA1", "The algorithm to use to when signing a SAML request. Default is SHA1, allowed algorithms: SHA1, SHA256, SHA384, SHA512", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "SHA1,SHA256,SHA384,SHA512"); - public static final ConfigKey SAMLAppendDomainSuffix = new ConfigKey("Advanced", Boolean.class, "saml2.append.idpdomain", "false", + ConfigKey SAMLAppendDomainSuffix = new ConfigKey("Advanced", Boolean.class, "saml2.append.idpdomain", "false", "If enabled, create account/user dialog with SAML SSO enabled will append the IdP domain to the user or account name in the UI dialog", true); - public static final ConfigKey SAMLTimeout = new ConfigKey("Advanced", Integer.class, "saml2.timeout", "1800", + ConfigKey SAMLTimeout = new ConfigKey("Advanced", Integer.class, "saml2.timeout", "1800", "SAML2 IDP Metadata refresh interval in seconds, minimum value is set to 300", true); + ConfigKey SAMLCheckSignature = new ConfigKey("Advanced", Boolean.class, "saml2.check.signature", "false", + "Whether SAML2 signature must be checked, when enforced and when the SAML response does not have a signature would lead to login exception", true); + public SAMLProviderMetadata getSPMetadata(); public SAMLProviderMetadata getIdPMetadata(String entityId); public Collection getAllIdPMetadata(); diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java index ba85b151eea..3ecebf4d185 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java @@ -535,6 +535,6 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage SAMLServiceProviderSingleSignOnURL, SAMLServiceProviderSingleLogOutURL, SAMLCloudStackRedirectionUrl, SAMLUserAttributeName, SAMLIdentityProviderMetadataURL, SAMLDefaultIdentityProviderId, - SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout}; + SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature}; } } diff --git a/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java b/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java index 39c8c231bf0..ebdc5be8fe9 100644 --- a/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java +++ b/plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java @@ -271,6 +271,30 @@ public class SAML2LoginAPIAuthenticatorCmdTest { verifyTestWhenFailToAuthenticateThrowExceptionOrRedirectToUrl(false, hasThrownServerApiException, 0, 0); } + private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException { + Field f = ConfigKey.class.getDeclaredField(name); + f.setAccessible(true); + f.set(configKey, o); + } + + @Test + public void testFailOnSAMLSignatureCheckWhenFalse() throws NoSuchFieldException, IllegalAccessException { + overrideDefaultConfigValue(SAML2AuthManager.SAMLCheckSignature, "_defaultValue", "false"); + SAML2LoginAPIAuthenticatorCmd cmd = new SAML2LoginAPIAuthenticatorCmd(); + try { + cmd.checkAndFailOnMissingSAMLSignature(null); + } catch(Exception e) { + Assert.fail("This shouldn't throw any exception"); + } + } + + @Test(expected = ServerApiException.class) + public void testFailOnSAMLSignatureCheckWhenTrue() throws NoSuchFieldException, IllegalAccessException { + overrideDefaultConfigValue(SAML2AuthManager.SAMLCheckSignature, "_defaultValue", "true"); + SAML2LoginAPIAuthenticatorCmd cmd = new SAML2LoginAPIAuthenticatorCmd(); + cmd.checkAndFailOnMissingSAMLSignature(null); + } + private UserAccountVO configureTestWhenFailToAuthenticateThrowExceptionOrRedirectToUrl(String entity, String configurationValue, Boolean isUserAuthorized) throws IOException { Mockito.when(samlAuthManager.isUserAuthorized(nullable(Long.class), nullable(String.class))).thenReturn(isUserAuthorized); From f45267174a1c49e38ced5fd7cecaf71726369cda Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 13 Jun 2024 11:58:36 +0530 Subject: [PATCH 03/17] ui: list only accessible networks during import (#9194) * ui: list only accessible networks during import Fixes #8612 Signed-off-by: Abhishek Kumar * fix Signed-off-by: Abhishek Kumar * space Signed-off-by: Abhishek Kumar * changes --------- Signed-off-by: Abhishek Kumar --- .../compute/wizard/MultiNetworkSelection.vue | 32 ++++++++++++++++--- .../views/tools/ImportUnmanagedInstance.vue | 6 +++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/ui/src/views/compute/wizard/MultiNetworkSelection.vue b/ui/src/views/compute/wizard/MultiNetworkSelection.vue index 048a70671d9..92d4a437599 100644 --- a/ui/src/views/compute/wizard/MultiNetworkSelection.vue +++ b/ui/src/views/compute/wizard/MultiNetworkSelection.vue @@ -99,6 +99,14 @@ export default { type: String, default: () => '' }, + domainid: { + type: String, + default: '' + }, + account: { + type: String, + default: '' + }, selectionEnabled: { type: Boolean, default: true @@ -144,7 +152,8 @@ export default { ipAddressesEnabled: {}, ipAddresses: {}, indexNum: 1, - sendValuesTimer: null + sendValuesTimer: null, + accountNetworkUpdateTimer: null } }, computed: { @@ -184,6 +193,14 @@ export default { }, zoneId () { this.fetchNetworks() + }, + account () { + clearTimeout(this.accountNetworkUpdateTimer) + this.accountNetworkUpdateTimer = setTimeout(() => { + if (this.account) { + this.fetchNetworks() + } + }, 750) } }, created () { @@ -196,13 +213,20 @@ export default { return } this.loading = true - api('listNetworks', { + var params = { zoneid: this.zoneId, listall: true - }).then(response => { + } + if (this.domainid && this.account) { + params.domainid = this.domainid + params.account = this.account + } + api('listNetworks', params).then(response => { this.networks = response.listnetworksresponse.network || [] - this.orderNetworks() + }).catch(() => { + this.networks = [] }).finally(() => { + this.orderNetworks() this.loading = false }) }, diff --git a/ui/src/views/tools/ImportUnmanagedInstance.vue b/ui/src/views/tools/ImportUnmanagedInstance.vue index 9eb74877365..4fb559b2628 100644 --- a/ui/src/views/tools/ImportUnmanagedInstance.vue +++ b/ui/src/views/tools/ImportUnmanagedInstance.vue @@ -297,6 +297,8 @@ Date: Thu, 13 Jun 2024 12:29:11 +0530 Subject: [PATCH 04/17] engine/schema: force index in user_vm_view to speed up VM instance listing (#9198) The user_vm_view can end up not picking the right index to join against the user_ip_address table causing full table scan on the user_ip_address table. This could be related to a MySQL bug https://bugs.mysql.com/bug.php?id=41220 In a test environment with 20k shared networks and over 20M IPs, the listVirtualMachines API was found to take over 17s to return list of just 10 VMs. However, with this fix it would now take under 200ms to return the list. MySQL slow query logging showed ~nearly 20M table scans of the IP address table: ``` # User@Host: cloud[cloud] @ localhost [127.0.0.1] Id: 39 # Query_time: 8.227541 Lock_time: 0.000014 Rows_sent: 12 Rows_examined: 19,667,235 SET timestamp=1715410270; SELECT user_vm_view.id, user_vm_view.name /*snipped*/ FROM user_vm_view WHERE user_vm_view.id IN (4,6,7,8,9,10,11,12,13,14,15,16); ``` Signed-off-by: Rohit Yadav --- .../src/main/resources/META-INF/db/views/cloud.user_vm_view.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index 25f95709721..62294ed5d89 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -196,7 +196,7 @@ FROM LEFT JOIN `networks` ON ((`nics`.`network_id` = `networks`.`id`))) LEFT JOIN `vpc` ON (((`networks`.`vpc_id` = `vpc`.`id`) AND ISNULL(`vpc`.`removed`)))) - LEFT JOIN `user_ip_address` ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`))) + LEFT JOIN `user_ip_address` FORCE INDEX(`fk_user_ip_address__vm_id`) ON ((`user_ip_address`.`vm_id` = `vm_instance`.`id`))) LEFT JOIN `user_vm_details` `ssh_details` ON (((`ssh_details`.`vm_id` = `vm_instance`.`id`) AND (`ssh_details`.`name` = 'SSH.KeyPairNames')))) LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_instance`.`id`) From b3c3f917186efe46bc84e1bf7955989f5cb0b935 Mon Sep 17 00:00:00 2001 From: dahn Date: Thu, 13 Jun 2024 09:06:20 +0200 Subject: [PATCH 05/17] api: add to cpu speed parameter a description of the cgroup2 case (#9191) This PR adds to the cpuSpeed parameter of CreateServiceOfferingCmd a description how it will be interpreted in the case cgroups are being used on KVM. Fixes: #6743 Co-authored-by: Bryan Lima <42067040+BryanMLima@users.noreply.github.com> --------- Co-authored-by: Bryan Lima <42067040+BryanMLima@users.noreply.github.com> --- .../command/admin/offering/CreateServiceOfferingCmd.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java index d947f6f0659..e3b2887ad33 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java @@ -56,7 +56,11 @@ public class CreateServiceOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.CPU_NUMBER, type = CommandType.INTEGER, required = false, description = "the CPU number of the service offering") private Integer cpuNumber; - @Parameter(name = ApiConstants.CPU_SPEED, type = CommandType.INTEGER, required = false, description = "the CPU speed of the service offering in MHz.") + @Parameter(name = ApiConstants.CPU_SPEED, type = CommandType.INTEGER, required = false, description = "For VMware and Xen based hypervisors this is the CPU speed of the service offering in MHz.\n" + + "For the KVM hypervisor," + + " the values of the parameters cpuSpeed and cpuNumber will be used to calculate the `shares` value. This value is used by the KVM hypervisor to calculate how much time" + + " the VM will have access to the host's CPU. The `shares` value does not have a unit, and its purpose is being a weight value for the host to compare between its guest" + + " VMs. For more information, see https://libvirt.org/formatdomain.html#cpu-tuning.") private Integer cpuSpeed; @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, description = "The display text of the service offering, defaults to 'name'.") From 517cddcb15cbbd95300e31ac5f5331140515bcd4 Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Thu, 13 Jun 2024 12:38:51 +0530 Subject: [PATCH 06/17] Fix error message if specific host does not have capacity (#9218) --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4d966d8e745..a8810dea54e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5409,7 +5409,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir final ServiceOfferingVO offering = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); Pair cpuCapabilityAndCapacity = _capacityMgr.checkIfHostHasCpuCapabilityAndCapacity(destinationHost, offering, false); if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - String errorMsg = "Cannot deploy the VM to specified host " + hostId + "; host has cpu capability? " + cpuCapabilityAndCapacity.first() + ", host has capacity? " + cpuCapabilityAndCapacity.second(); + String errorMsg; + if (!cpuCapabilityAndCapacity.first()) { + errorMsg = String.format("Cannot deploy the VM to specified host %d, requested CPU and speed is more than the host capability", hostId); + } else { + errorMsg = String.format("Cannot deploy the VM to specified host %d, host does not have enough free CPU or RAM, please check the logs", hostId); + } s_logger.info(errorMsg); if (!AllowDeployVmIfGivenHostFails.value()) { throw new InvalidParameterValueException(errorMsg); From 2fef0a32bc8ea797baf31bc6f6b7c81bb7740fc4 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 13 Jun 2024 13:08:19 +0530 Subject: [PATCH 07/17] cks: fix list apis response count (#8701) * cks: fix list apis count Fixes count value in listKubernetesClusters and listSupportedKubernetesVersions APIs response. --- .../cluster/KubernetesClusterManagerImpl.java | 8 ++++---- .../version/KubernetesVersionManagerImpl.java | 14 +++++++++----- .../version/KubernetesVersionServiceTest.java | 17 +++++++++-------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 281fe84089f..04c8a5d7554 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1486,13 +1486,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (clusterType != null) { sc.setParameters("cluster_type", clusterType); } - List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); - for (KubernetesClusterVO cluster : kubernetesClusters) { + Pair, Integer> kubernetesClustersAndCount = kubernetesClusterDao.searchAndCount(sc, searchFilter); + for (KubernetesClusterVO cluster : kubernetesClustersAndCount.first()) { KubernetesClusterResponse clusterResponse = createKubernetesClusterResponse(cluster.getId()); responsesList.add(clusterResponse); } - ListResponse response = new ListResponse(); - response.setResponses(responsesList); + ListResponse response = new ListResponse<>(); + response.setResponses(responsesList, kubernetesClustersAndCount.second()); return response; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index b2dafbf70bc..f31dc36768f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -52,6 +52,7 @@ import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.AccountManager; +import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.Filter; @@ -119,13 +120,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne return response; } - private ListResponse createKubernetesSupportedVersionListResponse(List versions) { + private ListResponse createKubernetesSupportedVersionListResponse( + List versions, Integer count) { List responseList = new ArrayList<>(); for (KubernetesSupportedVersionVO version : versions) { responseList.add(createKubernetesSupportedVersionResponse(version)); } ListResponse response = new ListResponse<>(); - response.setResponses(responseList); + response.setResponses(responseList, count); return response; } @@ -280,10 +282,12 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne if(keyword != null){ sc.setParameters("keyword", "%" + keyword + "%"); } - List versions = kubernetesSupportedVersionDao.search(sc, searchFilter); - versions = filterKubernetesSupportedVersions(versions, minimumSemanticVersion); + Pair, Integer> versionsAndCount = + kubernetesSupportedVersionDao.searchAndCount(sc, searchFilter); + List versions = + filterKubernetesSupportedVersions(versionsAndCount.first(), minimumSemanticVersion); - return createKubernetesSupportedVersionListResponse(versions); + return createKubernetesSupportedVersionListResponse(versions, versionsAndCount.second()); } @Override diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index d92b2c438c0..d3412bbf750 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.commons.collections.CollectionUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -66,6 +67,7 @@ import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.User; import com.cloud.user.UserVO; +import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; @@ -142,14 +144,13 @@ public class KubernetesVersionServiceTest { when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); versionVOs.add(versionVO); when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); - when(kubernetesSupportedVersionDao.search(Mockito.any(SearchCriteria.class), Mockito.any(Filter.class))).thenReturn(versionVOs); - ListResponse response = - kubernetesVersionService.listKubernetesSupportedVersions( - cmd); - Assert.assertNotNull(response); - Assert.assertEquals(Integer.valueOf(1), response.getCount()); - Assert.assertEquals(1, response.getResponses().size()); - Assert.assertEquals(KubernetesVersionService.MIN_KUBERNETES_VERSION, response.getResponses().get(0).getSemanticVersion()); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(SearchCriteria.class), + Mockito.any(Filter.class))).thenReturn(new Pair<>(versionVOs, versionVOs.size())); + ListResponse versionsResponse = + kubernetesVersionService.listKubernetesSupportedVersions(cmd); + Assert.assertEquals(versionVOs.size(), versionsResponse.getCount().intValue()); + Assert.assertTrue(CollectionUtils.isNotEmpty(versionsResponse.getResponses())); + Assert.assertEquals(versionVOs.size(), versionsResponse.getResponses().size()); } @Test(expected = InvalidParameterValueException.class) From abbc61c01ecef93d575efd6945746f4cb6edae63 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 13 Jun 2024 13:30:22 +0530 Subject: [PATCH 08/17] engine-orchestration: expunge destroyed system vm volume (#9197) Signed-off-by: Abhishek Kumar --- .../cloudstack/engine/orchestration/VolumeOrchestrator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 565c3f2101a..5ad9f5e5ddf 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1186,7 +1186,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati s_logger.error(String.format("Unable to destroy existing volume [%s] due to [%s].", volumeToString, e.getMessage())); } // In case of VMware VM will continue to use the old root disk until expunged, so force expunge old root disk - if (vm.getHypervisorType() == HypervisorType.VMware) { + // For system VM we do not need volume entry in Destroy state + if (vm.getHypervisorType() == HypervisorType.VMware || vm.getType().isUsedBySystem()) { s_logger.info(String.format("Trying to expunge volume [%s] from primary data storage.", volumeToString)); AsyncCallFuture future = volService.expungeVolumeAsync(volFactory.getVolume(existingVolume.getId())); try { From 19e9020c9bbbb65a97ea51bf7efd3d39fceaf486 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 13 Jun 2024 13:53:44 +0530 Subject: [PATCH 09/17] ui: fix dashboard retrievals based on permissions (#9237) Signed-off-by: Abhishek Kumar --- ui/src/views/dashboard/UsageDashboard.vue | 106 ++++++++++++++-------- 1 file changed, 66 insertions(+), 40 deletions(-) diff --git a/ui/src/views/dashboard/UsageDashboard.vue b/ui/src/views/dashboard/UsageDashboard.vue index fe835cbd0d0..9df3b38c6a9 100644 --- a/ui/src/views/dashboard/UsageDashboard.vue +++ b/ui/src/views/dashboard/UsageDashboard.vue @@ -51,7 +51,7 @@ - + - + - + - + - + - + - + - + - +