From 2397c22f4a042df5940ec06e331a2d92e0573d34 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 30 Jan 2024 23:40:37 -0300 Subject: [PATCH 1/8] Add node type to service offering mapping parameter on createKubernetesCluster API --- .../cluster/KubernetesClusterHelper.java | 5 ++ .../java/com/cloud/vm/VmDetailConstants.java | 2 + .../apache/cloudstack/api/ApiConstants.java | 1 + .../cluster/KubernetesClusterHelperImpl.java | 14 +++ .../cluster/CreateKubernetesClusterCmd.java | 35 ++++++++ .../KubernetesClusterHelperImplTest.java | 45 ++++++++++ .../CreateKubernetesClusterCmdTest.java | 90 +++++++++++++++++++ 7 files changed, 192 insertions(+) create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java create mode 100644 plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java index e160227749d..896bdcfde57 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java @@ -21,6 +21,11 @@ import org.apache.cloudstack.acl.ControlledEntity; public interface KubernetesClusterHelper extends Adapter { + enum KubernetesClusterNodeType { + WORKER, MASTER, ETCD + } + ControlledEntity findByUuid(String uuid); ControlledEntity findByVmId(long vmId); + boolean isValidNodeType(String nodeType); } diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 9338cc11cd4..d4e23659ec3 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -87,6 +87,8 @@ public interface VmDetailConstants { String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; String KEY_PAIR_NAMES = "keypairnames"; String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; + String CKS_NODE_TYPE = "node"; + String OFFERING = "offering"; // VMware to KVM VM migrations specific String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 51523b1863e..ef21916c96d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1036,6 +1036,7 @@ public class ApiConstants { public static final String AUTOSCALING_ENABLED = "autoscalingenabled"; public static final String MIN_SIZE = "minsize"; public static final String MAX_SIZE = "maxsize"; + public static final String NODE_TYPE_OFFERING_MAP = "nodetypeofferingmap"; public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java index 60bd81c7c5a..639e17bd560 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java @@ -22,6 +22,7 @@ import com.cloud.utils.component.AdapterBase; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import javax.inject.Inject; @@ -49,6 +50,19 @@ public class KubernetesClusterHelperImpl extends AdapterBase implements Kubernet return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId()); } + @Override + public boolean isValidNodeType(String nodeType) { + if (StringUtils.isBlank(nodeType)) { + return false; + } + try { + KubernetesClusterNodeType.valueOf(nodeType.toUpperCase()); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + @Override public String getConfigComponentName() { return KubernetesClusterHelper.class.getSimpleName(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 12a50c9e88f..493d9b61707 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -17,10 +17,15 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import java.security.InvalidParameterException; +import java.util.HashMap; +import java.util.Map; import javax.inject.Inject; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesClusterHelper; +import com.cloud.offering.ServiceOffering; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -40,6 +45,7 @@ import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -62,6 +68,8 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Inject public KubernetesClusterService kubernetesClusterService; + @Inject + protected KubernetesClusterHelper kubernetesClusterHelper; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -87,6 +95,11 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "the ID of the service offering for the virtual machines in the cluster.") private Long serviceOfferingId; + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_OFFERING_MAP, type = CommandType.MAP, + description = "(Optional) Node Type to Service Offering ID mapping. If provided, it overrides the serviceofferingid parameter") + protected Map> nodeTypeOfferingMap; + @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the" + " virtual machine. Must be used with domainId.") @@ -244,6 +257,28 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return clusterType; } + public Map getNodeTypeOfferingMap() { + Map mapping = new HashMap<>(); + if (MapUtils.isNotEmpty(nodeTypeOfferingMap)) { + for (Map entry : nodeTypeOfferingMap.values()) { + String nodeTypeStr = entry.get(VmDetailConstants.CKS_NODE_TYPE); + String serviceOfferingUuid = entry.get(VmDetailConstants.OFFERING); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Node Type: '%s' should use Service Offering ID: '%s'", nodeTypeStr, serviceOfferingUuid)); + } + ServiceOffering serviceOffering = _entityMgr.findByUuid(ServiceOffering.class, serviceOfferingUuid); + if (StringUtils.isAnyEmpty(nodeTypeStr, serviceOfferingUuid) || + !kubernetesClusterHelper.isValidNodeType(nodeTypeStr) || + serviceOffering == null) { + throw new InvalidParameterValueException(String.format("Service Offering ID: %s for Node Type: %s is invalid", serviceOfferingUuid, nodeTypeStr)); + } + KubernetesClusterHelper.KubernetesClusterNodeType nodeType = KubernetesClusterHelper.KubernetesClusterNodeType.valueOf(nodeTypeStr.toUpperCase()); + mapping.put(nodeType.name(), serviceOffering.getId()); + } + } + return mapping; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java new file mode 100644 index 00000000000..bab58d20604 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterHelperImplTest { + + private final KubernetesClusterHelperImpl helper = new KubernetesClusterHelperImpl(); + + @Test + public void testIsValidNodeTypeEmptyNodeType() { + Assert.assertFalse(helper.isValidNodeType(null)); + } + + @Test + public void testIsValidNodeTypeInvalidNodeType() { + String nodeType = "invalidNodeType"; + Assert.assertFalse(helper.isValidNodeType(nodeType)); + } + + @Test + public void testIsValidNodeTypeValidNodeTypeLowercase() { + String nodeType = KubernetesClusterHelper.KubernetesClusterNodeType.WORKER.name().toLowerCase(); + Assert.assertTrue(helper.isValidNodeType(nodeType)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java new file mode 100644 index 00000000000..cd61baedfaa --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import com.cloud.kubernetes.cluster.KubernetesClusterHelper; +import com.cloud.kubernetes.cluster.KubernetesClusterHelperImpl; +import com.cloud.offering.ServiceOffering; +import com.cloud.utils.db.EntityManager; +import com.cloud.vm.VmDetailConstants; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.MASTER; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER; + +@RunWith(MockitoJUnitRunner.class) +public class CreateKubernetesClusterCmdTest { + + @Mock + EntityManager entityManager; + KubernetesClusterHelper helper = new KubernetesClusterHelperImpl(); + + @Mock + ServiceOffering workerServiceOffering; + @Mock + ServiceOffering masterServiceOffering; + + private final CreateKubernetesClusterCmd cmd = new CreateKubernetesClusterCmd(); + + private static final String workerNodesOfferingId = UUID.randomUUID().toString(); + private static final String masterNodesOfferingId = UUID.randomUUID().toString(); + private static final Long workerOfferingId = 1L; + private static final Long masterOfferingId = 2L; + + @Before + public void setUp() { + cmd._entityMgr = entityManager; + cmd.kubernetesClusterHelper = helper; + Mockito.when(entityManager.findByUuid(ServiceOffering.class, workerNodesOfferingId)).thenReturn(workerServiceOffering); + Mockito.when(entityManager.findByUuid(ServiceOffering.class, masterNodesOfferingId)).thenReturn(masterServiceOffering); + Mockito.when(workerServiceOffering.getId()).thenReturn(workerOfferingId); + Mockito.when(masterServiceOffering.getId()).thenReturn(masterOfferingId); + } + + private Map createMapEntry(KubernetesClusterHelper.KubernetesClusterNodeType nodeType, + String nodeTypeOfferingUuid) { + Map map = new HashMap<>(); + map.put(VmDetailConstants.CKS_NODE_TYPE, nodeType.name().toLowerCase()); + map.put(VmDetailConstants.OFFERING, nodeTypeOfferingUuid); + return map; + } + + @Test + public void testNodeOfferingMap() { + cmd.nodeTypeOfferingMap = new HashMap<>(); + Map firstMap = createMapEntry(WORKER, workerNodesOfferingId); + Map secondMap = createMapEntry(MASTER, masterNodesOfferingId); + cmd.nodeTypeOfferingMap.put("map1", firstMap); + cmd.nodeTypeOfferingMap.put("map2", secondMap); + Map map = cmd.getNodeTypeOfferingMap(); + Assert.assertNotNull(map); + Assert.assertEquals(2, map.size()); + Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(MASTER.name())); + Assert.assertEquals(workerOfferingId, map.get(WORKER.name())); + Assert.assertEquals(masterOfferingId, map.get(MASTER.name())); + } +} From 8e95ded71c514800f2ebc7cc821a9c159321cf1e Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 31 Jan 2024 08:52:45 -0300 Subject: [PATCH 2/8] Refactor new methods --- .../cluster/CreateKubernetesClusterCmd.java | 58 ++++++++++++++----- .../CreateKubernetesClusterCmdTest.java | 18 ++++++ 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 493d9b61707..9b4067ce0a2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -24,6 +24,7 @@ import javax.inject.Inject; import com.cloud.exception.InvalidParameterValueException; import com.cloud.kubernetes.cluster.KubernetesClusterHelper; +import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType; import com.cloud.offering.ServiceOffering; import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.acl.RoleType; @@ -257,23 +258,54 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return clusterType; } + protected void checkNodeTypeOfferingEntryCompleteness(String nodeTypeStr, String serviceOfferingUuid) { + if (StringUtils.isAnyEmpty(nodeTypeStr, serviceOfferingUuid)) { + String error = String.format("Incomplete Node Type to Service Offering ID mapping: '%s' -> '%s'", nodeTypeStr, serviceOfferingUuid); + LOGGER.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void checkNodeTypeOfferingEntryValues(String nodeTypeStr, ServiceOffering serviceOffering, String serviceOfferingUuid) { + if (!kubernetesClusterHelper.isValidNodeType(nodeTypeStr)) { + String error = String.format("The provided value '%s' for Node Type is invalid", nodeTypeStr); + LOGGER.error(error); + throw new InvalidParameterValueException(String.format(error)); + } + if (serviceOffering == null) { + String error = String.format("Cannot find a service offering with ID %s", serviceOfferingUuid); + LOGGER.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void addNodeTypeOfferingEntry(String nodeTypeStr, String serviceOfferingUuid, ServiceOffering serviceOffering, Map mapping) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Node Type: '%s' should use Service Offering ID: '%s'", nodeTypeStr, serviceOfferingUuid)); + } + KubernetesClusterNodeType nodeType = KubernetesClusterNodeType.valueOf(nodeTypeStr.toUpperCase()); + mapping.put(nodeType.name(), serviceOffering.getId()); + } + + protected void processNodeTypeOfferingEntryAndAddToMappingIfValid(Map entry, Map mapping) { + if (MapUtils.isEmpty(entry)) { + return; + } + String nodeTypeStr = entry.get(VmDetailConstants.CKS_NODE_TYPE); + String serviceOfferingUuid = entry.get(VmDetailConstants.OFFERING); + checkNodeTypeOfferingEntryCompleteness(nodeTypeStr, serviceOfferingUuid); + + ServiceOffering serviceOffering = _entityMgr.findByUuid(ServiceOffering.class, serviceOfferingUuid); + checkNodeTypeOfferingEntryValues(nodeTypeStr, serviceOffering, serviceOfferingUuid); + + addNodeTypeOfferingEntry(nodeTypeStr, serviceOfferingUuid, serviceOffering, mapping); + } + public Map getNodeTypeOfferingMap() { Map mapping = new HashMap<>(); if (MapUtils.isNotEmpty(nodeTypeOfferingMap)) { for (Map entry : nodeTypeOfferingMap.values()) { - String nodeTypeStr = entry.get(VmDetailConstants.CKS_NODE_TYPE); - String serviceOfferingUuid = entry.get(VmDetailConstants.OFFERING); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Node Type: '%s' should use Service Offering ID: '%s'", nodeTypeStr, serviceOfferingUuid)); - } - ServiceOffering serviceOffering = _entityMgr.findByUuid(ServiceOffering.class, serviceOfferingUuid); - if (StringUtils.isAnyEmpty(nodeTypeStr, serviceOfferingUuid) || - !kubernetesClusterHelper.isValidNodeType(nodeTypeStr) || - serviceOffering == null) { - throw new InvalidParameterValueException(String.format("Service Offering ID: %s for Node Type: %s is invalid", serviceOfferingUuid, nodeTypeStr)); - } - KubernetesClusterHelper.KubernetesClusterNodeType nodeType = KubernetesClusterHelper.KubernetesClusterNodeType.valueOf(nodeTypeStr.toUpperCase()); - mapping.put(nodeType.name(), serviceOffering.getId()); + processNodeTypeOfferingEntryAndAddToMappingIfValid(entry, mapping); } } return mapping; diff --git a/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java index cd61baedfaa..25dbfae4d86 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.kubernetes.cluster; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.kubernetes.cluster.KubernetesClusterHelper; import com.cloud.kubernetes.cluster.KubernetesClusterHelperImpl; import com.cloud.offering.ServiceOffering; @@ -87,4 +88,21 @@ public class CreateKubernetesClusterCmdTest { Assert.assertEquals(workerOfferingId, map.get(WORKER.name())); Assert.assertEquals(masterOfferingId, map.get(MASTER.name())); } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryCompletenessInvalidParameters() { + cmd.checkNodeTypeOfferingEntryCompleteness(WORKER.name(), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesInvalidNodeType() { + String invalidNodeType = "invalidNodeTypeName"; + cmd.checkNodeTypeOfferingEntryValues(invalidNodeType, workerServiceOffering, workerNodesOfferingId); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesEmptyOffering() { + String nodeType = WORKER.name(); + cmd.checkNodeTypeOfferingEntryValues(nodeType, null, workerNodesOfferingId); + } } From cda85adc13d26647a9322602b2e52bca597827de Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 31 Jan 2024 10:16:20 -0300 Subject: [PATCH 3/8] Add schema changes --- .../main/resources/META-INF/db/schema-41810to41900.sql | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 26a7d546687..9de5ef5e38a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -360,3 +360,12 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.quarantined_ips', 'remover_account_i -- Explicitly add support for VMware 8.0b (8.0.0.2), 8.0c (8.0.0.3) INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '8.0.0.2', 1024, 0, 59, 64, 1, 1); INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '8.0.0.3', 1024, 0, 59, 64, 1, 1); + +-- Add support for different node types service offerings on CKS clusters +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `control_service_offering_id` bigint unsigned COMMENT 'service offering ID for Control Nodes'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `worker_service_offering_id` bigint unsigned COMMENT 'service offering ID for Worker Nodes'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `etcd_service_offering_id` bigint unsigned COMMENT 'service offering ID for etcd Nodes'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `etcd_node_count` bigint COMMENT 'the number of the etcd Nodes deployed for this Kubernetes cluster'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_service_offering_id` FOREIGN KEY `fk_cluster__control_service_offering_id`(`control_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_service_offering_id` FOREIGN KEY `fk_cluster__worker_service_offering_id`(`worker_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_service_offering_id` FOREIGN KEY `fk_cluster__etcd_service_offering_id`(`etcd_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; From 959f2b6a30bfd8bf4e0ae20a4b6ca63895f7a63f Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 31 Jan 2024 15:35:37 -0300 Subject: [PATCH 4/8] Refactor and add UI for createKubernetesCluster API advanced settings --- .../cluster/KubernetesClusterHelper.java | 2 +- .../apache/cloudstack/api/ApiConstants.java | 3 +- .../cluster/CreateKubernetesClusterCmd.java | 42 +++++++- .../CreateKubernetesClusterCmdTest.java | 55 +++++++++-- ui/public/locales/en.json | 4 + .../views/compute/CreateKubernetesCluster.vue | 97 +++++++++++++++++++ 6 files changed, 190 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java index 896bdcfde57..4b61fe5ad03 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java @@ -22,7 +22,7 @@ import org.apache.cloudstack.acl.ControlledEntity; public interface KubernetesClusterHelper extends Adapter { enum KubernetesClusterNodeType { - WORKER, MASTER, ETCD + CONTROL, WORKER, ETCD } ControlledEntity findByUuid(String uuid); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index ef21916c96d..350d8742fb2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1028,6 +1028,7 @@ public class ApiConstants { public static final String MASTER_NODES = "masternodes"; public static final String NODE_IDS = "nodeids"; public static final String CONTROL_NODES = "controlnodes"; + public static final String ETCD_NODES = "etcdnodes"; public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; @@ -1036,7 +1037,7 @@ public class ApiConstants { public static final String AUTOSCALING_ENABLED = "autoscalingenabled"; public static final String MIN_SIZE = "minsize"; public static final String MAX_SIZE = "maxsize"; - public static final String NODE_TYPE_OFFERING_MAP = "nodetypeofferingmap"; + public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings"; public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 9b4067ce0a2..99943408fc6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -17,7 +17,9 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import java.security.InvalidParameterException; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.inject.Inject; @@ -94,13 +96,19 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, description = "the ID of the service offering for the virtual machines in the cluster.") - private Long serviceOfferingId; + protected Long serviceOfferingId; @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.NODE_TYPE_OFFERING_MAP, type = CommandType.MAP, description = "(Optional) Node Type to Service Offering ID mapping. If provided, it overrides the serviceofferingid parameter") protected Map> nodeTypeOfferingMap; + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.ETCD_NODES, type = CommandType.LONG, + description = "(Optional) Number of Kubernetes cluster etcd nodes, default is 0." + + "In case the number is greater than 0, etcd nodes are separate from master nodes and are provisioned accordingly") + protected Long etcdNodes; + @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the" + " virtual machine. Must be used with domainId.") @@ -220,6 +228,10 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return controlNodes; } + public Long getEtcdNodes() { + return etcdNodes == null ? 0L : etcdNodes; + } + public String getExternalLoadBalancerIpAddress() { return externalLoadBalancerIpAddress; } @@ -307,10 +319,38 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { for (Map entry : nodeTypeOfferingMap.values()) { processNodeTypeOfferingEntryAndAddToMappingIfValid(entry, mapping); } + addMissingNodeTypeDefaultOffering(mapping, serviceOfferingId, etcdNodes); + } else { + addDefaultNodeTypeOfferingEntries(serviceOfferingId, etcdNodes, mapping); } return mapping; } + private void addMissingNodeTypeDefaultOffering(Map mapping, Long serviceOfferingId, Long etcdNodes) { + if (MapUtils.isEmpty(mapping)) { + return; + } + boolean addEtcdOffering = etcdNodes != null && etcdNodes > 0; + List keys = Arrays.asList(KubernetesClusterNodeType.CONTROL.name(), KubernetesClusterNodeType.WORKER.name(), KubernetesClusterNodeType.ETCD.name()); + for (String key : keys) { + if (mapping.containsKey(key)) { + continue; + } + if (!key.equalsIgnoreCase(KubernetesClusterNodeType.ETCD.name()) || + (addEtcdOffering && key.equalsIgnoreCase(KubernetesClusterNodeType.ETCD.name()))) { + mapping.put(key, serviceOfferingId); + } + } + } + + protected void addDefaultNodeTypeOfferingEntries(Long serviceOfferingId, Long etcdNodes, Map mapping) { + mapping.put(KubernetesClusterNodeType.CONTROL.name(), serviceOfferingId); + mapping.put(KubernetesClusterNodeType.WORKER.name(), serviceOfferingId); + if (etcdNodes != null && etcdNodes > 0) { + mapping.put(KubernetesClusterNodeType.ETCD.name(), serviceOfferingId); + } + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java index 25dbfae4d86..406516581a7 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmdTest.java @@ -34,7 +34,8 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; -import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.MASTER; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD; import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER; @RunWith(MockitoJUnitRunner.class) @@ -47,23 +48,29 @@ public class CreateKubernetesClusterCmdTest { @Mock ServiceOffering workerServiceOffering; @Mock - ServiceOffering masterServiceOffering; + ServiceOffering controlServiceOffering; + @Mock + ServiceOffering etcdServiceOffering; private final CreateKubernetesClusterCmd cmd = new CreateKubernetesClusterCmd(); private static final String workerNodesOfferingId = UUID.randomUUID().toString(); - private static final String masterNodesOfferingId = UUID.randomUUID().toString(); + private static final String controlNodesOfferingId = UUID.randomUUID().toString(); + private static final String etcdNodesOfferingId = UUID.randomUUID().toString(); private static final Long workerOfferingId = 1L; - private static final Long masterOfferingId = 2L; + private static final Long controlOfferingId = 2L; + private static final Long etcdOfferingId = 3L; @Before public void setUp() { cmd._entityMgr = entityManager; cmd.kubernetesClusterHelper = helper; Mockito.when(entityManager.findByUuid(ServiceOffering.class, workerNodesOfferingId)).thenReturn(workerServiceOffering); - Mockito.when(entityManager.findByUuid(ServiceOffering.class, masterNodesOfferingId)).thenReturn(masterServiceOffering); + Mockito.when(entityManager.findByUuid(ServiceOffering.class, controlNodesOfferingId)).thenReturn(controlServiceOffering); + Mockito.when(entityManager.findByUuid(ServiceOffering.class, etcdNodesOfferingId)).thenReturn(etcdServiceOffering); Mockito.when(workerServiceOffering.getId()).thenReturn(workerOfferingId); - Mockito.when(masterServiceOffering.getId()).thenReturn(masterOfferingId); + Mockito.when(controlServiceOffering.getId()).thenReturn(controlOfferingId); + Mockito.when(etcdServiceOffering.getId()).thenReturn(etcdOfferingId); } private Map createMapEntry(KubernetesClusterHelper.KubernetesClusterNodeType nodeType, @@ -75,18 +82,46 @@ public class CreateKubernetesClusterCmdTest { } @Test - public void testNodeOfferingMap() { + public void testNodeOfferingMapMissingEtcd() { cmd.nodeTypeOfferingMap = new HashMap<>(); Map firstMap = createMapEntry(WORKER, workerNodesOfferingId); - Map secondMap = createMapEntry(MASTER, masterNodesOfferingId); + Map secondMap = createMapEntry(CONTROL, controlNodesOfferingId); cmd.nodeTypeOfferingMap.put("map1", firstMap); cmd.nodeTypeOfferingMap.put("map2", secondMap); Map map = cmd.getNodeTypeOfferingMap(); Assert.assertNotNull(map); Assert.assertEquals(2, map.size()); - Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(MASTER.name())); + Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(CONTROL.name())); Assert.assertEquals(workerOfferingId, map.get(WORKER.name())); - Assert.assertEquals(masterOfferingId, map.get(MASTER.name())); + Assert.assertEquals(controlOfferingId, map.get(CONTROL.name())); + } + + @Test + public void testNodeOfferingMapNullMap() { + cmd.nodeTypeOfferingMap = null; + cmd.serviceOfferingId = controlOfferingId; + Map map = cmd.getNodeTypeOfferingMap(); + Assert.assertNotNull(map); + Assert.assertEquals(2, map.size()); + Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(CONTROL.name())); + Assert.assertEquals(controlOfferingId, map.get(WORKER.name())); + Assert.assertEquals(controlOfferingId, map.get(CONTROL.name())); + } + + @Test + public void testNodeOfferingMapEtcdNodes() { + cmd.nodeTypeOfferingMap = new HashMap<>(); + Map firstMap = createMapEntry(ETCD, etcdNodesOfferingId); + cmd.nodeTypeOfferingMap.put("map1", firstMap); + cmd.etcdNodes = 2L; + cmd.serviceOfferingId = controlOfferingId; + Map map = cmd.getNodeTypeOfferingMap(); + Assert.assertNotNull(map); + Assert.assertEquals(3, map.size()); + Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(CONTROL.name()) && map.containsKey(ETCD.name())); + Assert.assertEquals(controlOfferingId, map.get(WORKER.name())); + Assert.assertEquals(controlOfferingId, map.get(CONTROL.name())); + Assert.assertEquals(etcdOfferingId, map.get(ETCD.name())); } @Test(expected = InvalidParameterValueException.class) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 6098bec5346..2498d5688dc 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -451,9 +451,13 @@ "label.cisco.nexus1000v.password": "Nexus 1000v password", "label.cisco.nexus1000v.username": "Nexus 1000v username", "label.cks.cluster.autoscalingenabled": "Enable auto scaling on this cluster", +"label.cks.cluster.control.nodes.offeringid": "Service Offering for Control Nodes", +"label.cks.cluster.etcd.nodes": "Etcd Nodes", +"label.cks.cluster.etcd.nodes.offeringid": "Service Offering for etcd Nodes", "label.cks.cluster.maxsize": "Maximum cluster size (Worker nodes)", "label.cks.cluster.minsize": "Minimum cluster size (Worker nodes)", "label.cks.cluster.size": "Cluster size (Worker nodes)", +"label.cks.cluster.worker.nodes.offeringid": "Service Offering for Worker Nodes", "label.cleanup": "Clean up", "label.clear": "Clear", "label.clear.list": "Clear list", diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index ca1e424cca4..214b3022482 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -180,6 +180,81 @@ + + + + + + + + + + + {{ opt.name || opt.description }} + + + + + + + + {{ opt.name || opt.description }} + + + + + + + + + + + + {{ opt.name || opt.description }} + + + + +