diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java index e160227749d..548a016c1c5 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java @@ -19,8 +19,16 @@ package com.cloud.kubernetes.cluster; import com.cloud.utils.component.Adapter; import org.apache.cloudstack.acl.ControlledEntity; +import java.util.Map; + public interface KubernetesClusterHelper extends Adapter { + enum KubernetesClusterNodeType { + CONTROL, WORKER, ETCD, ALL + } + ControlledEntity findByUuid(String uuid); ControlledEntity findByVmId(long vmId); + boolean isValidNodeType(String nodeType); + Map getServiceOfferingNodeTypeMap(Map> serviceOfferingNodeTypeMap); } diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 9338cc11cd4..d4e23659ec3 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -87,6 +87,8 @@ public interface VmDetailConstants { String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; String KEY_PAIR_NAMES = "keypairnames"; String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; + String CKS_NODE_TYPE = "node"; + String OFFERING = "offering"; // VMware to KVM VM migrations specific String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 0148ad904b1..d0380bcdefd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -488,6 +488,12 @@ public class ApiConstants { public static final String VLAN = "vlan"; public static final String VLAN_RANGE = "vlanrange"; + public static final String WORKER_SERVICE_OFFERING_ID = "workerofferingid"; + public static final String WORKER_SERVICE_OFFERING_NAME = "workerofferingname"; + public static final String CONTROL_SERVICE_OFFERING_ID = "controlofferingid"; + public static final String CONTROL_SERVICE_OFFERING_NAME = "controlofferingname"; + public static final String ETCD_SERVICE_OFFERING_ID = "etcdofferingid"; + public static final String ETCD_SERVICE_OFFERING_NAME = "etcdofferingname"; public static final String REMOVE_VLAN = "removevlan"; public static final String VLAN_ID = "vlanid"; public static final String ISOLATED_PVLAN = "isolatedpvlan"; @@ -1029,6 +1035,7 @@ public class ApiConstants { public static final String MASTER_NODES = "masternodes"; public static final String NODE_IDS = "nodeids"; public static final String CONTROL_NODES = "controlnodes"; + public static final String ETCD_NODES = "etcdnodes"; public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; @@ -1037,6 +1044,7 @@ public class ApiConstants { public static final String AUTOSCALING_ENABLED = "autoscalingenabled"; public static final String MIN_SIZE = "minsize"; public static final String MAX_SIZE = "maxsize"; + public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings"; public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 39b6b8865ef..6330974112d 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -363,3 +363,12 @@ INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hyp -- Add for_cks column to the vm_template table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the template can be used for CKS cluster deployment"'); + +-- Add support for different node types service offerings on CKS clusters +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `control_service_offering_id` bigint unsigned COMMENT 'service offering ID for Control Nodes'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `worker_service_offering_id` bigint unsigned COMMENT 'service offering ID for Worker Nodes'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `etcd_service_offering_id` bigint unsigned COMMENT 'service offering ID for etcd Nodes'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `etcd_node_count` bigint COMMENT 'the number of the etcd Nodes deployed for this Kubernetes cluster'; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_service_offering_id` FOREIGN KEY `fk_cluster__control_service_offering_id`(`control_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_service_offering_id` FOREIGN KEY `fk_cluster__worker_service_offering_id`(`worker_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_service_offering_id` FOREIGN KEY `fk_cluster__etcd_service_offering_id`(`etcd_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 591da077aec..b83a3b73706 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -142,4 +142,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm Long getMaxSize(); Long getSecurityGroupId(); ClusterType getClusterType(); + Long getControlServiceOfferingId(); + Long getWorkerServiceOfferingId(); + Long getEtcdServiceOfferingId(); + Long getEtcdNodeCount(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java index 60bd81c7c5a..43802c91e41 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java @@ -16,24 +16,37 @@ // under the License. package com.cloud.kubernetes.cluster; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; @Component public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable { + public static final Logger LOGGER = Logger.getLogger(KubernetesClusterHelperImpl.class.getName()); + @Inject private KubernetesClusterDao kubernetesClusterDao; @Inject private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Inject + protected ServiceOfferingDao serviceOfferingDao; @Override public ControlledEntity findByUuid(String uuid) { @@ -49,6 +62,73 @@ public class KubernetesClusterHelperImpl extends AdapterBase implements Kubernet return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId()); } + @Override + public boolean isValidNodeType(String nodeType) { + if (StringUtils.isBlank(nodeType)) { + return false; + } + try { + KubernetesClusterNodeType.valueOf(nodeType.toUpperCase()); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + protected void checkNodeTypeOfferingEntryCompleteness(String nodeTypeStr, String serviceOfferingUuid) { + if (StringUtils.isAnyEmpty(nodeTypeStr, serviceOfferingUuid)) { + String error = String.format("Incomplete Node Type to Service Offering ID mapping: '%s' -> '%s'", nodeTypeStr, serviceOfferingUuid); + LOGGER.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void checkNodeTypeOfferingEntryValues(String nodeTypeStr, ServiceOffering serviceOffering, String serviceOfferingUuid) { + if (!isValidNodeType(nodeTypeStr)) { + String error = String.format("The provided value '%s' for Node Type is invalid", nodeTypeStr); + LOGGER.error(error); + throw new InvalidParameterValueException(String.format(error)); + } + if (serviceOffering == null) { + String error = String.format("Cannot find a service offering with ID %s", serviceOfferingUuid); + LOGGER.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void addNodeTypeOfferingEntry(String nodeTypeStr, String serviceOfferingUuid, ServiceOffering serviceOffering, Map mapping) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Node Type: '%s' should use Service Offering ID: '%s'", nodeTypeStr, serviceOfferingUuid)); + } + KubernetesClusterNodeType nodeType = KubernetesClusterNodeType.valueOf(nodeTypeStr.toUpperCase()); + mapping.put(nodeType.name(), serviceOffering.getId()); + } + + protected void processNodeTypeOfferingEntryAndAddToMappingIfValid(Map entry, Map mapping) { + if (MapUtils.isEmpty(entry)) { + return; + } + String nodeTypeStr = entry.get(VmDetailConstants.CKS_NODE_TYPE); + String serviceOfferingUuid = entry.get(VmDetailConstants.OFFERING); + checkNodeTypeOfferingEntryCompleteness(nodeTypeStr, serviceOfferingUuid); + + ServiceOffering serviceOffering = serviceOfferingDao.findByUuid(serviceOfferingUuid); + checkNodeTypeOfferingEntryValues(nodeTypeStr, serviceOffering, serviceOfferingUuid); + + addNodeTypeOfferingEntry(nodeTypeStr, serviceOfferingUuid, serviceOffering, mapping); + } + + @Override + public Map getServiceOfferingNodeTypeMap(Map> serviceOfferingNodeTypeMap) { + Map mapping = new HashMap<>(); + if (MapUtils.isNotEmpty(serviceOfferingNodeTypeMap)) { + for (Map entry : serviceOfferingNodeTypeMap.values()) { + processNodeTypeOfferingEntryAndAddToMappingIfValid(entry, mapping); + } + } + return mapping; + } + @Override public String getConfigComponentName() { return KubernetesClusterHelper.class.getSimpleName(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index fcb208c6718..47cedd996a2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -16,6 +16,10 @@ // under the License. package com.cloud.kubernetes.cluster; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ALL; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import static com.cloud.vm.UserVmManager.AllowUserExpungeRecoverVm; @@ -40,6 +44,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType; import com.cloud.uservm.UserVm; import com.cloud.vm.UserVmService; import org.apache.cloudstack.acl.ControlledEntity; @@ -72,6 +77,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -192,6 +198,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); + protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER.name(), CONTROL.name(), ETCD.name()); + ScheduledExecutorService _gcExecutor; ScheduledExecutorService _stateScanner; @@ -446,7 +454,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne validateIsolatedNetwork(network, clusterTotalNodeCount); } - private boolean validateServiceOffering(final ServiceOffering serviceOffering, final KubernetesSupportedVersion version) { + protected void validateServiceOffering(final ServiceOffering serviceOffering, final KubernetesSupportedVersion version) throws InvalidParameterValueException { if (serviceOffering.isDynamic()) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); } @@ -459,7 +467,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (serviceOffering.getRamSize() < version.getMinimumRamSize()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d MB RAM", serviceOffering.getUuid(), version.getUuid(), version.getMinimumRamSize())); } - return true; } private void validateDockerRegistryParams(final String dockerRegistryUserName, @@ -545,6 +552,33 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); } + protected void setNodeTypeServiceOfferingResponse(KubernetesClusterResponse response, + KubernetesClusterNodeType nodeType, + Long offeringId) { + if (offeringId == null) { + return; + } + ServiceOfferingVO offering = serviceOfferingDao.findById(offeringId); + if (offering != null) { + setServiceOfferingResponseForNodeType(response, offering, nodeType); + } + } + + protected void setServiceOfferingResponseForNodeType(KubernetesClusterResponse response, + ServiceOfferingVO offering, + KubernetesClusterNodeType nodeType) { + if (CONTROL == nodeType) { + response.setControlOfferingId(offering.getUuid()); + response.setControlOfferingName(offering.getName()); + } else if (WORKER == nodeType) { + response.setWorkerOfferingId(offering.getUuid()); + response.setWorkerOfferingName(offering.getName()); + } else if (ETCD == nodeType) { + response.setEtcdOfferingId(offering.getUuid()); + response.setEtcdOfferingName(offering.getName()); + } + } + @Override public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); @@ -568,6 +602,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne response.setServiceOfferingId(offering.getUuid()); response.setServiceOfferingName(offering.getName()); } + + setNodeTypeServiceOfferingResponse(response, WORKER, kubernetesCluster.getWorkerServiceOfferingId()); + setNodeTypeServiceOfferingResponse(response, CONTROL, kubernetesCluster.getControlServiceOfferingId()); + setNodeTypeServiceOfferingResponse(response, ETCD, kubernetesCluster.getEtcdServiceOfferingId()); + + if (kubernetesCluster.getEtcdNodeCount() != null) { + response.setEtcdNodes(kubernetesCluster.getEtcdNodeCount()); + } KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (version != null) { response.setKubernetesVersionId(version.getUuid()); @@ -736,7 +778,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final String name = cmd.getName(); final Long zoneId = cmd.getZoneId(); final Long kubernetesVersionId = cmd.getKubernetesVersionId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final Long networkId = cmd.getNetworkId(); final String sshKeyPair = cmd.getSSHKeyPairName(); @@ -747,6 +788,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); + final Map serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap(); + final Long defaultServiceOfferingId = cmd.getServiceOfferingId(); if (name == null || name.isEmpty()) { throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name: " + name); @@ -804,10 +847,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state for datacenter ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); } - ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("No service offering with ID: " + serviceOfferingId); - } + validateServiceOfferingsForNodeTypes(serviceOfferingNodeTypeMap, defaultServiceOfferingId, cmd.getEtcdNodes(), clusterKubernetesVersion); validateSshKeyPairForKubernetesCreateParameters(sshKeyPair, owner); @@ -815,10 +855,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InvalidParameterValueException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); } - if (!validateServiceOffering(serviceOffering, clusterKubernetesVersion)) { - throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); - } - validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl); Network network = validateAndGetNetworkForKubernetesCreateParameters(networkId); @@ -840,6 +876,37 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } + protected void validateServiceOfferingsForNodeTypes(Map map, + Long defaultServiceOfferingId, + Long etcdNodes, + KubernetesSupportedVersion clusterKubernetesVersion) { + for (String key : CLUSTER_NODES_TYPES_LIST) { + validateServiceOfferingForNode(map, defaultServiceOfferingId, key, etcdNodes, clusterKubernetesVersion); + } + } + + protected void validateServiceOfferingForNode(Map map, + Long defaultServiceOfferingId, + String key, Long etcdNodes, + KubernetesSupportedVersion clusterKubernetesVersion) { + if (ETCD.name().equalsIgnoreCase(key) && (etcdNodes == null || etcdNodes == 0)) { + return; + } + Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId); + ServiceOffering serviceOffering = serviceOfferingId != null ? serviceOfferingDao.findById(serviceOfferingId) : null; + if (serviceOffering == null) { + throw new InvalidParameterValueException("No service offering found with ID: " + serviceOfferingId); + } + try { + validateServiceOffering(serviceOffering, clusterKubernetesVersion); + } catch (InvalidParameterValueException e) { + String msg = String.format("Given service offering ID: %s for %s nodes is not suitable for the Kubernetes cluster version %s - %s", + serviceOffering, key, clusterKubernetesVersion, e.getMessage()); + LOGGER.error(msg); + throw new InvalidParameterValueException(msg); + } + } + private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int controlNodesCount, final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws CloudRuntimeException { Network network = null; @@ -943,12 +1010,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { final Long kubernetesClusterId = cmd.getId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); final Long clusterSize = cmd.getClusterSize(); final List nodeIds = cmd.getNodeIds(); final Boolean isAutoscalingEnabled = cmd.isAutoscalingEnabled(); final Long minSize = cmd.getMinSize(); final Long maxSize = cmd.getMaxSize(); + final Long defaultServiceOfferingId = cmd.getServiceOfferingId(); + final Map serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap(); if (kubernetesClusterId == null || kubernetesClusterId < 1L) { throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); @@ -964,7 +1032,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster : %s", kubernetesCluster.getName())); } - if (serviceOfferingId == null && clusterSize == null && nodeIds == null && isAutoscalingEnabled == null) { + if (defaultServiceOfferingId == null && isAnyNodeOfferingEmpty(serviceOfferingNodeTypeMap) + && clusterSize == null && nodeIds == null && isAutoscalingEnabled == null) { throw new InvalidParameterValueException(String.format("Kubernetes cluster %s cannot be scaled, either service offering or cluster size or nodeids to be removed or autoscaling must be passed", kubernetesCluster.getName())); } @@ -1011,8 +1080,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } + Long workerOfferingId = serviceOfferingNodeTypeMap != null ? serviceOfferingNodeTypeMap.getOrDefault(WORKER.name(), null) : null; if (nodeIds != null) { - if (clusterSize != null || serviceOfferingId != null) { + if (clusterSize != null || defaultServiceOfferingId != null || workerOfferingId != null) { throw new InvalidParameterValueException("nodeids can not be passed along with clustersize or service offering"); } List nodes = kubernetesClusterVmMapDao.listByClusterIdAndVmIdsIn(kubernetesCluster.getId(), nodeIds); @@ -1032,39 +1102,55 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } - ServiceOffering serviceOffering = null; - if (serviceOfferingId != null) { - serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); - } else { - if (serviceOffering.isDynamic()) { - throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster : %s, service offering : %s", kubernetesCluster.getName(), serviceOffering.getName())); - } - if (serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", - kubernetesCluster.getName(), serviceOffering.getName(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); - } - if (serviceOffering.getCpu() < clusterVersion.getMinimumCpu()) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d vCPUs", - kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumCpu())); - } - if (serviceOffering.getRamSize() < clusterVersion.getMinimumRamSize()) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d MB RAM", - kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumRamSize())); - } - } - final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || - serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed())) { - logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering : %s offers lesser resources as compared to service offering : %s of Kubernetes cluster : %s", - serviceOffering.getName(), existingServiceOffering.getName(), kubernetesCluster.getName())); - } - } + validateServiceOfferingsForNodeTypesScale(serviceOfferingNodeTypeMap, defaultServiceOfferingId, kubernetesCluster, clusterVersion); validateKubernetesClusterScaleSize(kubernetesCluster, clusterSize, maxClusterSize, zone); } + protected void validateServiceOfferingsForNodeTypesScale(Map map, Long defaultServiceOfferingId, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) { + for (String key : CLUSTER_NODES_TYPES_LIST) { + Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId); + if (serviceOfferingId != null) { + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); + } + checkServiceOfferingForNodesScale(serviceOffering, kubernetesCluster, clusterVersion); + final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || + serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed())) { + logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering : %s offers lesser resources as compared to service offering : %s of Kubernetes cluster : %s", + serviceOffering.getName(), existingServiceOffering.getName(), kubernetesCluster.getName())); + } + } + } + } + + protected void checkServiceOfferingForNodesScale(ServiceOffering serviceOffering, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) { + if (serviceOffering.isDynamic()) { + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster : %s, service offering : %s", kubernetesCluster.getName(), serviceOffering.getName())); + } + if (serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", + kubernetesCluster.getName(), serviceOffering.getName(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); + } + if (serviceOffering.getCpu() < clusterVersion.getMinimumCpu()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d vCPUs", + kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumCpu())); + } + if (serviceOffering.getRamSize() < clusterVersion.getMinimumRamSize()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d MB RAM", + kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumRamSize())); + } + } + + protected boolean isAnyNodeOfferingEmpty(Map map) { + if (MapUtils.isEmpty(map)) { + return false; + } + return map.values().stream().anyMatch(Objects::isNull); + } + private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { // Validate parameters validateEndpointUrl(); @@ -1154,6 +1240,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final long controlNodeCount = cmd.getControlNodes(); final long clusterSize = Objects.requireNonNullElse(cmd.getClusterSize(), 0L); final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); + Map nodeTypeOfferingMap = cmd.getServiceOfferingNodeTypeMap(); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); @@ -1203,20 +1290,15 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final DataCenter zone = dataCenterDao.findById(cmd.getZoneId()); final long controlNodeCount = cmd.getControlNodes(); final long clusterSize = cmd.getClusterSize(); - final long totalNodeCount = controlNodeCount + clusterSize; - final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); + final long etcdNodes = cmd.getEtcdNodes(); + final Map nodeTypeCount = Map.of(WORKER.name(), clusterSize, + CONTROL.name(), controlNodeCount, ETCD.name(), etcdNodes); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); - DeployDestination deployDestination = null; - try { - deployDestination = plan(totalNodeCount, zone, serviceOffering); - } catch (InsufficientCapacityException e) { - logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d nodes cluster in zone : %s with service offering : %s", totalNodeCount, zone.getName(), serviceOffering.getName())); - } - if (deployDestination == null || deployDestination.getCluster() == null) { - logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone : %s", zone.getName())); - } + Map serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap(); + Long defaultServiceOfferingId = cmd.getServiceOfferingId(); + Hypervisor.HypervisorType hypervisorType = getHypervisorTypeAndValidateNodeDeployments(serviceOfferingNodeTypeMap, defaultServiceOfferingId, nodeTypeCount, zone); SecurityGroup securityGroup = null; if (zone.isSecurityGroupEnabled()) { @@ -1224,9 +1306,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); - final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType()); - final long cores = serviceOffering.getCpu() * (controlNodeCount + clusterSize); - final long memory = serviceOffering.getRamSize() * (controlNodeCount + clusterSize); + final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, hypervisorType); + // Set the service_offering_id as the ID of the worker nodes offering for backwards compatibility + final ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingNodeTypeMap.getOrDefault(WORKER.name(), defaultServiceOfferingId)); + Pair capacityPair = calculateClusterCapacity(serviceOfferingNodeTypeMap, nodeTypeCount); + final long cores = capacityPair.first(); + final long memory = capacityPair.second(); final SecurityGroup finalSecurityGroup = securityGroup; final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { @@ -1236,6 +1321,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne serviceOffering.getId(), finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), owner.getAccountId(), controlNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), "", KubernetesCluster.ClusterType.CloudManaged); + if (serviceOfferingNodeTypeMap.containsKey(WORKER.name())) { + newCluster.setWorkerServiceOfferingId(serviceOfferingNodeTypeMap.get(WORKER.name())); + } + if (serviceOfferingNodeTypeMap.containsKey(CONTROL.name())) { + newCluster.setControlServiceOfferingId(serviceOfferingNodeTypeMap.get(CONTROL.name())); + } + if (etcdNodes > 0 && serviceOfferingNodeTypeMap.containsKey(ETCD.name())) { + newCluster.setEtcdNodeCount(etcdNodes); + newCluster.setEtcdServiceOfferingId(serviceOfferingNodeTypeMap.get(ETCD.name())); + } if (zone.isSecurityGroupEnabled()) { newCluster.setSecurityGroupId(finalSecurityGroup.getId()); } @@ -1252,6 +1347,48 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return cluster; } + protected Pair calculateClusterCapacity(Map map, Map nodeTypeCount) { + long cores = 0L; + long memory = 0L; + for (String key : CLUSTER_NODES_TYPES_LIST) { + if (!map.containsKey(key)) { + continue; + } + ServiceOffering serviceOffering = serviceOfferingDao.findById(map.get(key)); + Long nodes = nodeTypeCount.get(key); + cores = cores + (serviceOffering.getCpu() * nodes); + memory = memory + (serviceOffering.getRamSize() * nodes); + } + return new Pair<>(cores, memory); + } + + protected Hypervisor.HypervisorType getHypervisorTypeAndValidateNodeDeployments(Map serviceOfferingNodeTypeMap, + Long defaultServiceOfferingId, + Map nodeTypeCount, DataCenter zone) { + Hypervisor.HypervisorType hypervisorType = null; + for (String nodeType : CLUSTER_NODES_TYPES_LIST) { + ServiceOffering serviceOffering = null; + Long nodes = nodeTypeCount.getOrDefault(nodeType, defaultServiceOfferingId); + try { + if (nodeType.equalsIgnoreCase(ETCD.name()) && + (!serviceOfferingNodeTypeMap.containsKey(ETCD.name()) || nodes == 0)) { + continue; + } + serviceOffering = serviceOfferingDao.findById(serviceOfferingNodeTypeMap.get(nodeType)); + DeployDestination deployDestination = plan(nodes, zone, serviceOffering); + if (deployDestination.getCluster() == null) { + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone : %s", zone.getName())); + } + if (hypervisorType == null) { + hypervisorType = deployDestination.getCluster().getHypervisorType(); + } + } catch (InsufficientCapacityException e) { + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d nodes cluster in zone : %s with service offering : %s", nodes, zone.getName(), serviceOffering.getName())); + } + } + return hypervisorType; + } + private SecurityGroup getOrCreateSecurityGroupForAccount(Account owner) { String securityGroupName = String.format("%s-%s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME, owner.getUuid()); String securityGroupDesc = String.format("%s and account %s", KubernetesClusterActionWorker.CKS_SECURITY_GROUP_DESCRIPTION, owner.getName()); @@ -1536,12 +1673,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } validateKubernetesClusterScaleParameters(cmd); + Map nodeToOfferingMap = createNodeTypeToServiceOfferingMap(cmd.getServiceOfferingNodeTypeMap(), cmd.getServiceOfferingId()); KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); String[] keys = getServiceUserKeys(kubernetesCluster); KubernetesClusterScaleWorker scaleWorker = new KubernetesClusterScaleWorker(kubernetesClusterDao.findById(cmd.getId()), serviceOfferingDao.findById(cmd.getServiceOfferingId()), + nodeToOfferingMap, cmd.getClusterSize(), cmd.getNodeIds(), cmd.isAutoscalingEnabled(), @@ -1553,6 +1692,22 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return scaleWorker.scaleCluster(); } + protected Map createNodeTypeToServiceOfferingMap(Map idsMapping, + Long serviceOfferingId) { + Map map = new HashMap<>(); + if (MapUtils.isEmpty(idsMapping) && serviceOfferingId != null) { + map.put(ALL.name(), serviceOfferingDao.findById(serviceOfferingId)); + return map; + } + for (String key : CLUSTER_NODES_TYPES_LIST) { + if (!idsMapping.containsKey(key)) { + continue; + } + map.put(key, serviceOfferingDao.findById(idsMapping.get(key))); + } + return map; + } + @Override public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws CloudRuntimeException { if (!KubernetesServiceEnabled.value()) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 270916aab7e..deeca36e389 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -117,6 +117,18 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "cluster_type") private ClusterType clusterType; + @Column(name = "control_service_offering_id") + private Long controlServiceOfferingId; + + @Column(name = "worker_service_offering_id") + private Long workerServiceOfferingId; + + @Column(name = "etcd_service_offering_id") + private Long etcdServiceOfferingId; + + @Column(name = "etcd_node_count") + private Long etcdNodeCount; + @Override public long getId() { return id; @@ -406,4 +418,36 @@ public class KubernetesClusterVO implements KubernetesCluster { public Class getEntityType() { return KubernetesCluster.class; } + + public Long getControlServiceOfferingId() { + return controlServiceOfferingId; + } + + public void setControlServiceOfferingId(Long controlServiceOfferingId) { + this.controlServiceOfferingId = controlServiceOfferingId; + } + + public Long getWorkerServiceOfferingId() { + return workerServiceOfferingId; + } + + public void setWorkerServiceOfferingId(Long workerServiceOfferingId) { + this.workerServiceOfferingId = workerServiceOfferingId; + } + + public Long getEtcdServiceOfferingId() { + return etcdServiceOfferingId; + } + + public void setEtcdServiceOfferingId(Long etcdServiceOfferingId) { + this.etcdServiceOfferingId = etcdServiceOfferingId; + } + + public Long getEtcdNodeCount() { + return etcdNodeCount; + } + + public void setEtcdNodeCount(Long etcdNodeCount) { + this.etcdNodeCount = etcdNodeCount; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index a84320e4d7f..5b6fa94a2c3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -31,6 +31,8 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType; +import com.cloud.offering.ServiceOffering; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.config.ApiServiceConfiguration; @@ -689,4 +691,27 @@ public class KubernetesClusterActionWorker { public void setKeys(String[] keys) { this.keys = keys; } + + protected ServiceOffering getServiceOfferingForNodeTypeOnCluster(KubernetesClusterNodeType nodeType, + KubernetesCluster cluster) { + Long offeringId = null; + Long defaultOfferingId = cluster.getServiceOfferingId(); + Long controlOfferingId = cluster.getControlServiceOfferingId(); + Long workerOfferingId = cluster.getWorkerServiceOfferingId(); + Long etcdOfferingId = cluster.getEtcdServiceOfferingId(); + if (KubernetesClusterNodeType.CONTROL == nodeType) { + offeringId = controlOfferingId != null ? controlOfferingId : defaultOfferingId; + } else if (KubernetesClusterNodeType.WORKER == nodeType) { + offeringId = workerOfferingId != null ? workerOfferingId : defaultOfferingId; + } else if (KubernetesClusterNodeType.ETCD == nodeType && cluster.getEtcdNodeCount() != null && cluster.getEtcdNodeCount() > 0) { + offeringId = etcdOfferingId != null ? etcdOfferingId : defaultOfferingId; + } + + if (offeringId == null) { + String msg = String.format("Cannot find a service offering for the %s nodes on the Kubernetes cluster %s", nodeType.name(), cluster.getName()); + LOGGER.error(msg); + throw new CloudRuntimeException(msg); + } + return serviceOfferingDao.findById(offeringId); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 08a1ad3452d..c0e38287639 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -17,6 +17,7 @@ package com.cloud.kubernetes.cluster.actionworkers; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.io.File; @@ -376,7 +377,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu ResourceUnavailableException, InsufficientCapacityException { UserVm nodeVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(WORKER, kubernetesCluster); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Account owner = accountDao.findById(kubernetesCluster.getAccountId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index df94642a881..1468b1cca5f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import javax.inject.Inject; @@ -64,6 +65,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif protected VMInstanceDao vmInstanceDao; private ServiceOffering serviceOffering; + private Map serviceOfferingNodeTypeMap; private Long clusterSize; private List nodeIds; private KubernetesCluster.State originalState; @@ -75,6 +77,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif public KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final ServiceOffering serviceOffering, + final Map serviceOfferingNodeTypeMap, final Long clusterSize, final List nodeIds, final Boolean isAutoscalingEnabled, @@ -83,6 +86,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); this.serviceOffering = serviceOffering; + this.serviceOfferingNodeTypeMap = serviceOfferingNodeTypeMap; this.nodeIds = nodeIds; this.isAutoscalingEnabled = isAutoscalingEnabled; this.minSize = minSize; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 84ad9bdc0a6..f0989d9d1e9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -75,6 +75,8 @@ import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VmDetailConstants; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.CONTROL; + public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { private KubernetesSupportedVersion kubernetesClusterVersion; @@ -183,7 +185,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif ResourceUnavailableException, InsufficientCapacityException { UserVm controlVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(CONTROL, kubernetesCluster); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Pair> ipAddresses = getKubernetesControlNodeIpAddresses(zone, network, owner); @@ -263,7 +265,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif ResourceUnavailableException, InsufficientCapacityException { UserVm additionalControlVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(CONTROL, kubernetesCluster); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Network.IpAddresses addrs = new Network.IpAddresses(null, null); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 12a50c9e88f..466b55c0d2b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -17,10 +17,12 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import java.security.InvalidParameterException; +import java.util.Map; import javax.inject.Inject; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetes.cluster.KubernetesClusterHelper; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -62,6 +64,8 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Inject public KubernetesClusterService kubernetesClusterService; + @Inject + protected KubernetesClusterHelper kubernetesClusterHelper; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -85,7 +89,18 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, description = "the ID of the service offering for the virtual machines in the cluster.") - private Long serviceOfferingId; + protected Long serviceOfferingId; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_OFFERING_MAP, type = CommandType.MAP, + description = "(Optional) Node Type to Service Offering ID mapping. If provided, it overrides the serviceofferingid parameter") + protected Map> serviceOfferingNodeTypeMap; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.ETCD_NODES, type = CommandType.LONG, + description = "(Optional) Number of Kubernetes cluster etcd nodes, default is 0." + + "In case the number is greater than 0, etcd nodes are separate from master nodes and are provisioned accordingly") + protected Long etcdNodes; @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the" + @@ -206,6 +221,10 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return controlNodes; } + public long getEtcdNodes() { + return etcdNodes == null ? 0 : etcdNodes; + } + public String getExternalLoadBalancerIpAddress() { return externalLoadBalancerIpAddress; } @@ -244,6 +263,10 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return clusterType; } + public Map getServiceOfferingNodeTypeMap() { + return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index e5a5c902f4d..ade4684e1cd 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -17,9 +17,11 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import java.util.List; +import java.util.Map; import javax.inject.Inject; +import com.cloud.kubernetes.cluster.KubernetesClusterHelper; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ACL; @@ -55,6 +57,8 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { @Inject public KubernetesClusterService kubernetesClusterService; + @Inject + protected KubernetesClusterHelper kubernetesClusterHelper; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -69,6 +73,11 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { description = "the ID of the service offering for the virtual machines in the cluster.") private Long serviceOfferingId; + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_OFFERING_MAP, type = CommandType.MAP, + description = "(Optional) Node Type to Service Offering ID mapping. If provided, it overrides the serviceofferingid parameter") + protected Map> serviceOfferingNodeTypeMap; + @Parameter(name=ApiConstants.SIZE, type = CommandType.LONG, description = "number of Kubernetes cluster nodes") private Long clusterSize; @@ -104,6 +113,10 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { return serviceOfferingId; } + public Map getServiceOfferingNodeTypeMap() { + return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(this.serviceOfferingNodeTypeMap); + } + public Long getClusterSize() { return clusterSize; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 168dfaf6091..c14fd9812f5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -58,6 +58,34 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "the name of the service offering of the Kubernetes cluster") private String serviceOfferingName; + @SerializedName(ApiConstants.WORKER_SERVICE_OFFERING_ID) + @Param(description = "the ID of the service offering of the worker nodes on the Kubernetes cluster") + private String workerOfferingId; + + @SerializedName(ApiConstants.WORKER_SERVICE_OFFERING_NAME) + @Param(description = "the name of the service offering of the worker nodes on the Kubernetes cluster") + private String workerOfferingName; + + @SerializedName(ApiConstants.CONTROL_SERVICE_OFFERING_ID) + @Param(description = "the ID of the service offering of the control nodes on the Kubernetes cluster") + private String controlOfferingId; + + @SerializedName(ApiConstants.CONTROL_SERVICE_OFFERING_NAME) + @Param(description = "the name of the service offering of the control nodes on the Kubernetes cluster") + private String controlOfferingName; + + @SerializedName(ApiConstants.ETCD_SERVICE_OFFERING_ID) + @Param(description = "the ID of the service offering of the etcd nodes on the Kubernetes cluster") + private String etcdOfferingId; + + @SerializedName(ApiConstants.ETCD_SERVICE_OFFERING_NAME) + @Param(description = "the name of the service offering of the etcd nodes on the Kubernetes cluster") + private String etcdOfferingName; + + @SerializedName(ApiConstants.ETCD_NODES) + @Param(description = "the number of the etcd nodes on the Kubernetes cluster") + private Long etcdNodes; + @SerializedName(ApiConstants.TEMPLATE_ID) @Param(description = "the ID of the template of the Kubernetes cluster") private String templateId; @@ -359,6 +387,62 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple this.serviceOfferingName = serviceOfferingName; } + public String getWorkerOfferingId() { + return workerOfferingId; + } + + public void setWorkerOfferingId(String workerOfferingId) { + this.workerOfferingId = workerOfferingId; + } + + public String getWorkerOfferingName() { + return workerOfferingName; + } + + public void setWorkerOfferingName(String workerOfferingName) { + this.workerOfferingName = workerOfferingName; + } + + public String getControlOfferingId() { + return controlOfferingId; + } + + public void setControlOfferingId(String controlOfferingId) { + this.controlOfferingId = controlOfferingId; + } + + public String getControlOfferingName() { + return controlOfferingName; + } + + public void setControlOfferingName(String controlOfferingName) { + this.controlOfferingName = controlOfferingName; + } + + public String getEtcdOfferingId() { + return etcdOfferingId; + } + + public void setEtcdOfferingId(String etcdOfferingId) { + this.etcdOfferingId = etcdOfferingId; + } + + public String getEtcdOfferingName() { + return etcdOfferingName; + } + + public void setEtcdOfferingName(String etcdOfferingName) { + this.etcdOfferingName = etcdOfferingName; + } + + public Long getEtcdNodes() { + return etcdNodes; + } + + public void setEtcdNodes(Long etcdNodes) { + this.etcdNodes = etcdNodes; + } + public void setVirtualMachines(List virtualMachines) { this.virtualMachines = virtualMachines; } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java new file mode 100644 index 00000000000..ba54ff38d38 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.vm.VmDetailConstants; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterHelperImplTest { + + @Mock + private ServiceOfferingDao serviceOfferingDao; + @Mock + private ServiceOfferingVO workerServiceOffering; + @Mock + private ServiceOfferingVO controlServiceOffering; + @Mock + private ServiceOfferingVO etcdServiceOffering; + + private static final String workerNodesOfferingId = UUID.randomUUID().toString(); + private static final String controlNodesOfferingId = UUID.randomUUID().toString(); + private static final String etcdNodesOfferingId = UUID.randomUUID().toString(); + private static final Long workerOfferingId = 1L; + private static final Long controlOfferingId = 2L; + private static final Long etcdOfferingId = 3L; + + private final KubernetesClusterHelperImpl helper = new KubernetesClusterHelperImpl(); + + @Before + public void setUp() { + helper.serviceOfferingDao = serviceOfferingDao; + Mockito.when(serviceOfferingDao.findByUuid(workerNodesOfferingId)).thenReturn(workerServiceOffering); + Mockito.when(serviceOfferingDao.findByUuid(controlNodesOfferingId)).thenReturn(controlServiceOffering); + Mockito.when(serviceOfferingDao.findByUuid(etcdNodesOfferingId)).thenReturn(etcdServiceOffering); + Mockito.when(workerServiceOffering.getId()).thenReturn(workerOfferingId); + Mockito.when(controlServiceOffering.getId()).thenReturn(controlOfferingId); + Mockito.when(etcdServiceOffering.getId()).thenReturn(etcdOfferingId); + } + + @Test + public void testIsValidNodeTypeEmptyNodeType() { + Assert.assertFalse(helper.isValidNodeType(null)); + } + + @Test + public void testIsValidNodeTypeInvalidNodeType() { + String nodeType = "invalidNodeType"; + Assert.assertFalse(helper.isValidNodeType(nodeType)); + } + + @Test + public void testIsValidNodeTypeValidNodeTypeLowercase() { + String nodeType = KubernetesClusterHelper.KubernetesClusterNodeType.WORKER.name().toLowerCase(); + Assert.assertTrue(helper.isValidNodeType(nodeType)); + } + + private Map createMapEntry(KubernetesClusterHelper.KubernetesClusterNodeType nodeType, + String nodeTypeOfferingUuid) { + Map map = new HashMap<>(); + map.put(VmDetailConstants.CKS_NODE_TYPE, nodeType.name().toLowerCase()); + map.put(VmDetailConstants.OFFERING, nodeTypeOfferingUuid); + return map; + } + + @Test + public void testNodeOfferingMap() { + Map> serviceOfferingNodeTypeMap = new HashMap<>(); + Map firstMap = createMapEntry(WORKER, workerNodesOfferingId); + Map secondMap = createMapEntry(CONTROL, controlNodesOfferingId); + serviceOfferingNodeTypeMap.put("map1", firstMap); + serviceOfferingNodeTypeMap.put("map2", secondMap); + Map map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + Assert.assertNotNull(map); + Assert.assertEquals(2, map.size()); + Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(CONTROL.name())); + Assert.assertEquals(workerOfferingId, map.get(WORKER.name())); + Assert.assertEquals(controlOfferingId, map.get(CONTROL.name())); + } + + @Test + public void testNodeOfferingMapNullMap() { + Map map = helper.getServiceOfferingNodeTypeMap(null); + Assert.assertTrue(map.isEmpty()); + } + + @Test + public void testNodeOfferingMapEtcdNodes() { + Map> serviceOfferingNodeTypeMap = new HashMap<>(); + Map firstMap = createMapEntry(ETCD, etcdNodesOfferingId); + serviceOfferingNodeTypeMap.put("map1", firstMap); + Map map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + Assert.assertNotNull(map); + Assert.assertEquals(1, map.size()); + Assert.assertTrue(map.containsKey(ETCD.name())); + Assert.assertEquals(etcdOfferingId, map.get(ETCD.name())); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryCompletenessInvalidParameters() { + helper.checkNodeTypeOfferingEntryCompleteness(WORKER.name(), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesInvalidNodeType() { + String invalidNodeType = "invalidNodeTypeName"; + helper.checkNodeTypeOfferingEntryValues(invalidNodeType, workerServiceOffering, workerNodesOfferingId); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesEmptyOffering() { + String nodeType = WORKER.name(); + helper.checkNodeTypeOfferingEntryValues(nodeType, null, workerNodesOfferingId); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index a6d46ffc9aa..49bddb969e4 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -27,16 +27,21 @@ import com.cloud.exception.PermissionDeniedException; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.vpc.NetworkACL; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; +import com.cloud.utils.Pair; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.BaseCmd; @@ -44,6 +49,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachi import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.commons.collections.MapUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -59,7 +65,13 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterManagerImplTest { @@ -85,6 +97,9 @@ public class KubernetesClusterManagerImplTest { @Mock private AccountManager accountManager; + @Mock + private ServiceOfferingDao serviceOfferingDao; + @Spy @InjectMocks KubernetesClusterManagerImpl kubernetesClusterManager; @@ -292,4 +307,111 @@ public class KubernetesClusterManagerImplTest { Mockito.when(kubernetesClusterDao.findById(Mockito.anyLong())).thenReturn(cluster); Assert.assertTrue(kubernetesClusterManager.removeVmsFromCluster(cmd).size() > 0); } + + @Test + public void testValidateServiceOfferingNodeType() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), 2L); + ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering); + Mockito.when(serviceOffering.isDynamic()).thenReturn(false); + Mockito.when(serviceOffering.getCpu()).thenReturn(2); + Mockito.when(serviceOffering.getRamSize()).thenReturn(2048); + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(version.getMinimumCpu()).thenReturn(2); + Mockito.when(version.getMinimumRamSize()).thenReturn(2048); + kubernetesClusterManager.validateServiceOfferingForNode(map, 1L, WORKER.name(), null, version); + Mockito.verify(kubernetesClusterManager).validateServiceOffering(serviceOffering, version); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateServiceOfferingNodeTypeInvalidOffering() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), 2L); + ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering); + Mockito.when(serviceOffering.isDynamic()).thenReturn(true); + kubernetesClusterManager.validateServiceOfferingForNode(map, 1L, WORKER.name(), null, null); + } + + @Test + public void testClusterCapacity() { + long workerOfferingId = 1L; + long controlOfferingId = 2L; + long workerCount = 2L; + long controlCount = 2L; + + int workerOfferingCpus = 4; + int workerOfferingMemory = 4096; + int controlOfferingCpus = 2; + int controlOfferingMemory = 2048; + + Map map = Map.of(WORKER.name(), workerOfferingId, CONTROL.name(), controlOfferingId); + Map nodeCount = Map.of(WORKER.name(), workerCount, CONTROL.name(), controlCount); + + ServiceOfferingVO workerOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(workerOfferingId)).thenReturn(workerOffering); + ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(controlOfferingId)).thenReturn(controlOffering); + Mockito.when(workerOffering.getCpu()).thenReturn(workerOfferingCpus); + Mockito.when(workerOffering.getRamSize()).thenReturn(workerOfferingMemory); + Mockito.when(controlOffering.getCpu()).thenReturn(controlOfferingCpus); + Mockito.when(controlOffering.getRamSize()).thenReturn(controlOfferingMemory); + + Pair pair = kubernetesClusterManager.calculateClusterCapacity(map, nodeCount); + Long expectedCpu = (workerOfferingCpus * workerCount) + (controlOfferingCpus * controlCount); + Long expectedMemory = (workerOfferingMemory * workerCount) + (controlOfferingMemory * controlCount); + Assert.assertEquals(expectedCpu, pair.first()); + Assert.assertEquals(expectedMemory, pair.second()); + } + + @Test + public void testIsAnyNodeOfferingEmptyNullMap() { + Assert.assertFalse(kubernetesClusterManager.isAnyNodeOfferingEmpty(null)); + } + + @Test + public void testIsAnyNodeOfferingEmptyNullValue() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), null); + map.put(ETCD.name(), 2L); + Assert.assertTrue(kubernetesClusterManager.isAnyNodeOfferingEmpty(map)); + } + + @Test + public void testIsAnyNodeOfferingEmpty() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), 2L); + Assert.assertFalse(kubernetesClusterManager.isAnyNodeOfferingEmpty(map)); + } + + @Test + public void testCreateNodeTypeToServiceOfferingMapNullMap() { + Map mapping = kubernetesClusterManager.createNodeTypeToServiceOfferingMap(null, null); + Assert.assertTrue(MapUtils.isEmpty(mapping)); + } + + @Test + public void testCreateNodeTypeToServiceOfferingMap() { + Map idsMap = new HashMap<>(); + long workerOfferingId = 1L; + long controlOfferingId = 2L; + idsMap.put(WORKER.name(), workerOfferingId); + idsMap.put(CONTROL.name(), controlOfferingId); + + ServiceOfferingVO workerOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(workerOfferingId)).thenReturn(workerOffering); + ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(controlOfferingId)).thenReturn(controlOffering); + + Map mapping = kubernetesClusterManager.createNodeTypeToServiceOfferingMap(idsMap, null); + Assert.assertEquals(2, mapping.size()); + Assert.assertTrue(mapping.containsKey(WORKER.name()) && mapping.containsKey(CONTROL.name())); + Assert.assertEquals(workerOffering, mapping.get(WORKER.name())); + Assert.assertEquals(controlOffering, mapping.get(CONTROL.name())); + } } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 86ef8228cc0..f66e4486500 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -451,9 +451,13 @@ "label.cisco.nexus1000v.password": "Nexus 1000v password", "label.cisco.nexus1000v.username": "Nexus 1000v username", "label.cks.cluster.autoscalingenabled": "Enable auto scaling on this cluster", +"label.cks.cluster.control.nodes.offeringid": "Service Offering for Control Nodes", +"label.cks.cluster.etcd.nodes": "Etcd Nodes", +"label.cks.cluster.etcd.nodes.offeringid": "Service Offering for etcd Nodes", "label.cks.cluster.maxsize": "Maximum cluster size (Worker nodes)", "label.cks.cluster.minsize": "Minimum cluster size (Worker nodes)", "label.cks.cluster.size": "Cluster size (Worker nodes)", +"label.cks.cluster.worker.nodes.offeringid": "Service Offering for Worker Nodes", "label.cleanup": "Clean up", "label.clear": "Clear", "label.clear.list": "Clear list", @@ -1884,6 +1888,9 @@ "label.service.lb.netscaler.servicepackages": "Netscaler service packages", "label.service.lb.netscaler.servicepackages.description": "Service package description", "label.service.offering": "Service offering", +"label.service.offering.controlnodes": "Compute offering for Control Nodes", +"label.service.offering.etcdnodes": "Compute offering for etcd Nodes", +"label.service.offering.workernodes": "Compute offering for Worker Nodes", "label.service.staticnat.associatepublicip": "Associate public IP", "label.service.staticnat.elasticipcheckbox": "Elastic IP", "label.servicegroupuuid": "Service Group", @@ -2399,6 +2406,7 @@ "label.bucket.delete": "Delete Bucket", "label.quotagb": "Quota in GB", "label.encryption": "Encryption", +"label.etcdnodes": "Number of etcd nodes", "label.versioning": "Versioning", "label.objectlocking": "Object Lock", "label.bucket.policy": "Bucket Policy", diff --git a/ui/src/components/view/InfoCard.vue b/ui/src/components/view/InfoCard.vue index 69c40233d08..38db32c6244 100644 --- a/ui/src/components/view/InfoCard.vue +++ b/ui/src/components/view/InfoCard.vue @@ -534,6 +534,39 @@ {{ resource.serviceofferingname || resource.serviceofferingid }} +
+
{{ $t('label.service.offering.controlnodes') }}
+
+ + {{ resource.controlofferingname || resource.controlofferingid }} + {{ resource.controlofferingname || resource.controlofferingid }} + {{ resource.controlofferingname || resource.controlofferingid }} +
+
+
+
{{ $t('label.service.offering.workernodes') }}
+
+ + {{ resource.workerofferingname || resource.workerofferingid }} + {{ resource.workerofferingname || resource.workerofferingid }} + {{ resource.workerofferingname || resource.workerofferingid }} +
+
+
+
{{ $t('label.service.offering.etcdnodes') }}
+
+ + {{ resource.etcdofferingname || resource.etcdofferingid }} + {{ resource.etcdofferingname || resource.etcdofferingid }} + {{ resource.etcdofferingname || resource.etcdofferingid }} +
+
+
+
{{ $t('label.etcdnodes') }}
+
+ {{ resource.etcdnodes }} +
+
{{ $t('label.diskoffering') }}
diff --git a/ui/src/views/compute/CreateKubernetesCluster.vue b/ui/src/views/compute/CreateKubernetesCluster.vue index ca1e424cca4..214b3022482 100644 --- a/ui/src/views/compute/CreateKubernetesCluster.vue +++ b/ui/src/views/compute/CreateKubernetesCluster.vue @@ -180,6 +180,81 @@ + + + + + + + + + + + {{ opt.name || opt.description }} + + + + + + + + {{ opt.name || opt.description }} + + + + + + + + + + + + {{ opt.name || opt.description }} + + + + +