From 36a02b81d0e449f1c0b900291a991dae721ca0fa Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 8 Jan 2025 11:57:30 -0300 Subject: [PATCH] Address review comments: plan CKS cluster deployment based on the node type --- .../cluster/KubernetesClusterManagerImpl.java | 8 ++--- .../KubernetesClusterActionWorker.java | 6 ++++ ...esClusterResourceModifierActionWorker.java | 35 +++++++++++++++---- .../KubernetesClusterStartWorker.java | 25 ++++++++++--- .../KubernetesClusterManagerImplTest.java | 2 +- 5 files changed, 60 insertions(+), 16 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 1400dac8303..4ac1a5bf1d0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1037,7 +1037,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId); ServiceOffering serviceOffering = serviceOfferingId != null ? serviceOfferingDao.findById(serviceOfferingId) : null; if (serviceOffering == null) { - throw new InvalidParameterValueException("No service offering found with ID: " + serviceOfferingId); + throw new InvalidParameterValueException("When serviceofferingid is not specified, " + + "service offerings for each node type must be specified in the nodeofferings parameter."); } try { validateServiceOffering(serviceOffering, clusterKubernetesVersion); @@ -1294,7 +1295,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected boolean isAnyNodeOfferingEmpty(Map map) { if (MapUtils.isEmpty(map)) { - return false; + return true; } return map.values().stream().anyMatch(Objects::isNull); } @@ -1479,13 +1480,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVO doInTransaction(TransactionStatus status) { - final ServiceOffering defaultServiceOffering = serviceOfferingDao.findById(defaultServiceOfferingId); Pair capacityPair = calculateClusterCapacity(serviceOfferingNodeTypeMap, nodeTypeCount, defaultServiceOfferingId); final long cores = capacityPair.first(); final long memory = capacityPair.second(); KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(), - defaultServiceOffering.getId(), Objects.nonNull(finalTemplate) ? finalTemplate.getId() : null, + defaultServiceOfferingId, Objects.nonNull(finalTemplate) ? finalTemplate.getId() : null, defaultNetwork.getId(), owner.getDomainId(), owner.getAccountId(), controlNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), "", KubernetesCluster.ClusterType.CloudManaged); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 867006d6d6a..be155985561 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -128,6 +128,10 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + public class KubernetesClusterActionWorker { @@ -144,6 +148,8 @@ public class KubernetesClusterActionWorker { protected Logger logger = LogManager.getLogger(getClass()); + protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER, CONTROL, ETCD); + protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); @Inject diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 62e72288964..f2b3ae01915 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -259,13 +259,35 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); } - protected DeployDestination plan(Long domainId, Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException { - ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + /** + * Plan Kubernetes Cluster Deployment + * @return a map of DeployDestination per node type + */ + protected Map planKubernetesCluster(Long domainId, Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException { + Map destinationMap = new HashMap<>(); DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); if (logger.isDebugEnabled()) { logger.debug(String.format("Checking deployment destination for Kubernetes cluster : %s in zone : %s", kubernetesCluster.getName(), zone.getName())); } - return plan(kubernetesCluster.getTotalNodeCount(), zone, offering, domainId, accountId, hypervisorType); + long controlNodeCount = kubernetesCluster.getControlNodeCount(); + long clusterSize = kubernetesCluster.getNodeCount(); + long etcdNodes = kubernetesCluster.getEtcdNodeCount(); + Map nodeTypeCount = Map.of(WORKER.name(), clusterSize, + CONTROL.name(), controlNodeCount, ETCD.name(), etcdNodes); + + for (KubernetesClusterNodeType nodeType : CLUSTER_NODES_TYPES_LIST) { + Long nodes = nodeTypeCount.getOrDefault(nodeType.name(), kubernetesCluster.getServiceOfferingId()); + if (nodes == null || nodes == 0) { + continue; + } + ServiceOffering nodeOffering = getServiceOfferingForNodeTypeOnCluster(nodeType, kubernetesCluster); + if (logger.isDebugEnabled()) { + logger.debug(String.format("Checking deployment destination for %s nodes on Kubernetes cluster : %s in zone : %s", nodeType.name(), kubernetesCluster.getName(), zone.getName())); + } + DeployDestination planForNodeType = plan(nodes, zone, nodeOffering, domainId, accountId, hypervisorType); + destinationMap.put(nodeType.name(), planForNodeType); + } + return destinationMap; } protected void resizeNodeVolume(final UserVm vm) throws ManagementServerException { @@ -288,7 +310,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } } - protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId) throws ManagementServerException { + protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId, KubernetesClusterNodeType nodeType) throws ManagementServerException { CallContext vmContext = null; if (!ApiCommandResourceType.VirtualMachine.equals(CallContext.current().getEventResourceType())); { vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine); @@ -298,7 +320,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu if (Objects.nonNull(domainId) && !listDedicatedHostsInDomain(domainId).isEmpty()) { DeployDestination dest = null; try { - dest = plan(domainId, accountId, vm.getHypervisorType()); + Map destinationMap = planKubernetesCluster(domainId, accountId, vm.getHypervisorType()); + dest = destinationMap.get(nodeType.name()); } catch (InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } @@ -341,7 +364,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } - startKubernetesVM(vm, domainId, accountId); + startKubernetesVM(vm, domainId, accountId, WORKER); vm = userVmDao.findById(vm.getId()); if (vm == null) { throw new ManagementServerException(String.format("Failed to provision worker VM for Kubernetes cluster : %s", kubernetesCluster.getName())); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index e5fda6f1b28..bbcc748b8a5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -34,6 +34,7 @@ import java.util.stream.Collectors; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.PermissionDeniedException; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper; import com.cloud.network.vpc.NetworkACL; import com.cloud.storage.VMTemplateVO; import com.cloud.user.UserDataVO; @@ -85,6 +86,7 @@ import org.apache.logging.log4j.Level; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { @@ -515,7 +517,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(k8sControlVM); } - startKubernetesVM(k8sControlVM, domainId, accountId); + startKubernetesVM(k8sControlVM, domainId, accountId, CONTROL); k8sControlVM = userVmDao.findById(k8sControlVM.getId()); if (k8sControlVM == null) { throw new ManagementServerException(String.format("Failed to provision control VM for Kubernetes cluster : %s" , kubernetesCluster.getName())); @@ -538,7 +540,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } - startKubernetesVM(vm, domainId, accountId); + startKubernetesVM(vm, domainId, accountId, CONTROL); vm = userVmDao.findById(vm.getId()); if (vm == null) { throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName())); @@ -560,7 +562,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif for (int i = 0; i < kubernetesCluster.getEtcdNodeCount(); i++) { UserVm vm = createEtcdNode(etcdNodeGuestIps, etcdHostnames, i, domainId, accountId); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, true, true); - startKubernetesVM(vm, domainId, accountId); + startKubernetesVM(vm, domainId, accountId, ETCD); vm = userVmDao.findById(vm.getId()); if (vm == null) { throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName())); @@ -669,7 +671,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } try { resizeNodeVolume(vm); - startKubernetesVM(vm, domainId, accountId); + KubernetesClusterVmMapVO map = kubernetesClusterVmMapDao.findByVmId(vm.getId()); + KubernetesServiceHelper.KubernetesClusterNodeType nodeType = getNodeTypeFromClusterVMMapRecord(map); + startKubernetesVM(vm, domainId, accountId, nodeType); } catch (ManagementServerException ex) { logger.warn(String.format("Failed to start VM : %s in Kubernetes cluster : %s due to ", vm.getDisplayName(), kubernetesCluster.getName()) + ex); // don't bail out here. proceed further to stop the reset of the VM's @@ -683,6 +687,16 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } } + private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromClusterVMMapRecord(KubernetesClusterVmMapVO map) { + if (map.isControlNode()) { + return CONTROL; + } else if (map.isEtcdNode()) { + return ETCD; + } else { + return WORKER; + } + } + private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) { if (StringUtils.isEmpty(publicIpAddress)) { KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); @@ -733,7 +747,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif DeployDestination dest = null; try { VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId()); - dest = plan(domainId, accountId, clusterTemplate.getHypervisorType()); + Map destinationMap = planKubernetesCluster(domainId, accountId, clusterTemplate.getHypervisorType()); + dest = destinationMap.get(WORKER.name()); } catch (InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index 41ec4de2a04..a9cb7096b8a 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -370,7 +370,7 @@ public class KubernetesClusterManagerImplTest { @Test public void testIsAnyNodeOfferingEmptyNullMap() { - Assert.assertFalse(kubernetesClusterManager.isAnyNodeOfferingEmpty(null)); + Assert.assertTrue(kubernetesClusterManager.isAnyNodeOfferingEmpty(null)); } @Test