etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Address review comments: plan CKS cluster deployment based on the node type

This commit is contained in:
nvazquez 2025-01-08 11:57:30 -03:00
parent 89effc7eda
commit 36a02b81d0
No known key found for this signature in database
GPG Key ID: 656E1BCC8CB54F84
5 changed files with 60 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View File

@ -1037,7 +1037,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId);
ServiceOffering serviceOffering = serviceOfferingId != null ? serviceOfferingDao.findById(serviceOfferingId) : null;
if (serviceOffering == null) {
throw new InvalidParameterValueException("No service offering found with ID: " + serviceOfferingId);
throw new InvalidParameterValueException("When serviceofferingid is not specified, " +
"service offerings for each node type must be specified in the nodeofferings parameter.");
}
try {
validateServiceOffering(serviceOffering, clusterKubernetesVersion);
@ -1294,7 +1295,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
protected boolean isAnyNodeOfferingEmpty(Map<String, Long> map) {
if (MapUtils.isEmpty(map)) {
return false;
return true;
}
return map.values().stream().anyMatch(Objects::isNull);
}
@ -1479,13 +1480,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback<KubernetesClusterVO>() {
@Override
public KubernetesClusterVO doInTransaction(TransactionStatus status) {
final ServiceOffering defaultServiceOffering = serviceOfferingDao.findById(defaultServiceOfferingId);
Pair<Long, Long> capacityPair = calculateClusterCapacity(serviceOfferingNodeTypeMap, nodeTypeCount, defaultServiceOfferingId);
final long cores = capacityPair.first();
final long memory = capacityPair.second();
KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(),
defaultServiceOffering.getId(), Objects.nonNull(finalTemplate) ? finalTemplate.getId() : null,
defaultServiceOfferingId, Objects.nonNull(finalTemplate) ? finalTemplate.getId() : null,
defaultNetwork.getId(), owner.getDomainId(), owner.getAccountId(), controlNodeCount, clusterSize,
KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory,
cmd.getNodeRootDiskSize(), "", KubernetesCluster.ClusterType.CloudManaged);

6
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
View File

@ -128,6 +128,10 @@ import com.cloud.vm.VmDetailConstants;
import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.UserVmDetailsDao;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER;
public class KubernetesClusterActionWorker {
@ -144,6 +148,8 @@ public class KubernetesClusterActionWorker {
protected Logger logger = LogManager.getLogger(getClass());
protected final static List<KubernetesClusterNodeType> CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER, CONTROL, ETCD);
protected StateMachine2<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> _stateMachine = KubernetesCluster.State.getStateMachine();
@Inject

35
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
View File

@ -259,13 +259,35 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId());
}
protected DeployDestination plan(Long domainId, Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException {
ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
/**
* Plan Kubernetes Cluster Deployment
* @return a map of DeployDestination per node type
*/
protected Map<String, DeployDestination> planKubernetesCluster(Long domainId, Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException {
Map<String, DeployDestination> destinationMap = new HashMap<>();
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
if (logger.isDebugEnabled()) {
logger.debug(String.format("Checking deployment destination for Kubernetes cluster : %s in zone : %s", kubernetesCluster.getName(), zone.getName()));
}
return plan(kubernetesCluster.getTotalNodeCount(), zone, offering, domainId, accountId, hypervisorType);
long controlNodeCount = kubernetesCluster.getControlNodeCount();
long clusterSize = kubernetesCluster.getNodeCount();
long etcdNodes = kubernetesCluster.getEtcdNodeCount();
Map<String, Long> nodeTypeCount = Map.of(WORKER.name(), clusterSize,
CONTROL.name(), controlNodeCount, ETCD.name(), etcdNodes);
for (KubernetesClusterNodeType nodeType : CLUSTER_NODES_TYPES_LIST) {
Long nodes = nodeTypeCount.getOrDefault(nodeType.name(), kubernetesCluster.getServiceOfferingId());
if (nodes == null || nodes == 0) {
continue;
}
ServiceOffering nodeOffering = getServiceOfferingForNodeTypeOnCluster(nodeType, kubernetesCluster);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Checking deployment destination for %s nodes on Kubernetes cluster : %s in zone : %s", nodeType.name(), kubernetesCluster.getName(), zone.getName()));
}
DeployDestination planForNodeType = plan(nodes, zone, nodeOffering, domainId, accountId, hypervisorType);
destinationMap.put(nodeType.name(), planForNodeType);
}
return destinationMap;
}
protected void resizeNodeVolume(final UserVm vm) throws ManagementServerException {
@ -288,7 +310,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
}
}
protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId) throws ManagementServerException {
protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId, KubernetesClusterNodeType nodeType) throws ManagementServerException {
CallContext vmContext = null;
if (!ApiCommandResourceType.VirtualMachine.equals(CallContext.current().getEventResourceType())); {
vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine);
@ -298,7 +320,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
if (Objects.nonNull(domainId) && !listDedicatedHostsInDomain(domainId).isEmpty()) {
DeployDestination dest = null;
try {
dest = plan(domainId, accountId, vm.getHypervisorType());
Map<String, DeployDestination> destinationMap = planKubernetesCluster(domainId, accountId, vm.getHypervisorType());
dest = destinationMap.get(nodeType.name());
} catch (InsufficientCapacityException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
}
@ -341,7 +364,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}
startKubernetesVM(vm, domainId, accountId);
startKubernetesVM(vm, domainId, accountId, WORKER);
vm = userVmDao.findById(vm.getId());
if (vm == null) {
throw new ManagementServerException(String.format("Failed to provision worker VM for Kubernetes cluster : %s", kubernetesCluster.getName()));

25
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
View File

@ -34,6 +34,7 @@ import java.util.stream.Collectors;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.kubernetes.cluster.KubernetesServiceHelper;
import com.cloud.network.vpc.NetworkACL;
import com.cloud.storage.VMTemplateVO;
import com.cloud.user.UserDataVO;
@ -85,6 +86,7 @@ import org.apache.logging.log4j.Level;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER;
public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker {
@ -515,7 +517,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(k8sControlVM);
}
startKubernetesVM(k8sControlVM, domainId, accountId);
startKubernetesVM(k8sControlVM, domainId, accountId, CONTROL);
k8sControlVM = userVmDao.findById(k8sControlVM.getId());
if (k8sControlVM == null) {
throw new ManagementServerException(String.format("Failed to provision control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@ -538,7 +540,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}
startKubernetesVM(vm, domainId, accountId);
startKubernetesVM(vm, domainId, accountId, CONTROL);
vm = userVmDao.findById(vm.getId());
if (vm == null) {
throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@ -560,7 +562,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
for (int i = 0; i < kubernetesCluster.getEtcdNodeCount(); i++) {
UserVm vm = createEtcdNode(etcdNodeGuestIps, etcdHostnames, i, domainId, accountId);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, true, true);
startKubernetesVM(vm, domainId, accountId);
startKubernetesVM(vm, domainId, accountId, ETCD);
vm = userVmDao.findById(vm.getId());
if (vm == null) {
throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@ -669,7 +671,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
}
try {
resizeNodeVolume(vm);
startKubernetesVM(vm, domainId, accountId);
KubernetesClusterVmMapVO map = kubernetesClusterVmMapDao.findByVmId(vm.getId());
KubernetesServiceHelper.KubernetesClusterNodeType nodeType = getNodeTypeFromClusterVMMapRecord(map);
startKubernetesVM(vm, domainId, accountId, nodeType);
} catch (ManagementServerException ex) {
logger.warn(String.format("Failed to start VM : %s in Kubernetes cluster : %s due to ", vm.getDisplayName(), kubernetesCluster.getName()) + ex);
// don't bail out here. proceed further to stop the reset of the VM's
@ -683,6 +687,16 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
}
}
private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromClusterVMMapRecord(KubernetesClusterVmMapVO map) {
if (map.isControlNode()) {
return CONTROL;
} else if (map.isEtcdNode()) {
return ETCD;
} else {
return WORKER;
}
}
private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) {
if (StringUtils.isEmpty(publicIpAddress)) {
KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData");
@ -733,7 +747,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
DeployDestination dest = null;
try {
VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId());
dest = plan(domainId, accountId, clusterTemplate.getHypervisorType());
Map<String, DeployDestination> destinationMap = planKubernetesCluster(domainId, accountId, clusterTemplate.getHypervisorType());
dest = destinationMap.get(WORKER.name());
} catch (InsufficientCapacityException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
}

2
plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java
View File

@ -370,7 +370,7 @@ public class KubernetesClusterManagerImplTest {
@Test
public void testIsAnyNodeOfferingEmptyNullMap() {
Assert.assertFalse(kubernetesClusterManager.isAnyNodeOfferingEmpty(null));
Assert.assertTrue(kubernetesClusterManager.isAnyNodeOfferingEmpty(null));
}
@Test