etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CKS Mark Nodes for Manual Upgrade and Filter Nodes to add to CKS cluster from the same network

This commit is contained in:
Nicolas Vazquez 2024-05-02 08:41:28 -03:00 committed by nvazquez
parent 469c08d1c2
commit f103c43f09
No known key found for this signature in database
GPG Key ID: 656E1BCC8CB54F84
13 changed files with 149 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
View File

@ -295,6 +295,7 @@ public class ApiConstants {
public static final String LBID = "lbruleid";
public static final String LB_PROVIDER = "lbprovider";
public static final String MAC_ADDRESS = "macaddress";
public static final String MANUAL_UPGRADE = "manualupgrade";
public static final String MAX = "max";
public static final String MAX_SNAPS = "maxsnaps";
public static final String MAX_CPU_NUMBER = "maxcpunumber";

1
engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql
View File

@ -337,6 +337,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_template
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_template_id', 'bigint unsigned COMMENT "template id to be used for etcd Nodes"');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','etcd_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the VM is an etcd node"');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','external_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node was imported into the Kubernetes cluster"');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','manual_upgrade', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node is marked for manual upgrade and excluded from the Kubernetes cluster upgrade operation"');
ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_service_offering_id` FOREIGN KEY `fk_cluster__control_service_offering_id`(`control_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE;
ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_service_offering_id` FOREIGN KEY `fk_cluster__worker_service_offering_id`(`worker_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE;

8
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View File

@ -1855,6 +1855,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
@Override
public boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd) {
if (!KubernetesServiceEnabled.value()) {
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId());
long networkId = kubernetesCluster.getNetworkId();
NetworkVO networkVO = networkDao.findById(networkId);
@ -1864,11 +1867,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
KubernetesClusterAddWorker addWorker = new KubernetesClusterAddWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
addWorker = ComponentContext.inject(addWorker);
return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr());
return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr(), cmd.isManualUpgrade());
}
@Override
public boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesClusterCmd cmd) throws Exception {
if (!KubernetesServiceEnabled.value()) {
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId());
List<Long> validNodeIds = validateNodes(cmd.getNodeIds(), null, null, kubernetesCluster, true);
if (validNodeIds.isEmpty()) {

11
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java
View File

@ -48,6 +48,9 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap {
@Column(name = "external_node")
boolean externalNode;
@Column(name = "manual_upgrade")
boolean manualUpgrade;
public KubernetesClusterVmMapVO() {
}
@ -105,4 +108,12 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap {
public void setExternalNode(boolean externalNode) {
this.externalNode = externalNode;
}
public boolean isManualUpgrade() {
return manualUpgrade;
}
public void setManualUpgrade(boolean manualUpgrade) {
this.manualUpgrade = manualUpgrade;
}
}

4
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
View File

@ -354,12 +354,14 @@ public class KubernetesClusterActionWorker {
return new File(keyFile);
}
protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode, boolean isExternalNode) {
protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode,
boolean isExternalNode, boolean markForManualUpgrade) {
return Transaction.execute(new TransactionCallback<KubernetesClusterVmMapVO>() {
@Override
public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) {
KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId, isControlNode);
newClusterVmMap.setExternalNode(isExternalNode);
newClusterVmMap.setManualUpgrade(markForManualUpgrade);
kubernetesClusterVmMapDao.persist(newClusterVmMap);
return newClusterVmMap;
}

8
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java
View File

@ -70,7 +70,7 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker {
super(kubernetesCluster, clusterManager);
}
public boolean addNodesToCluster(List<Long> nodeIds, boolean mountCksIsoOnVr) throws CloudRuntimeException {
public boolean addNodesToCluster(List<Long> nodeIds, boolean mountCksIsoOnVr, boolean manualUpgrade) throws CloudRuntimeException {
try {
init();
addNodeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterAddNodeTimeout.value() * 1000;
@ -90,7 +90,7 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker {
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AddNodeRequested);
Ternary<Integer, Long, Long> nodesAddedAndMemory = importNodeToCluster(nodeIds, network, publicIp, mountCksIsoOnVr);
int nodesAdded = nodesAddedAndMemory.first();
updateKubernetesCluster(kubernetesCluster.getId(), nodesAddedAndMemory);
updateKubernetesCluster(kubernetesCluster.getId(), nodesAddedAndMemory, manualUpgrade);
if (nodeIds.size() != nodesAdded) {
String msg = String.format("Not every node was added to the CKS cluster %s, nodes added: %s out of %s", kubernetesCluster.getUuid(), nodesAdded, nodeIds.size());
logger.info(msg);
@ -255,7 +255,7 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker {
return new Pair<>(true, ++nodeIndex);
}
private void updateKubernetesCluster(long clusterId, Ternary<Integer, Long, Long> additionalNodesDetails) {
private void updateKubernetesCluster(long clusterId, Ternary<Integer, Long, Long> additionalNodesDetails, boolean manualUpgrade) {
int additionalNodeCount = additionalNodesDetails.first();
KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId);
kubernetesClusterVO.setNodeCount(kubernetesClusterVO.getNodeCount() + additionalNodeCount);
@ -264,7 +264,7 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker {
kubernetesClusterDao.update(clusterId, kubernetesClusterVO);
kubernetesCluster = kubernetesClusterVO;
finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true));
finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true, manualUpgrade));
}

2
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
View File

@ -363,7 +363,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
List<UserVm> nodes = new ArrayList<>();
for (int i = offset + 1; i <= nodeCount; i++) {
UserVm vm = createKubernetesNode(publicIpAddress);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}

4
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
View File

@ -331,7 +331,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
ManagementServerException, InsufficientCapacityException, ResourceUnavailableException {
UserVm k8sControlVM = null;
k8sControlVM = createKubernetesControlNode(network, publicIpAddress);
addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false);
addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(k8sControlVM);
}
@ -353,7 +353,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
for (int i = 1; i < kubernetesCluster.getControlNodeCount(); i++) {
UserVm vm = null;
vm = createKubernetesAdditionalControlNode(publicIpAddress, i);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}

15
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
View File

@ -20,7 +20,9 @@ package com.cloud.kubernetes.cluster.actionworkers;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Level;
@ -40,7 +42,7 @@ import com.cloud.utils.ssh.SshHelper;
public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker {
private List<UserVm> clusterVMs = new ArrayList<>();
protected List<UserVm> clusterVMs = new ArrayList<>();
private KubernetesSupportedVersion upgradeVersion;
private final String upgradeScriptFilename = "upgrade-kubernetes.sh";
private File upgradeScriptFile;
@ -171,6 +173,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
if (CollectionUtils.isEmpty(clusterVMs)) {
logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster : %s, unable to retrieve VMs for cluster", kubernetesCluster.getName()));
}
filterOutManualUpgradeNodesFromClusterUpgrade();
retrieveScriptFiles();
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested);
attachIsoKubernetesVMs(clusterVMs, upgradeVersion);
@ -186,4 +189,14 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke
}
return updated;
}
protected void filterOutManualUpgradeNodesFromClusterUpgrade() {
if (CollectionUtils.isEmpty(clusterVMs)) {
return;
}
clusterVMs = clusterVMs.stream().filter(x -> {
KubernetesClusterVmMapVO mapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(x.getId());
return mapVO != null && !mapVO.isManualUpgrade();
}).collect(Collectors.toList());
}
}

11
plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java
View File

@ -31,6 +31,7 @@ import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.log4j.Logger;
import javax.inject.Inject;
@ -69,6 +70,10 @@ public class AddNodesToKubernetesClusterCmd extends BaseAsyncCmd {
since = "4.20.0")
private Boolean mountCksIsoOnVr;
@Parameter(name = ApiConstants.MANUAL_UPGRADE, type = CommandType.BOOLEAN,
description = "(optional) indicates if the node is marked for manual upgrade and excluded from the Kubernetes cluster upgrade operation",
since = "4.20.0")
private Boolean manualUpgrade;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
@ -83,7 +88,11 @@ public class AddNodesToKubernetesClusterCmd extends BaseAsyncCmd {
}
public boolean isMountCksIsoOnVr() {
return mountCksIsoOnVr != null && mountCksIsoOnVr;
return BooleanUtils.isTrue(mountCksIsoOnVr);
}
public boolean isManualUpgrade() {
return BooleanUtils.isTrue(manualUpgrade);
}
/////////////////////////////////////////////////////

83
plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java Normal file
View File

@ -0,0 +1,83 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster.actionworkers;
import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.kubernetes.version.KubernetesSupportedVersion;
import com.cloud.uservm.UserVm;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
@RunWith(MockitoJUnitRunner.class)
public class KubernetesClusterUpgradeWorkerTest {
@Mock
private KubernetesCluster kubernetesCluster;
@Mock
private KubernetesSupportedVersion kubernetesSupportedVersion;
@Mock
private KubernetesClusterManagerImpl clusterManager;
@Mock
private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
private KubernetesClusterUpgradeWorker worker;
@Before
public void setUp() {
String[] keys = {};
worker = new KubernetesClusterUpgradeWorker(kubernetesCluster, kubernetesSupportedVersion, clusterManager, keys);
worker.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao;
}
@Test
public void testFilterOutManualUpgradeNodesFromClusterUpgrade() {
long controlNodeId = 1L;
long workerNode1Id = 2L;
long workerNode2Id = 3L;
UserVm controlNode = Mockito.mock(UserVm.class);
Mockito.when(controlNode.getId()).thenReturn(controlNodeId);
UserVm workerNode1 = Mockito.mock(UserVm.class);
Mockito.when(workerNode1.getId()).thenReturn(workerNode1Id);
UserVm workerNode2 = Mockito.mock(UserVm.class);
Mockito.when(workerNode2.getId()).thenReturn(workerNode2Id);
KubernetesClusterVmMapVO controlNodeMap = Mockito.mock(KubernetesClusterVmMapVO.class);
KubernetesClusterVmMapVO workerNode1Map = Mockito.mock(KubernetesClusterVmMapVO.class);
KubernetesClusterVmMapVO workerNode2Map = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerNode2Map.isManualUpgrade()).thenReturn(true);
Mockito.when(kubernetesClusterVmMapDao.getClusterMapFromVmId(controlNodeId)).thenReturn(controlNodeMap);
Mockito.when(kubernetesClusterVmMapDao.getClusterMapFromVmId(workerNode1Id)).thenReturn(workerNode1Map);
Mockito.when(kubernetesClusterVmMapDao.getClusterMapFromVmId(workerNode2Id)).thenReturn(workerNode2Map);
worker.clusterVMs = Arrays.asList(controlNode, workerNode1, workerNode2);
worker.filterOutManualUpgradeNodesFromClusterUpgrade();
Assert.assertEquals(2, worker.clusterVMs.size());
List<Long> ids = worker.clusterVMs.stream().map(UserVm::getId).collect(Collectors.toList());
Assert.assertTrue(ids.contains(controlNodeId) && ids.contains(workerNode1Id));
Assert.assertFalse(ids.contains(workerNode2Id));
}
}

3
ui/public/locales/en.json
View File

@ -469,6 +469,7 @@
"label.cks.cluster.etcd.nodes.templateid": "Template for etcd Nodes",
"label.cks.cluster.maxsize": "Maximum cluster size (Worker nodes)",
"label.cks.cluster.minsize": "Minimum cluster size (Worker nodes)",
"label.cks.cluster.node.manual.upgrade": "Mark nodes for manual upgrade",
"label.cks.cluster.size": "Cluster size (Worker nodes)",
"label.cks.cluster.worker.nodes.offeringid": "Service Offering for Worker Nodes",
"label.cks.cluster.worker.nodes.templateid": "Template for Worker Nodes",
@ -1380,7 +1381,7 @@
"label.monitor.url": "URL Path",
"label.monthly": "Monthly",
"label.more.access.dashboard.ui": "More about accessing dashboard UI",
"label.mount.cks.iso.on.vr": "Mount and serve CKS ISO on the CKS cluster's network Virtual Router",
"label.mount.cks.iso.on.vr": "Use CKS packages from Virtual Router",
"label.move.down.row": "Move down one row",
"label.move.to.bottom": "Move to bottom",
"label.move.to.top": "Move to top",

11
ui/src/views/compute/KubernetesAddNodes.vue
View File

@ -48,6 +48,11 @@
{{ $t('label.mount.cks.iso.on.vr') }}
</a-checkbox>
</a-form-item>
<a-form-item name="manualupgrade" ref="manualupgrade">
<a-checkbox v-model:checked="form.manualupgrade">
{{ $t('label.cks.cluster.node.manual.upgrade') }}
</a-checkbox>
</a-form-item>
</div>
<p v-else v-html="$t('label.vms.empty')" />
@ -111,7 +116,8 @@ export default {
accountId: accountId,
domainId: domainId,
details: 'min',
listall: 'true'
listall: 'true',
networkid: this.resource.networkid
}).then(json => {
const vms = json.listvirtualmachinesresponse.virtualmachine || []
resolve(vms)
@ -135,6 +141,9 @@ export default {
if (values.mountcksiso) {
params.mountcksisoonvr = values.mountcksiso
}
if (values.manualupgrade) {
params.manualupgrade = values.manualupgrade
}
this.loading = true
try {
const jobId = await this.addNodesToKubernetesCluster(params)