Compare commits

...

41 Commits

Author SHA1 Message Date
Daman Arora a57420cc4c
Merge bbeb4e0e8d into 420bf6dff8 2026-01-22 09:17:14 +01:00
Suresh Kumar Anaparti 420bf6dff8
Merge branch '4.22' 2026-01-22 13:24:08 +05:30
Suresh Kumar Anaparti b1f870ae83
Merge branch '4.20' into 4.22 2026-01-22 13:23:21 +05:30
Wei Zhou 036489b288
CKS: fix resource limitation check on cpu when scale cks cluster (#12379) 2026-01-21 09:59:21 +01:00
Suresh Kumar Anaparti 8db7cab7ba
Storage pool monitor disconnect improvements (#12398) 2026-01-20 09:08:39 +01:00
Nicolas Vazquez 496bc0329c
Fix: Condition for aborting migration, resume paused VMs on destination (#12331) 2026-01-20 08:56:32 +01:00
Abhisar Sinha cf36fb0000
Set nfsVersion in ssvm agent.properties only if it is not null (#12445) 2026-01-20 08:25:16 +01:00
Daman Arora da518e9036
CKS: Add image store validation for Kubernetes version registration (#12418)
Co-authored-by: Daman Arora <daman.arora@shapeblue.com>
2026-01-20 08:13:15 +01:00
Henrique Sato 03d24ff851
Fix NPE on primary storage delete (#11817) 2026-01-20 08:12:16 +01:00
Vitor Hugo Homem Marzarotto 2a6ce0c8a8
Adds url kubernetes iso (#10862)
Co-authored-by: Vitor Hugo Homem Marzarotto <vitor.marzarotto@scclouds.com.br>
Co-authored-by: Henrique Sato <henriquesato2003@gmail.com>
2026-01-20 08:10:42 +01:00
Manoj Kumar 42f1e19362
Mask vncPasswd being logged in agent.log (#12404) 2026-01-19 14:20:18 +01:00
Abhishek Kumar a4b1a27c7d
ui: fix 404 on login after forgot password (#12448)
Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
2026-01-19 08:50:07 +01:00
Daman Arora bbeb4e0e8d Add integration tests 2026-01-16 11:52:34 -05:00
Daman Arora fc548975d3 add affinity group support Kubernetes cluster creation 2026-01-13 06:39:00 -05:00
Daman Arora d62b9f3b73 refactor test mocks to use lenient behavior 2026-01-12 09:19:33 -05:00
Daman Arora cd37b81147 implement node affinity group validation method 2026-01-12 09:16:03 -05:00
Daman Arora d27b2f45be update user VM response handling in KubernetesClusterManagerImpl 2026-01-08 14:30:13 -05:00
Daman Arora 8f5ee6dae3 add affinity group details to user VM response 2026-01-08 13:27:48 -05:00
Daman Arora a05581cec3 add unit tests 2026-01-08 11:27:29 -05:00
Daman Arora 96c0705b10 cleanup 2026-01-07 13:24:21 -05:00
Daman Arora 2405249414 add affinty groups to cks list response 2026-01-07 12:29:18 -05:00
Daman Arora 201e5639e9 remove affinity group on cleanup in mcloud managed cks 2026-01-07 09:45:29 -05:00
Daman Arora e0d41831b7 use @component for spring bean 2026-01-07 09:44:54 -05:00
Daman Arora c58dee04d7 remove affinity group mappings when a cluster is deleted 2026-01-07 09:15:46 -05:00
Daman Arora af97ea3911 use DAO query instead of parsing comma-separated UUIDs 2026-01-07 08:45:26 -05:00
Daman Arora 35a7bab9ca use updated getAffinityGroupNodeTypeMap 2026-01-07 08:41:46 -05:00
Daman Arora f625d6ed93 Refactor affinity group mapping 2026-01-07 08:41:17 -05:00
Daman Arora 6e3ede9d78 add new resource KubernetesClusterAffinityGroupMap 2026-01-07 08:39:09 -05:00
Daman Arora a13f360bcd use a new table kubernetes_cluster_affinity_group_map instead of existing kubernetes_cluster 2026-01-07 07:22:02 -05:00
Daman Arora 0706410a3f Add per node type affinity group support for cks 2026-01-06 15:23:30 -05:00
Daman Arora 58799c25ba Refactor affinity group tests in KubernetesServiceHelperImplTest 2026-01-06 12:38:48 -05:00
Daman Arora 4da3bcec83 Update affinity group handling to support multiple IDs in KubernetesServiceHelper and related classes 2026-01-06 12:38:23 -05:00
Daman Arora fe5c0260d6 Refactor affinity group handling in KubernetesCluster and KubernetesClusterVO to support multiple IDs 2026-01-06 12:37:42 -05:00
Daman Arora 4706d0315e Add affinity group handling for worker, control, and etcd nodes in KubernetesClusterManagerImpl 2026-01-06 11:07:43 -05:00
Daman Arora 8bf7a453c9 Add affinity group ID fields and accessors to KubernetesCluster and KubernetesClusterVO 2026-01-06 11:04:48 -05:00
Daman Arora 319c0f6f94 Add affinity group columns to kubernetes_cluster table 2026-01-06 11:04:22 -05:00
Daman Arora 1114f759c2 Refactor KubernetesServiceHelperImplTest to include affinity group handling and enhance node type validation tests 2026-01-06 10:27:03 -05:00
Daman Arora 58804a39a7 Rename kubernetesClusterHelper to kubernetesServiceHelper for consistency 2026-01-06 10:26:51 -05:00
Daman Arora a7e5270336 Implement getAffinityGroupNodeTypeMap in kubernetes service helper 2026-01-06 10:26:41 -05:00
Daman Arora 9f137af735 Merge branch 'main' into implement-cks-node-affinity 2026-01-06 09:05:19 -05:00
Daman Arora fe0a2a3397 Add NODE_TYPE_AFFINITY_GROUP_MAP constant and affinity group mapping to CreateKubernetesClusterCmd 2026-01-06 08:51:43 -05:00
39 changed files with 2876 additions and 250 deletions

View File

@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster;
import org.apache.cloudstack.acl.ControlledEntity;
import java.util.List;
import java.util.Map;
import com.cloud.user.Account;
@ -36,5 +37,6 @@ public interface KubernetesServiceHelper extends Adapter {
boolean isValidNodeType(String nodeType);
Map<String, Long> getServiceOfferingNodeTypeMap(Map<String, Map<String, String>> serviceOfferingNodeTypeMap);
Map<String, Long> getTemplateNodeTypeMap(Map<String, Map<String, String>> templateNodeTypeMap);
Map<String, List<Long>> getAffinityGroupNodeTypeMap(Map<String, Map<String, String>> affinityGroupNodeTypeMap);
void cleanupForAccount(Account account);
}

View File

@ -96,6 +96,7 @@ public interface VmDetailConstants {
String CKS_NODE_TYPE = "node";
String OFFERING = "offering";
String TEMPLATE = "template";
String AFFINITY_GROUP = "affinitygroup";
// VMware to KVM VM migrations specific
String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm";

View File

@ -1216,6 +1216,7 @@ public class ApiConstants {
public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail";
public static final String ISO_NAME = "isoname";
public static final String ISO_STATE = "isostate";
public static final String ISO_URL = "isourl";
public static final String SEMANTIC_VERSION = "semanticversion";
public static final String KUBERNETES_VERSION_ID = "kubernetesversionid";
public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname";
@ -1236,6 +1237,13 @@ public class ApiConstants {
public static final String MAX_SIZE = "maxsize";
public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings";
public static final String NODE_TYPE_TEMPLATE_MAP = "nodetemplates";
public static final String NODE_TYPE_AFFINITY_GROUP_MAP = "nodeaffinitygroups";
public static final String CONTROL_AFFINITY_GROUP_IDS = "controlaffinitygroupids";
public static final String CONTROL_AFFINITY_GROUP_NAMES = "controlaffinitygroupnames";
public static final String WORKER_AFFINITY_GROUP_IDS = "workeraffinitygroupids";
public static final String WORKER_AFFINITY_GROUP_NAMES = "workeraffinitygroupnames";
public static final String ETCD_AFFINITY_GROUP_IDS = "etcdaffinitygroupids";
public static final String ETCD_AFFINITY_GROUP_NAMES = "etcdaffinitygroupnames";
public static final String BOOT_TYPE = "boottype";
public static final String BOOT_MODE = "bootmode";

View File

@ -34,6 +34,19 @@ CREATE TABLE `cloud`.`backup_offering_details` (
UPDATE `cloud`.`configuration` SET value='random' WHERE name IN ('vm.allocation.algorithm', 'volume.allocation.algorithm') AND value='userconcentratedpod_random';
UPDATE `cloud`.`configuration` SET value='firstfit' WHERE name IN ('vm.allocation.algorithm', 'volume.allocation.algorithm') AND value='userconcentratedpod_firstfit';
-- Create kubernetes_cluster_affinity_group_map table for CKS per-node-type affinity groups
CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_affinity_group_map` (
`id` bigint unsigned NOT NULL AUTO_INCREMENT,
`cluster_id` bigint unsigned NOT NULL COMMENT 'kubernetes cluster id',
`node_type` varchar(32) NOT NULL COMMENT 'CONTROL, WORKER, or ETCD',
`affinity_group_id` bigint unsigned NOT NULL COMMENT 'affinity group id',
PRIMARY KEY (`id`),
CONSTRAINT `fk_kubernetes_cluster_ag_map__cluster_id` FOREIGN KEY (`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE,
CONSTRAINT `fk_kubernetes_cluster_ag_map__ag_id` FOREIGN KEY (`affinity_group_id`) REFERENCES `affinity_group`(`id`) ON DELETE CASCADE,
INDEX `i_kubernetes_cluster_ag_map__cluster_id`(`cluster_id`),
INDEX `i_kubernetes_cluster_ag_map__ag_id`(`affinity_group_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Create webhook_filter table
DROP TABLE IF EXISTS `cloud`.`webhook_filter`;
CREATE TABLE IF NOT EXISTS `cloud`.`webhook_filter` (

View File

@ -163,7 +163,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
final String target = command.getDestinationIp();
xmlDesc = dm.getXMLDesc(xmlFlag);
if (logger.isDebugEnabled()) {
logger.debug(String.format("VM [%s] with XML configuration [%s] will be migrated to host [%s].", vmName, xmlDesc, target));
logger.debug("VM {} with XML configuration {} will be migrated to host {}.", vmName, maskSensitiveInfoInXML(xmlDesc), target);
}
// Limit the VNC password in case the length is greater than 8 characters
@ -178,7 +178,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
logger.debug(String.format("Editing mount path of ISO from %s to %s", oldIsoVolumePath, newIsoVolumePath));
xmlDesc = replaceDiskSourceFile(xmlDesc, newIsoVolumePath, vmName);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Replaced disk mount point [%s] with [%s] in Instance [%s] XML configuration. New XML configuration is [%s].", oldIsoVolumePath, newIsoVolumePath, vmName, xmlDesc));
logger.debug("Replaced disk mount point {} with {} in Instance {} XML configuration. New XML configuration is {}.", oldIsoVolumePath, newIsoVolumePath, vmName, maskSensitiveInfoInXML(xmlDesc));
}
}
@ -209,11 +209,11 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
if (migrateStorage) {
if (logger.isDebugEnabled()) {
logger.debug(String.format("Changing VM [%s] volumes during migration to host: [%s].", vmName, target));
logger.debug("Changing VM {} volumes during migration to host: {}.", vmName, target);
}
xmlDesc = replaceStorage(xmlDesc, mapMigrateStorage, migrateStorageManaged);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Changed VM [%s] XML configuration of used storage. New XML configuration is [%s].", vmName, xmlDesc));
logger.debug("Changed VM {} XML configuration of used storage. New XML configuration is {}.", vmName, maskSensitiveInfoInXML(xmlDesc));
}
migrateDiskLabels = getMigrateStorageDeviceLabels(disks, mapMigrateStorage);
}
@ -221,11 +221,11 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
Map<String, DpdkTO> dpdkPortsMapping = command.getDpdkInterfaceMapping();
if (MapUtils.isNotEmpty(dpdkPortsMapping)) {
if (logger.isTraceEnabled()) {
logger.trace(String.format("Changing VM [%s] DPDK interfaces during migration to host: [%s].", vmName, target));
logger.trace("Changing VM {} DPDK interfaces during migration to host: {}.", vmName, target);
}
xmlDesc = replaceDpdkInterfaces(xmlDesc, dpdkPortsMapping);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Changed VM [%s] XML configuration of DPDK interfaces. New XML configuration is [%s].", vmName, xmlDesc));
logger.debug("Changed VM {} XML configuration of DPDK interfaces. New XML configuration is {}.", vmName, maskSensitiveInfoInXML(xmlDesc));
}
}
@ -240,7 +240,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
}
//run migration in thread so we can monitor it
logger.info(String.format("Starting live migration of instance [%s] to destination host [%s] having the final XML configuration: [%s].", vmName, dconn.getURI(), xmlDesc));
logger.info("Starting live migration of instance {} to destination host {} having the final XML configuration: {}.", vmName, dconn.getURI(), maskSensitiveInfoInXML(xmlDesc));
final ExecutorService executor = Executors.newFixedThreadPool(1);
boolean migrateNonSharedInc = command.isMigrateNonSharedInc() && !migrateStorageManaged;
@ -256,22 +256,23 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
final Future<Domain> migrateThread = executor.submit(worker);
executor.shutdown();
long sleeptime = 0;
final int migrateDowntime = libvirtComputingResource.getMigrateDowntime();
boolean isMigrateDowntimeSet = false;
while (!executor.isTerminated()) {
Thread.sleep(100);
sleeptime += 100;
if (sleeptime == 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state
final int migrateDowntime = libvirtComputingResource.getMigrateDowntime();
if (migrateDowntime > 0 ) {
if (!isMigrateDowntimeSet && migrateDowntime > 0 && sleeptime >= 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state
try {
final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime);
if (setDowntime == 0 ) {
isMigrateDowntimeSet = true;
logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms");
}
} catch (final LibvirtException e) {
logger.debug("Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage());
}
}
}
if (sleeptime % 1000 == 0) {
logger.info("Waiting for migration of " + vmName + " to complete, waited " + sleeptime + "ms");
}
@ -287,7 +288,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
} catch (final LibvirtException e) {
logger.info("Couldn't get VM domain state after " + sleeptime + "ms: " + e.getMessage());
}
if (state != null && state == DomainState.VIR_DOMAIN_RUNNING) {
if (state != null && (state == DomainState.VIR_DOMAIN_RUNNING || state == DomainState.VIR_DOMAIN_PAUSED)) {
try {
DomainJobInfo job = dm.getJobInfo();
logger.warn("Aborting migration of VM {} with domain job [{}] due to timeout after {} seconds. " +
@ -334,6 +335,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
if (logger.isDebugEnabled()) {
logger.debug(String.format("Cleaning the disks of VM [%s] in the source pool after VM migration finished.", vmName));
}
resumeDomainIfPaused(destDomain, vmName);
deleteOrDisconnectDisksOnSourcePool(libvirtComputingResource, migrateDiskInfoList, disks);
libvirtComputingResource.cleanOldSecretsByDiskDef(conn, disks);
}
@ -408,6 +410,28 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
return new MigrateAnswer(command, result == null, result, null);
}
private DomainState getDestDomainState(Domain destDomain, String vmName) {
DomainState dmState = null;
try {
dmState = destDomain.getInfo().state;
} catch (final LibvirtException e) {
logger.info("Failed to get domain state for VM: " + vmName + " due to: " + e.getMessage());
}
return dmState;
}
private void resumeDomainIfPaused(Domain destDomain, String vmName) {
DomainState dmState = getDestDomainState(destDomain, vmName);
if (dmState == DomainState.VIR_DOMAIN_PAUSED) {
logger.info("Resuming VM " + vmName + " on destination after migration");
try {
destDomain.resume();
} catch (final Exception e) {
logger.error("Failed to resume vm " + vmName + " on destination after migration due to : " + e.getMessage());
}
}
}
/**
* Gets the disk labels (vda, vdb...) of the disks mapped for migration on mapMigrateStorage.
* @param diskDefinitions list of all the disksDefinitions of the VM.
@ -715,9 +739,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
graphElem = graphElem.replaceAll("passwd='([^\\s]+)'", "passwd='" + vncPassword + "'");
}
xmlDesc = xmlDesc.replaceAll(GRAPHICS_ELEM_START + CONTENTS_WILDCARD + GRAPHICS_ELEM_END, graphElem);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Replaced the VNC IP address [%s] with [%s] in VM [%s].", originalGraphElem, graphElem, vmName));
}
logger.debug("Replaced the VNC IP address {} with {} in VM {}.", maskSensitiveInfoInXML(originalGraphElem), maskSensitiveInfoInXML(graphElem), vmName);
}
}
return xmlDesc;
@ -1036,4 +1058,10 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper<MigrateCo
}
return false;
}
public static String maskSensitiveInfoInXML(String xmlDesc) {
if (xmlDesc == null) return null;
return xmlDesc.replaceAll("(graphics\\s+[^>]*type=['\"]vnc['\"][^>]*passwd=['\"])([^'\"]*)(['\"])",
"$1*****$3");
}
}

View File

@ -84,8 +84,9 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper<StartComman
}
libvirtComputingResource.createVifs(vmSpec, vm);
logger.debug("starting " + vmName + ": " + vm.toString());
if (logger.isDebugEnabled()) {
logger.debug("Starting {} : {}", vmName, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(vm.toString()));
}
String vmInitialSpecification = vm.toString();
String vmFinalSpecification = performXmlTransformHook(vmInitialSpecification, libvirtComputingResource);
libvirtComputingResource.startVM(conn, vmName, vmFinalSpecification);

View File

@ -658,7 +658,7 @@ public class LibvirtMigrateCommandWrapperTest {
@Test
public void testReplaceIpForVNCInDescFile() {
final String targetIp = "192.168.22.21";
final String result = libvirtMigrateCmdWrapper.replaceIpForVNCInDescFileAndNormalizePassword(fullfile, targetIp, null, "");
final String result = libvirtMigrateCmdWrapper.replaceIpForVNCInDescFileAndNormalizePassword(fullfile, targetIp, "vncSecretPwd", "");
assertEquals("transformation does not live up to expectation:\n" + result, targetfile, result);
}
@ -1089,6 +1089,30 @@ public class LibvirtMigrateCommandWrapperTest {
Assert.assertTrue(finalXml.contains(newIsoVolumePath));
}
@Test
public void testMaskVncPwdDomain() {
// Test case 1: Single quotes
String xml1 = "<graphics type='vnc' port='5900' passwd='secret123'/>";
String expected1 = "<graphics type='vnc' port='5900' passwd='*****'/>";
assertEquals(expected1, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml1));
// Test case 2: Double quotes
String xml2 = "<graphics type=\"vnc\" port=\"5901\" passwd=\"mypassword\"/>";
String expected2 = "<graphics type=\"vnc\" port=\"5901\" passwd=\"*****\"/>";
assertEquals(expected2, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml2));
// Test case 3: Non-VNC graphics (should remain unchanged)
String xml3 = "<graphics type='spice' port='5902' passwd='notvnc'/>";
assertEquals(xml3, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml3));
// Test case 4: Multiple VNC entries in one string
String xml4 = "<graphics type='vnc' port='5900' passwd='a'/>\n" +
"<graphics type='vnc' port='5901' passwd='b'/>";
String expected4 = "<graphics type='vnc' port='5900' passwd='*****'/>\n" +
"<graphics type='vnc' port='5901' passwd='*****'/>";
assertEquals(expected4, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml4));
}
@Test
public void updateGpuDevicesIfNeededTestNoGpuDevice() throws Exception {
Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine();

View File

@ -0,0 +1,83 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
import org.apache.cloudstack.api.InternalIdentity;
@Entity
@Table(name = "kubernetes_cluster_affinity_group_map")
public class KubernetesClusterAffinityGroupMapVO implements InternalIdentity {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
@Column(name = "id")
private Long id;
@Column(name = "cluster_id")
private long clusterId;
@Column(name = "node_type")
private String nodeType;
@Column(name = "affinity_group_id")
private long affinityGroupId;
public KubernetesClusterAffinityGroupMapVO() {
}
public KubernetesClusterAffinityGroupMapVO(long clusterId, String nodeType, long affinityGroupId) {
this.clusterId = clusterId;
this.nodeType = nodeType;
this.affinityGroupId = affinityGroupId;
}
@Override
public long getId() {
return id;
}
public long getClusterId() {
return clusterId;
}
public void setClusterId(long clusterId) {
this.clusterId = clusterId;
}
public String getNodeType() {
return nodeType;
}
public void setNodeType(String nodeType) {
this.nodeType = nodeType;
}
public long getAffinityGroupId() {
return affinityGroupId;
}
public void setAffinityGroupId(long affinityGroupId) {
this.affinityGroupId = affinityGroupId;
}
}

View File

@ -169,6 +169,7 @@ import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterScaleWorker;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStopWorker;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterUpgradeWorker;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
@ -315,6 +316,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
@Inject
public KubernetesClusterDetailsDao kubernetesClusterDetailsDao;
@Inject
public KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao;
@Inject
public KubernetesSupportedVersionDao kubernetesSupportedVersionDao;
@Inject
protected SSHKeyPairDao sshKeyPairDao;
@ -858,24 +861,38 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
List<KubernetesUserVmResponse> vmResponses = new ArrayList<>();
List<KubernetesClusterVmMapVO> vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
ResponseView respView = ResponseView.Restricted;
ResponseView userVmResponseView = ResponseView.Restricted;
Account caller = CallContext.current().getCallingAccount();
if (accountService.isRootAdmin(caller.getId())) {
respView = ResponseView.Full;
userVmResponseView = ResponseView.Full;
}
final String responseName = "virtualmachine";
if (vmList != null && !vmList.isEmpty()) {
for (KubernetesClusterVmMapVO vmMapVO : vmList) {
UserVmJoinVO userVM = userVmJoinDao.findById(vmMapVO.getVmId());
if (userVM != null) {
UserVmResponse vmResponse = ApiDBUtils.newUserVmResponse(respView, responseName, userVM,
EnumSet.of(VMDetails.nics), caller);
Map<Long, KubernetesClusterVmMapVO> vmMapById = vmList.stream()
.collect(Collectors.toMap(KubernetesClusterVmMapVO::getVmId, vm -> vm));
Long[] vmIds = vmMapById.keySet().toArray(new Long[0]);
List<UserVmJoinVO> userVmJoinVOs = userVmJoinDao.searchByIds(vmIds);
if (userVmJoinVOs != null && !userVmJoinVOs.isEmpty()) {
Map<Long, UserVmResponse> vmResponseMap = new HashMap<>();
for (UserVmJoinVO userVM : userVmJoinVOs) {
Long vmId = userVM.getId();
UserVmResponse vmResponse = vmResponseMap.get(vmId);
if (vmResponse == null) {
vmResponse = ApiDBUtils.newUserVmResponse(userVmResponseView, responseName, userVM,
EnumSet.of(VMDetails.nics, VMDetails.affgrp), caller);
vmResponseMap.put(vmId, vmResponse);
} else {
ApiDBUtils.fillVmDetails(userVmResponseView, vmResponse, userVM);
}
}
for (Map.Entry<Long, UserVmResponse> vmIdResponseEntry : vmResponseMap.entrySet()) {
KubernetesUserVmResponse kubernetesUserVmResponse = new KubernetesUserVmResponse();
try {
BeanUtils.copyProperties(kubernetesUserVmResponse, vmResponse);
BeanUtils.copyProperties(kubernetesUserVmResponse, vmIdResponseEntry.getValue());
} catch (IllegalAccessException | InvocationTargetException e) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate zone metrics response");
}
KubernetesClusterVmMapVO vmMapVO = vmMapById.get(vmIdResponseEntry.getKey());
kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode());
kubernetesUserVmResponse.setEtcdNode(vmMapVO.isEtcdNode());
kubernetesUserVmResponse.setNodeVersion(vmMapVO.getNodeVersion());
@ -905,10 +922,45 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
response.setClusterType(kubernetesCluster.getClusterType());
response.setCsiEnabled(kubernetesCluster.isCsiEnabled());
response.setCreated(kubernetesCluster.getCreated());
setNodeTypeAffinityGroupResponse(response, kubernetesCluster.getId());
return response;
}
protected void setNodeTypeAffinityGroupResponse(KubernetesClusterResponse response, long clusterId) {
setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name());
setAffinityGroupResponseForNodeType(response, clusterId, WORKER.name());
setAffinityGroupResponseForNodeType(response, clusterId, ETCD.name());
}
protected void setAffinityGroupResponseForNodeType(KubernetesClusterResponse response, long clusterId, String nodeType) {
List<Long> affinityGroupIds = kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, nodeType);
if (affinityGroupIds == null || affinityGroupIds.isEmpty()) {
return;
}
List<String> affinityGroupUuids = new ArrayList<>();
List<String> affinityGroupNames = new ArrayList<>();
for (Long affinityGroupId : affinityGroupIds) {
AffinityGroupVO affinityGroup = affinityGroupDao.findById(affinityGroupId);
if (affinityGroup != null) {
affinityGroupUuids.add(affinityGroup.getUuid());
affinityGroupNames.add(affinityGroup.getName());
}
}
String affinityGroupUuidsCsv = String.join(",", affinityGroupUuids);
String affinityGroupNamesCsv = String.join(",", affinityGroupNames);
if (CONTROL.name().equals(nodeType)) {
response.setControlAffinityGroupIds(affinityGroupUuidsCsv);
response.setControlAffinityGroupNames(affinityGroupNamesCsv);
} else if (WORKER.name().equals(nodeType)) {
response.setWorkerAffinityGroupIds(affinityGroupUuidsCsv);
response.setWorkerAffinityGroupNames(affinityGroupNamesCsv);
} else if (ETCD.name().equals(nodeType)) {
response.setEtcdAffinityGroupIds(affinityGroupUuidsCsv);
response.setEtcdAffinityGroupNames(affinityGroupNamesCsv);
}
}
private DataCenter validateAndGetZoneForKubernetesCreateParameters(Long zoneId, Long networkId) {
DataCenter zone = dataCenterDao.findById(zoneId);
if (zone == null) {
@ -1187,6 +1239,20 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
return network;
}
private void persistAffinityGroupMappings(long clusterId, Map<String, List<Long>> affinityGroupNodeTypeMap) {
if (MapUtils.isEmpty(affinityGroupNodeTypeMap)) {
return;
}
for (Map.Entry<String, List<Long>> nodeTypeAffinityGroupEntry : affinityGroupNodeTypeMap.entrySet()) {
String nodeType = nodeTypeAffinityGroupEntry.getKey();
List<Long> affinityGroupIds = nodeTypeAffinityGroupEntry.getValue();
for (Long affinityGroupId : affinityGroupIds) {
kubernetesClusterAffinityGroupMapDao.persist(
new KubernetesClusterAffinityGroupMapVO(clusterId, nodeType, affinityGroupId));
}
}
}
private void addKubernetesClusterDetails(final KubernetesCluster kubernetesCluster, final Network network, final CreateKubernetesClusterCmd cmd) {
final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress();
final String dockerRegistryUserName = cmd.getDockerRegistryUserName();
@ -1383,8 +1449,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
totalAdditionalVms += additional;
long effectiveCpu = (long) so.getCpu() * so.getSpeed();
totalAdditionalCpuUnits += effectiveCpu * additional;
totalAdditionalCpuUnits += so.getCpu() * additional;
totalAdditionalRamMb += so.getRamSize() * additional;
try {
@ -1627,6 +1692,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
Map<String, Long> templateNodeTypeMap = cmd.getTemplateNodeTypeMap();
Map<String, List<Long>> affinityGroupNodeTypeMap = cmd.getAffinityGroupNodeTypeMap();
final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, DEFAULT, clusterKubernetesVersion);
final VMTemplateVO controlNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, CONTROL, clusterKubernetesVersion);
final VMTemplateVO workerNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, WORKER, clusterKubernetesVersion);
@ -1672,6 +1738,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
newCluster.setCsiEnabled(cmd.getEnableCsi());
kubernetesClusterDao.persist(newCluster);
persistAffinityGroupMappings(newCluster.getId(), affinityGroupNodeTypeMap);
addKubernetesClusterDetails(newCluster, defaultNetwork, cmd);
return newCluster;
}
@ -2289,6 +2356,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
if (validNodeIds.isEmpty()) {
throw new CloudRuntimeException("No valid nodes found to be added to the Kubernetes cluster");
}
validateNodeAffinityGroups(validNodeIds, kubernetesCluster);
KubernetesClusterAddWorker addWorker = new KubernetesClusterAddWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
addWorker = ComponentContext.inject(addWorker);
return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr(), cmd.isManualUpgrade());
@ -2348,6 +2416,83 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
return validNodeIds;
}
protected void validateNodeAffinityGroups(List<Long> nodeIds, KubernetesCluster cluster) {
List<Long> workerAffinityGroupIds = kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(
cluster.getId(), WORKER.name());
if (CollectionUtils.isEmpty(workerAffinityGroupIds)) {
return;
}
List<KubernetesClusterVmMapVO> existingWorkerVms = kubernetesClusterVmMapDao.listByClusterIdAndVmType(cluster.getId(), WORKER);
Set<Long> existingWorkerHostIds = new HashSet<>();
for (KubernetesClusterVmMapVO workerVmMap : existingWorkerVms) {
VMInstanceVO workerVm = vmInstanceDao.findById(workerVmMap.getVmId());
if (workerVm != null && workerVm.getHostId() != null) {
existingWorkerHostIds.add(workerVm.getHostId());
}
}
for (Long affinityGroupId : workerAffinityGroupIds) {
AffinityGroupVO affinityGroup = affinityGroupDao.findById(affinityGroupId);
if (affinityGroup == null) {
continue;
}
String affinityGroupType = affinityGroup.getType();
for (Long nodeId : nodeIds) {
VMInstanceVO node = vmInstanceDao.findById(nodeId);
if (node == null || node.getHostId() == null) {
continue;
}
Long nodeHostId = node.getHostId();
HostVO nodeHost = hostDao.findById(nodeHostId);
String nodeHostName = nodeHost != null ? nodeHost.getName() : String.valueOf(nodeHostId);
if ("host anti-affinity".equalsIgnoreCase(affinityGroupType)) {
if (existingWorkerHostIds.contains(nodeHostId)) {
throw new InvalidParameterValueException(String.format(
"Cannot add VM %s to cluster %s. VM is running on host %s which violates the cluster's " +
"host anti-affinity rule (affinity group: %s). Existing worker VMs are already running on this host.",
node.getInstanceName(), cluster.getName(), nodeHostName, affinityGroup.getName()));
}
} else if ("host affinity".equalsIgnoreCase(affinityGroupType)) {
if (!existingWorkerHostIds.isEmpty() && !existingWorkerHostIds.contains(nodeHostId)) {
List<String> existingHostNames = new ArrayList<>();
for (Long hostId : existingWorkerHostIds) {
HostVO host = hostDao.findById(hostId);
existingHostNames.add(host != null ? host.getName() : String.valueOf(hostId));
}
throw new InvalidParameterValueException(String.format(
"Cannot add VM %s to cluster %s. VM is running on host %s which violates the cluster's " +
"host affinity rule (affinity group: %s). All worker VMs must run on the same host. " +
"Existing workers are on host(s): %s.",
node.getInstanceName(), cluster.getName(), nodeHostName, affinityGroup.getName(),
String.join(", ", existingHostNames)));
}
}
}
if ("host anti-affinity".equalsIgnoreCase(affinityGroupType)) {
Set<Long> newNodeHostIds = new HashSet<>();
for (Long nodeId : nodeIds) {
VMInstanceVO node = vmInstanceDao.findById(nodeId);
if (node != null && node.getHostId() != null) {
Long nodeHostId = node.getHostId();
if (newNodeHostIds.contains(nodeHostId)) {
HostVO nodeHost = hostDao.findById(nodeHostId);
String nodeHostName = nodeHost != null ? nodeHost.getName() : String.valueOf(nodeHostId);
throw new InvalidParameterValueException(String.format(
"Cannot add VM %s to cluster %s. Multiple VMs being added are running on the same host %s, " +
"which violates the cluster's host anti-affinity rule (affinity group: %s).",
node.getInstanceName(), cluster.getName(), nodeHostName, affinityGroup.getName()));
}
newNodeHostIds.add(nodeHostId);
}
}
}
}
}
@Override
public List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd) {
if (!KubernetesServiceEnabled.value()) {

View File

@ -18,12 +18,16 @@ package com.cloud.kubernetes.cluster;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.inject.Inject;
import org.apache.cloudstack.affinity.AffinityGroup;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.dao.ServiceOfferingDao;
@ -66,6 +70,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet
@Inject
protected VMTemplateDao vmTemplateDao;
@Inject
protected AffinityGroupDao affinityGroupDao;
@Inject
KubernetesClusterService kubernetesClusterService;
protected void setEventTypeEntityDetails(Class<?> eventTypeDefinedClass, Class<?> entityClass) {
@ -244,6 +250,81 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet
return mapping;
}
protected void checkNodeTypeAffinityGroupEntryCompleteness(String nodeType, String affinityGroupUuids) {
if (StringUtils.isAnyBlank(nodeType, affinityGroupUuids)) {
String error = String.format("Any Node Type to Affinity Group entry should have a valid '%s' and '%s' values",
VmDetailConstants.CKS_NODE_TYPE, VmDetailConstants.AFFINITY_GROUP);
logger.error(error);
throw new InvalidParameterValueException(error);
}
}
protected void checkNodeTypeAffinityGroupEntryNodeType(String nodeType) {
if (!isValidNodeType(nodeType)) {
String error = String.format("The provided value '%s' for Node Type is invalid", nodeType);
logger.error(error);
throw new InvalidParameterValueException(error);
}
}
protected Long validateAffinityGroupUuidAndGetId(String affinityGroupUuid) {
if (StringUtils.isBlank(affinityGroupUuid)) {
String error = "Empty affinity group UUID provided";
logger.error(error);
throw new InvalidParameterValueException(error);
}
AffinityGroup affinityGroup = affinityGroupDao.findByUuid(affinityGroupUuid);
if (affinityGroup == null) {
String error = String.format("Cannot find an affinity group with ID %s", affinityGroupUuid);
logger.error(error);
throw new InvalidParameterValueException(error);
}
return affinityGroup.getId();
}
protected List<Long> validateAndGetAffinityGroupIds(String affinityGroupUuids) {
String[] uuids = affinityGroupUuids.split(",");
List<Long> affinityGroupIds = new ArrayList<>();
for (String uuid : uuids) {
String trimmedUuid = uuid.trim();
Long affinityGroupId = validateAffinityGroupUuidAndGetId(trimmedUuid);
affinityGroupIds.add(affinityGroupId);
}
return affinityGroupIds;
}
protected void addNodeTypeAffinityGroupEntry(String nodeType, List<Long> affinityGroupIds, Map<String, List<Long>> nodeTypeToAffinityGroupIds) {
if (logger.isDebugEnabled()) {
logger.debug(String.format("Node Type: '%s' should use affinity group IDs: '%s'", nodeType, affinityGroupIds));
}
KubernetesClusterNodeType clusterNodeType = KubernetesClusterNodeType.valueOf(nodeType.toUpperCase());
nodeTypeToAffinityGroupIds.put(clusterNodeType.name(), affinityGroupIds);
}
protected void processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(Map<String, String> nodeTypeAffinityConfig, Map<String, List<Long>> nodeTypeToAffinityGroupIds) {
if (MapUtils.isEmpty(nodeTypeAffinityConfig)) {
return;
}
String nodeType = nodeTypeAffinityConfig.get(VmDetailConstants.CKS_NODE_TYPE);
String affinityGroupUuids = nodeTypeAffinityConfig.get(VmDetailConstants.AFFINITY_GROUP);
checkNodeTypeAffinityGroupEntryCompleteness(nodeType, affinityGroupUuids);
checkNodeTypeAffinityGroupEntryNodeType(nodeType);
List<Long> affinityGroupIds = validateAndGetAffinityGroupIds(affinityGroupUuids);
addNodeTypeAffinityGroupEntry(nodeType, affinityGroupIds, nodeTypeToAffinityGroupIds);
}
@Override
public Map<String, List<Long>> getAffinityGroupNodeTypeMap(Map<String, Map<String, String>> affinityGroupNodeTypeMap) {
Map<String, List<Long>> nodeTypeToAffinityGroupIds = new HashMap<>();
if (MapUtils.isNotEmpty(affinityGroupNodeTypeMap)) {
for (Map<String, String> nodeTypeAffinityConfig : affinityGroupNodeTypeMap.values()) {
processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(nodeTypeAffinityConfig, nodeTypeToAffinityGroupIds);
}
}
return nodeTypeToAffinityGroupIds;
}
public void cleanupForAccount(Account account) {
kubernetesClusterService.cleanupForAccount(account);
}

View File

@ -90,6 +90,7 @@ import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesClusterVO;
import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
@ -217,6 +218,7 @@ public class KubernetesClusterActionWorker {
protected KubernetesClusterDao kubernetesClusterDao;
protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
protected KubernetesClusterDetailsDao kubernetesClusterDetailsDao;
protected KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao;
protected KubernetesSupportedVersionDao kubernetesSupportedVersionDao;
protected KubernetesCluster kubernetesCluster;
@ -251,6 +253,7 @@ public class KubernetesClusterActionWorker {
this.kubernetesClusterDao = clusterManager.kubernetesClusterDao;
this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao;
this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao;
this.kubernetesClusterAffinityGroupMapDao = clusterManager.kubernetesClusterAffinityGroupMapDao;
this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao;
this.manager = clusterManager;
}
@ -1112,4 +1115,18 @@ public class KubernetesClusterActionWorker {
}
return null;
}
protected List<Long> getAffinityGroupIdsForNodeType(KubernetesClusterNodeType nodeType) {
return new ArrayList<>(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(
kubernetesCluster.getId(), nodeType.name()));
}
protected List<Long> getMergedAffinityGroupIds(KubernetesClusterNodeType nodeType, Long domainId, Long accountId) {
List<Long> affinityGroupIds = getAffinityGroupIdsForNodeType(nodeType);
Long explicitAffinityGroupId = getExplicitAffinityGroup(domainId, accountId);
if (explicitAffinityGroupId != null && !affinityGroupIds.contains(explicitAffinityGroupId)) {
affinityGroupIds.add(explicitAffinityGroupId);
}
return affinityGroupIds.isEmpty() ? null : affinityGroupIds;
}
}

View File

@ -348,6 +348,7 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
annotationDao.removeByEntityType(AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), kubernetesCluster.getUuid());
kubernetesClusterDetailsDao.removeDetails(kubernetesCluster.getId());
kubernetesClusterAffinityGroupMapDao.removeByClusterId(kubernetesCluster.getId());
boolean deleted = kubernetesClusterDao.remove(kubernetesCluster.getId());
if (!deleted) {
logMessage(Level.WARN, String.format("Failed to delete Kubernetes cluster: %s", kubernetesCluster), null);

View File

@ -26,7 +26,6 @@ import static com.cloud.utils.db.Transaction.execute;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -426,21 +425,19 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) {
keypairs.add(kubernetesCluster.getKeyPair());
}
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
List<Long> affinityGroupIds = getMergedAffinityGroupIds(WORKER, domainId, accountId);
if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, List.of(kubernetesCluster.getSecurityGroupId()))) {
List<Long> securityGroupIds = new ArrayList<>();
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
nodeVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, securityGroupIds, owner,
hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs,
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null,
null, true, null, UserVmManager.CKS_NODE, null, null);
} else {
nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, owner,
hostName, hostName, null, null, null, null,
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs,
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
}
if (logger.isInfoEnabled()) {
logger.info("Created node VM : {}, {} in the Kubernetes cluster : {}", hostName, nodeVm, kubernetesCluster.getName());

View File

@ -270,7 +270,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
keypairs.add(kubernetesCluster.getKeyPair());
}
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
List<Long> affinityGroupIds = getMergedAffinityGroupIds(CONTROL, domainId, accountId);
String userDataDetails = kubernetesCluster.getCniConfigDetails();
if (kubernetesCluster.getSecurityGroupId() != null &&
networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds,
@ -279,15 +279,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
controlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner,
hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, userDataId, userDataDetails, keypairs,
requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
requestedIps, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null,
null, true, null, UserVmManager.CKS_NODE, null, null);
} else {
controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner,
hostName, hostName, null, null, null, null,
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, userDataId, userDataDetails, keypairs,
requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
requestedIps, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
}
if (logger.isInfoEnabled()) {
logger.info("Created control VM: {}, {} in the Kubernetes cluster: {}", controlVm, hostName, kubernetesCluster);
@ -439,7 +437,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
keypairs.add(kubernetesCluster.getKeyPair());
}
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
List<Long> affinityGroupIds = getMergedAffinityGroupIds(CONTROL, domainId, accountId);
if (kubernetesCluster.getSecurityGroupId() != null &&
networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds,
List.of(kubernetesCluster.getSecurityGroupId()))) {
@ -447,15 +445,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
additionalControlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner,
hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs,
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null,
null, true, null, UserVmManager.CKS_NODE, null, null);
} else {
additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner,
hostName, hostName, null, null, null, null,
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs,
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
null, addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
}
if (logger.isInfoEnabled()) {
@ -483,7 +479,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) {
keypairs.add(kubernetesCluster.getKeyPair());
}
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
List<Long> affinityGroupIds = getMergedAffinityGroupIds(ETCD, domainId, accountId);
String hostName = etcdNodeHostnames.get(etcdNodeIndex);
Map<String, String> customParameterMap = new HashMap<String, String>();
if (zone.isSecurityGroupEnabled()) {
@ -491,15 +487,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
etcdNode = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, securityGroupIds, owner,
hostName, hostName, null, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs,
Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, affinityGroupIds, customParameterMap, null, null, null,
null, true, null, null, null, null);
} else {
etcdNode = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, owner,
hostName, hostName, null, null, null, null,
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs,
Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, affinityGroupIds, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null, null, null);
}
if (logger.isInfoEnabled()) {

View File

@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster.dao;
import java.util.List;
import com.cloud.kubernetes.cluster.KubernetesClusterAffinityGroupMapVO;
import com.cloud.utils.db.GenericDao;
public interface KubernetesClusterAffinityGroupMapDao extends GenericDao<KubernetesClusterAffinityGroupMapVO, Long> {
List<KubernetesClusterAffinityGroupMapVO> listByClusterIdAndNodeType(long clusterId, String nodeType);
List<Long> listAffinityGroupIdsByClusterIdAndNodeType(long clusterId, String nodeType);
int removeByClusterId(long clusterId);
}

View File

@ -0,0 +1,67 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster.dao;
import java.util.List;
import java.util.stream.Collectors;
import org.springframework.stereotype.Component;
import com.cloud.kubernetes.cluster.KubernetesClusterAffinityGroupMapVO;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
@Component
public class KubernetesClusterAffinityGroupMapDaoImpl extends GenericDaoBase<KubernetesClusterAffinityGroupMapVO, Long>
implements KubernetesClusterAffinityGroupMapDao {
private final SearchBuilder<KubernetesClusterAffinityGroupMapVO> clusterIdAndNodeTypeSearch;
private final SearchBuilder<KubernetesClusterAffinityGroupMapVO> clusterIdSearch;
public KubernetesClusterAffinityGroupMapDaoImpl() {
clusterIdAndNodeTypeSearch = createSearchBuilder();
clusterIdAndNodeTypeSearch.and("clusterId", clusterIdAndNodeTypeSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
clusterIdAndNodeTypeSearch.and("nodeType", clusterIdAndNodeTypeSearch.entity().getNodeType(), SearchCriteria.Op.EQ);
clusterIdAndNodeTypeSearch.done();
clusterIdSearch = createSearchBuilder();
clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
clusterIdSearch.done();
}
@Override
public List<KubernetesClusterAffinityGroupMapVO> listByClusterIdAndNodeType(long clusterId, String nodeType) {
SearchCriteria<KubernetesClusterAffinityGroupMapVO> sc = clusterIdAndNodeTypeSearch.create();
sc.setParameters("clusterId", clusterId);
sc.setParameters("nodeType", nodeType);
return listBy(sc);
}
@Override
public List<Long> listAffinityGroupIdsByClusterIdAndNodeType(long clusterId, String nodeType) {
List<KubernetesClusterAffinityGroupMapVO> maps = listByClusterIdAndNodeType(clusterId, nodeType);
return maps.stream().map(KubernetesClusterAffinityGroupMapVO::getAffinityGroupId).collect(Collectors.toList());
}
@Override
public int removeByClusterId(long clusterId) {
SearchCriteria<KubernetesClusterAffinityGroupMapVO> sc = clusterIdSearch.create();
sc.setParameters("clusterId", clusterId);
return remove(sc);
}
}

View File

@ -37,6 +37,9 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.cloud.api.query.dao.TemplateJoinDao;
@ -57,6 +60,7 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplateZoneDao;
import com.cloud.template.TemplateApiService;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ComponentContext;
@ -84,12 +88,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
@Inject
private DataCenterDao dataCenterDao;
@Inject
private ImageStoreDao imageStoreDao;
@Inject
private TemplateApiService templateService;
public static final String MINIMUN_AUTOSCALER_SUPPORTED_VERSION = "1.15.0";
protected void updateTemplateDetailsInKubernetesSupportedVersionResponse(
final KubernetesSupportedVersion kubernetesSupportedVersion, KubernetesSupportedVersionResponse response) {
final KubernetesSupportedVersion kubernetesSupportedVersion, KubernetesSupportedVersionResponse response, boolean isRootAdmin) {
TemplateJoinVO template = templateJoinDao.findById(kubernetesSupportedVersion.getIsoId());
if (template == null) {
return;
@ -99,11 +105,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
if (template.getState() != null) {
response.setIsoState(template.getState().toString());
}
if (isRootAdmin) {
response.setIsoUrl(template.getUrl());
}
response.setIsoArch(template.getArch().getType());
response.setDirectDownload(template.isDirectDownload());
}
private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) {
private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion, boolean isRootAdmin) {
KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse();
response.setObjectName("kubernetessupportedversion");
response.setId(kubernetesSupportedVersion.getUuid());
@ -122,7 +131,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
response.setSupportsHA(compareSemanticVersions(kubernetesSupportedVersion.getSemanticVersion(),
KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0);
response.setSupportsAutoscaling(versionSupportsAutoscaling(kubernetesSupportedVersion));
updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response);
updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response, isRootAdmin);
response.setCreated(kubernetesSupportedVersion.getCreated());
return response;
}
@ -130,8 +139,11 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
private ListResponse<KubernetesSupportedVersionResponse> createKubernetesSupportedVersionListResponse(
List<KubernetesSupportedVersionVO> versions, Integer count) {
List<KubernetesSupportedVersionResponse> responseList = new ArrayList<>();
Account caller = CallContext.current().getCallingAccount();
boolean isRootAdmin = accountManager.isRootAdmin(caller.getId());
for (KubernetesSupportedVersionVO version : versions) {
responseList.add(createKubernetesSupportedVersionResponse(version));
responseList.add(createKubernetesSupportedVersionResponse(version, isRootAdmin));
}
ListResponse<KubernetesSupportedVersionResponse> response = new ListResponse<>();
response.setResponses(responseList, count);
@ -347,6 +359,32 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
return createKubernetesSupportedVersionListResponse(versions, versionsAndCount.second());
}
private void validateImageStoreForZone(Long zoneId, boolean directDownload) {
if (directDownload) {
return;
}
if (zoneId != null) {
List<ImageStoreVO> imageStores = imageStoreDao.listStoresByZoneId(zoneId);
if (CollectionUtils.isEmpty(imageStores)) {
DataCenterVO zone = dataCenterDao.findById(zoneId);
String zoneName = zone != null ? zone.getName() : String.valueOf(zoneId);
throw new InvalidParameterValueException(String.format("Unable to register Kubernetes version ISO. No image store available in zone: %s", zoneName));
}
} else {
List<DataCenterVO> zones = dataCenterDao.listAllZones();
List<String> zonesWithoutStorage = new ArrayList<>();
for (DataCenterVO zone : zones) {
List<ImageStoreVO> imageStores = imageStoreDao.listStoresByZoneId(zone.getId());
if (CollectionUtils.isEmpty(imageStores)) {
zonesWithoutStorage.add(zone.getName());
}
}
if (!zonesWithoutStorage.isEmpty()) {
throw new InvalidParameterValueException(String.format("Unable to register Kubernetes version ISO for all zones. The following zones have no image store: %s", String.join(", ", zonesWithoutStorage)));
}
}
}
private void validateKubernetesSupportedVersion(Long zoneId, String semanticVersion, Integer minimumCpu,
Integer minimumRamSize, boolean isDirectDownload) {
if (minimumCpu == null || minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) {
@ -398,6 +436,8 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
}
}
validateImageStoreForZone(zoneId, isDirectDownload);
VMTemplateVO template = null;
try {
VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(zoneId, name, isoUrl, isoChecksum, isDirectDownload, arch);
@ -411,7 +451,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO);
CallContext.current().putContextParameter(KubernetesSupportedVersion.class, supportedVersionVO.getUuid());
return createKubernetesSupportedVersionResponse(supportedVersionVO);
return createKubernetesSupportedVersionResponse(supportedVersionVO, true);
}
@Override
@ -496,7 +536,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne
}
version = kubernetesSupportedVersionDao.findById(versionId);
}
return createKubernetesSupportedVersionResponse(version);
return createKubernetesSupportedVersionResponse(version, true);
}
@Override

View File

@ -17,6 +17,7 @@
package org.apache.cloudstack.api.command.user.kubernetes.cluster;
import java.security.InvalidParameterException;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -79,7 +80,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
@Inject
public KubernetesClusterService kubernetesClusterService;
@Inject
protected KubernetesServiceHelper kubernetesClusterHelper;
protected KubernetesServiceHelper kubernetesServiceHelper;
@Inject
private ConfigurationDao configurationDao;
@Inject
@ -125,6 +126,12 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
since = "4.21.0")
private Map<String, Map<String, String>> templateNodeTypeMap;
@ACL(accessType = AccessType.UseEntry)
@Parameter(name = ApiConstants.NODE_TYPE_AFFINITY_GROUP_MAP, type = CommandType.MAP,
description = "(Optional) Node Type to Affinity Group ID mapping. If provided, VMs of each node type will be added to the specified affinity group",
since = "4.23.0")
private Map<String, Map<String, String>> affinityGroupNodeTypeMap;
@ACL(accessType = AccessType.UseEntry)
@Parameter(name = ApiConstants.ETCD_NODES, type = CommandType.LONG,
description = "(Optional) Number of Kubernetes cluster etcd nodes, default is 0." +
@ -314,11 +321,15 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
}
public Map<String, Long> getServiceOfferingNodeTypeMap() {
return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap);
return kubernetesServiceHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap);
}
public Map<String, Long> getTemplateNodeTypeMap() {
return kubernetesClusterHelper.getTemplateNodeTypeMap(templateNodeTypeMap);
return kubernetesServiceHelper.getTemplateNodeTypeMap(templateNodeTypeMap);
}
public Map<String, List<Long>> getAffinityGroupNodeTypeMap() {
return kubernetesServiceHelper.getAffinityGroupNodeTypeMap(affinityGroupNodeTypeMap);
}
public Hypervisor.HypervisorType getHypervisorType() {

View File

@ -57,7 +57,7 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd {
@Inject
public KubernetesClusterService kubernetesClusterService;
@Inject
protected KubernetesServiceHelper kubernetesClusterHelper;
protected KubernetesServiceHelper kubernetesServiceHelper;
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
@ -114,7 +114,7 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd {
}
public Map<String, Long> getServiceOfferingNodeTypeMap() {
return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(this.serviceOfferingNodeTypeMap);
return kubernetesServiceHelper.getServiceOfferingNodeTypeMap(this.serviceOfferingNodeTypeMap);
}
public Long getClusterSize() {

View File

@ -220,6 +220,30 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
@Param(description = "The date when this Kubernetes cluster was created")
private Date created;
@SerializedName(ApiConstants.CONTROL_AFFINITY_GROUP_IDS)
@Param(description = "The IDs of affinity groups associated with control nodes", since = "4.23.0")
private String controlAffinityGroupIds;
@SerializedName(ApiConstants.CONTROL_AFFINITY_GROUP_NAMES)
@Param(description = "The names of affinity groups associated with control nodes", since = "4.23.0")
private String controlAffinityGroupNames;
@SerializedName(ApiConstants.WORKER_AFFINITY_GROUP_IDS)
@Param(description = "The IDs of affinity groups associated with worker nodes", since = "4.23.0")
private String workerAffinityGroupIds;
@SerializedName(ApiConstants.WORKER_AFFINITY_GROUP_NAMES)
@Param(description = "The names of affinity groups associated with worker nodes", since = "4.23.0")
private String workerAffinityGroupNames;
@SerializedName(ApiConstants.ETCD_AFFINITY_GROUP_IDS)
@Param(description = "The IDs of affinity groups associated with etcd nodes", since = "4.23.0")
private String etcdAffinityGroupIds;
@SerializedName(ApiConstants.ETCD_AFFINITY_GROUP_NAMES)
@Param(description = "The names of affinity groups associated with etcd nodes", since = "4.23.0")
private String etcdAffinityGroupNames;
public KubernetesClusterResponse() {
}
@ -535,4 +559,28 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
public void setCsiEnabled(Boolean csiEnabled) {
isCsiEnabled = csiEnabled;
}
public void setControlAffinityGroupIds(String controlAffinityGroupIds) {
this.controlAffinityGroupIds = controlAffinityGroupIds;
}
public void setControlAffinityGroupNames(String controlAffinityGroupNames) {
this.controlAffinityGroupNames = controlAffinityGroupNames;
}
public void setWorkerAffinityGroupIds(String workerAffinityGroupIds) {
this.workerAffinityGroupIds = workerAffinityGroupIds;
}
public void setWorkerAffinityGroupNames(String workerAffinityGroupNames) {
this.workerAffinityGroupNames = workerAffinityGroupNames;
}
public void setEtcdAffinityGroupIds(String etcdAffinityGroupIds) {
this.etcdAffinityGroupIds = etcdAffinityGroupIds;
}
public void setEtcdAffinityGroupNames(String etcdAffinityGroupNames) {
this.etcdAffinityGroupNames = etcdAffinityGroupNames;
}
}

View File

@ -50,6 +50,10 @@ public class KubernetesSupportedVersionResponse extends BaseResponse {
@Param(description = "The name of the binaries ISO for Kubernetes supported version")
private String isoName;
@SerializedName(ApiConstants.ISO_URL)
@Param(description = "the URL of the binaries ISO for Kubernetes supported version")
private String isoUrl;
@SerializedName(ApiConstants.ISO_STATE)
@Param(description = "The state of the binaries ISO for Kubernetes supported version")
private String isoState;
@ -134,6 +138,14 @@ public class KubernetesSupportedVersionResponse extends BaseResponse {
this.isoName = isoName;
}
public String getIsoUrl() {
return isoUrl;
}
public void setIsoUrl(String isoUrl) {
this.isoUrl = isoUrl;
}
public String getIsoState() {
return isoState;
}

View File

@ -32,6 +32,7 @@
<bean id="kubernetesClusterDaoImpl" class="com.cloud.kubernetes.cluster.dao.KubernetesClusterDaoImpl" />
<bean id="kubernetesClusterDetailsDaoImpl" class="com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDaoImpl" />
<bean id="kubernetesClusterVmMapDaoImpl" class="com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDaoImpl" />
<bean id="kubernetesClusterAffinityGroupMapDaoImpl" class="com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDaoImpl" />
<bean id="kubernetesClusterManagerImpl" class="com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl" />
<bean id="kubernetesServiceHelper" class="com.cloud.kubernetes.cluster.KubernetesServiceHelperImpl" >

View File

@ -0,0 +1,87 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster;
import org.junit.Assert;
import org.junit.Test;
public class KubernetesClusterAffinityGroupMapVOTest {
@Test
public void testConstructorAndGetters() {
KubernetesClusterAffinityGroupMapVO vo =
new KubernetesClusterAffinityGroupMapVO(1L, "CONTROL", 100L);
Assert.assertEquals(1L, vo.getClusterId());
Assert.assertEquals("CONTROL", vo.getNodeType());
Assert.assertEquals(100L, vo.getAffinityGroupId());
}
@Test
public void testDefaultConstructor() {
KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO();
Assert.assertNotNull(vo);
}
@Test
public void testSetClusterId() {
KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO();
vo.setClusterId(2L);
Assert.assertEquals(2L, vo.getClusterId());
}
@Test
public void testSetNodeType() {
KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO();
vo.setNodeType("WORKER");
Assert.assertEquals("WORKER", vo.getNodeType());
}
@Test
public void testSetAffinityGroupId() {
KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO();
vo.setAffinityGroupId(200L);
Assert.assertEquals(200L, vo.getAffinityGroupId());
}
@Test
public void testAllNodeTypes() {
KubernetesClusterAffinityGroupMapVO controlVo =
new KubernetesClusterAffinityGroupMapVO(1L, "CONTROL", 10L);
KubernetesClusterAffinityGroupMapVO workerVo =
new KubernetesClusterAffinityGroupMapVO(1L, "WORKER", 20L);
KubernetesClusterAffinityGroupMapVO etcdVo =
new KubernetesClusterAffinityGroupMapVO(1L, "ETCD", 30L);
Assert.assertEquals("CONTROL", controlVo.getNodeType());
Assert.assertEquals("WORKER", workerVo.getNodeType());
Assert.assertEquals("ETCD", etcdVo.getNodeType());
}
@Test
public void testSettersChain() {
KubernetesClusterAffinityGroupMapVO vo = new KubernetesClusterAffinityGroupMapVO();
vo.setClusterId(5L);
vo.setNodeType("ETCD");
vo.setAffinityGroupId(500L);
Assert.assertEquals(5L, vo.getClusterId());
Assert.assertEquals("ETCD", vo.getNodeType());
Assert.assertEquals(500L, vo.getAffinityGroupId());
}
}

View File

@ -1,145 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.vm.VmDetailConstants;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD;
import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER;
@RunWith(MockitoJUnitRunner.class)
public class KubernetesClusterHelperImplTest {
@Mock
private ServiceOfferingDao serviceOfferingDao;
@Mock
private ServiceOfferingVO workerServiceOffering;
@Mock
private ServiceOfferingVO controlServiceOffering;
@Mock
private ServiceOfferingVO etcdServiceOffering;
private static final String workerNodesOfferingId = UUID.randomUUID().toString();
private static final String controlNodesOfferingId = UUID.randomUUID().toString();
private static final String etcdNodesOfferingId = UUID.randomUUID().toString();
private static final Long workerOfferingId = 1L;
private static final Long controlOfferingId = 2L;
private static final Long etcdOfferingId = 3L;
private final KubernetesServiceHelperImpl helper = new KubernetesServiceHelperImpl();
@Before
public void setUp() {
helper.serviceOfferingDao = serviceOfferingDao;
Mockito.when(serviceOfferingDao.findByUuid(workerNodesOfferingId)).thenReturn(workerServiceOffering);
Mockito.when(serviceOfferingDao.findByUuid(controlNodesOfferingId)).thenReturn(controlServiceOffering);
Mockito.when(serviceOfferingDao.findByUuid(etcdNodesOfferingId)).thenReturn(etcdServiceOffering);
Mockito.when(workerServiceOffering.getId()).thenReturn(workerOfferingId);
Mockito.when(controlServiceOffering.getId()).thenReturn(controlOfferingId);
Mockito.when(etcdServiceOffering.getId()).thenReturn(etcdOfferingId);
}
@Test
public void testIsValidNodeTypeEmptyNodeType() {
Assert.assertFalse(helper.isValidNodeType(null));
}
@Test
public void testIsValidNodeTypeInvalidNodeType() {
String nodeType = "invalidNodeType";
Assert.assertFalse(helper.isValidNodeType(nodeType));
}
@Test
public void testIsValidNodeTypeValidNodeTypeLowercase() {
String nodeType = KubernetesServiceHelper.KubernetesClusterNodeType.WORKER.name().toLowerCase();
Assert.assertTrue(helper.isValidNodeType(nodeType));
}
private Map<String, String> createMapEntry(KubernetesServiceHelper.KubernetesClusterNodeType nodeType,
String nodeTypeOfferingUuid) {
Map<String, String> map = new HashMap<>();
map.put(VmDetailConstants.CKS_NODE_TYPE, nodeType.name().toLowerCase());
map.put(VmDetailConstants.OFFERING, nodeTypeOfferingUuid);
return map;
}
@Test
public void testNodeOfferingMap() {
Map<String, Map<String, String>> serviceOfferingNodeTypeMap = new HashMap<>();
Map<String, String> firstMap = createMapEntry(WORKER, workerNodesOfferingId);
Map<String, String> secondMap = createMapEntry(CONTROL, controlNodesOfferingId);
serviceOfferingNodeTypeMap.put("map1", firstMap);
serviceOfferingNodeTypeMap.put("map2", secondMap);
Map<String, Long> map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap);
Assert.assertNotNull(map);
Assert.assertEquals(2, map.size());
Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(CONTROL.name()));
Assert.assertEquals(workerOfferingId, map.get(WORKER.name()));
Assert.assertEquals(controlOfferingId, map.get(CONTROL.name()));
}
@Test
public void testNodeOfferingMapNullMap() {
Map<String, Long> map = helper.getServiceOfferingNodeTypeMap(null);
Assert.assertTrue(map.isEmpty());
}
@Test
public void testNodeOfferingMapEtcdNodes() {
Map<String, Map<String, String>> serviceOfferingNodeTypeMap = new HashMap<>();
Map<String, String> firstMap = createMapEntry(ETCD, etcdNodesOfferingId);
serviceOfferingNodeTypeMap.put("map1", firstMap);
Map<String, Long> map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap);
Assert.assertNotNull(map);
Assert.assertEquals(1, map.size());
Assert.assertTrue(map.containsKey(ETCD.name()));
Assert.assertEquals(etcdOfferingId, map.get(ETCD.name()));
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeOfferingEntryCompletenessInvalidParameters() {
helper.checkNodeTypeOfferingEntryCompleteness(WORKER.name(), null);
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeOfferingEntryValuesInvalidNodeType() {
String invalidNodeType = "invalidNodeTypeName";
helper.checkNodeTypeOfferingEntryValues(invalidNodeType, workerServiceOffering, workerNodesOfferingId);
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeOfferingEntryValuesEmptyOffering() {
String nodeType = WORKER.name();
helper.checkNodeTypeOfferingEntryValues(nodeType, null, workerNodesOfferingId);
}
}

View File

@ -26,6 +26,7 @@ import com.cloud.dc.DataCenter;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.kubernetes.version.KubernetesSupportedVersion;
@ -46,9 +47,14 @@ import com.cloud.utils.Pair;
import com.cloud.utils.net.NetUtils;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import org.apache.cloudstack.affinity.AffinityGroupVO;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.commons.collections.MapUtils;
@ -103,6 +109,15 @@ public class KubernetesClusterManagerImplTest {
@Mock
private ServiceOfferingDao serviceOfferingDao;
@Mock
private KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao;
@Mock
private AffinityGroupDao affinityGroupDao;
@Mock
private HostDao hostDao;
@Spy
@InjectMocks
KubernetesClusterManagerImpl kubernetesClusterManager;
@ -441,4 +456,462 @@ public class KubernetesClusterManagerImplTest {
String cksClusterPreferredArch = kubernetesClusterManager.getCksClusterPreferredArch(systemVMArch, cksIso);
Assert.assertEquals(CPU.CPUArch.amd64.getType(), cksClusterPreferredArch);
}
@Test
public void testSetAffinityGroupResponseForNodeTypeControl() {
KubernetesClusterResponse response = new KubernetesClusterResponse();
long clusterId = 1L;
AffinityGroupVO ag1 = Mockito.mock(AffinityGroupVO.class);
AffinityGroupVO ag2 = Mockito.mock(AffinityGroupVO.class);
Mockito.when(ag1.getUuid()).thenReturn("uuid-1");
Mockito.when(ag1.getName()).thenReturn("affinity-group-1");
Mockito.when(ag2.getUuid()).thenReturn("uuid-2");
Mockito.when(ag2.getName()).thenReturn("affinity-group-2");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name()))
.thenReturn(Arrays.asList(1L, 2L));
Mockito.when(affinityGroupDao.findById(1L)).thenReturn(ag1);
Mockito.when(affinityGroupDao.findById(2L)).thenReturn(ag2);
kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name());
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name());
Mockito.verify(affinityGroupDao).findById(1L);
Mockito.verify(affinityGroupDao).findById(2L);
}
@Test
public void testSetAffinityGroupResponseForNodeTypeWorker() {
KubernetesClusterResponse response = new KubernetesClusterResponse();
long clusterId = 1L;
AffinityGroupVO ag = Mockito.mock(AffinityGroupVO.class);
Mockito.when(ag.getUuid()).thenReturn("worker-uuid");
Mockito.when(ag.getName()).thenReturn("worker-affinity");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(ag);
kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, WORKER.name());
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, WORKER.name());
Mockito.verify(affinityGroupDao).findById(10L);
}
@Test
public void testSetAffinityGroupResponseForNodeTypeEtcd() {
KubernetesClusterResponse response = new KubernetesClusterResponse();
long clusterId = 1L;
AffinityGroupVO ag = Mockito.mock(AffinityGroupVO.class);
Mockito.when(ag.getUuid()).thenReturn("etcd-uuid");
Mockito.when(ag.getName()).thenReturn("etcd-affinity");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name()))
.thenReturn(Arrays.asList(20L));
Mockito.when(affinityGroupDao.findById(20L)).thenReturn(ag);
kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, ETCD.name());
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name());
Mockito.verify(affinityGroupDao).findById(20L);
}
@Test
public void testSetAffinityGroupResponseForNodeTypeEmptyList() {
KubernetesClusterResponse response = new KubernetesClusterResponse();
long clusterId = 1L;
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name()))
.thenReturn(Collections.emptyList());
kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name());
Mockito.verify(affinityGroupDao, Mockito.never()).findById(Mockito.anyLong());
}
@Test
public void testSetAffinityGroupResponseForNodeTypeNullList() {
KubernetesClusterResponse response = new KubernetesClusterResponse();
long clusterId = 1L;
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name()))
.thenReturn(null);
kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, ETCD.name());
Mockito.verify(affinityGroupDao, Mockito.never()).findById(Mockito.anyLong());
}
@Test
public void testSetAffinityGroupResponseForNodeTypeNullAffinityGroup() {
KubernetesClusterResponse response = new KubernetesClusterResponse();
long clusterId = 1L;
AffinityGroupVO ag1 = Mockito.mock(AffinityGroupVO.class);
Mockito.when(ag1.getUuid()).thenReturn("uuid-1");
Mockito.when(ag1.getName()).thenReturn("affinity-group-1");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name()))
.thenReturn(Arrays.asList(1L, 2L));
Mockito.when(affinityGroupDao.findById(1L)).thenReturn(ag1);
Mockito.when(affinityGroupDao.findById(2L)).thenReturn(null);
kubernetesClusterManager.setAffinityGroupResponseForNodeType(response, clusterId, CONTROL.name());
Mockito.verify(affinityGroupDao).findById(1L);
Mockito.verify(affinityGroupDao).findById(2L);
}
@Test
public void testSetNodeTypeAffinityGroupResponse() {
KubernetesClusterResponse response = new KubernetesClusterResponse();
long clusterId = 1L;
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(Mockito.eq(clusterId), Mockito.anyString()))
.thenReturn(Collections.emptyList());
kubernetesClusterManager.setNodeTypeAffinityGroupResponse(response, clusterId);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, CONTROL.name());
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, WORKER.name());
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name());
}
@Test
public void testValidateNodeAffinityGroupsNoAffinityGroups() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
List<Long> nodeIds = Arrays.asList(100L, 101L);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Collections.emptyList());
kubernetesClusterManager.validateNodeAffinityGroups(nodeIds, cluster);
Mockito.verify(kubernetesClusterVmMapDao, Mockito.never()).listByClusterIdAndVmType(Mockito.anyLong(), Mockito.any());
}
@Test
public void testValidateNodeAffinityGroupsNullAffinityGroups() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
List<Long> nodeIds = Arrays.asList(100L, 101L);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(null);
kubernetesClusterManager.validateNodeAffinityGroups(nodeIds, cluster);
Mockito.verify(kubernetesClusterVmMapDao, Mockito.never()).listByClusterIdAndVmType(Mockito.anyLong(), Mockito.any());
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateNodeAffinityGroupsAntiAffinityNewNodeOnExistingHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long sharedHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(sharedHostId);
Mockito.when(newNode.getInstanceName()).thenReturn("new-node-vm");
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(sharedHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getName()).thenReturn("host-1");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
Mockito.when(hostDao.findById(sharedHostId)).thenReturn(host);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
}
@Test
public void testValidateNodeAffinityGroupsAntiAffinityNewNodeOnDifferentHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.lenient().when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long existingHostId = 1000L;
Long newNodeHostId = 1001L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.lenient().when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(newNodeHostId);
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(existingHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
@Test
public void testValidateNodeAffinityGroupsAffinityNewNodeOnSameHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.lenient().when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long sharedHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host affinity");
Mockito.lenient().when(affinityGroup.getName()).thenReturn("affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(sharedHostId);
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(sharedHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateNodeAffinityGroupsAffinityNewNodeOnDifferentHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long existingHostId = 1000L;
Long newNodeHostId = 1001L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host affinity");
Mockito.when(affinityGroup.getName()).thenReturn("affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(newNodeHostId);
Mockito.when(newNode.getInstanceName()).thenReturn("new-node-vm");
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(existingHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
HostVO newHost = Mockito.mock(HostVO.class);
Mockito.when(newHost.getName()).thenReturn("host-2");
HostVO existingHost = Mockito.mock(HostVO.class);
Mockito.when(existingHost.getName()).thenReturn("host-1");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
Mockito.when(hostDao.findById(newNodeHostId)).thenReturn(newHost);
Mockito.when(hostDao.findById(existingHostId)).thenReturn(existingHost);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateNodeAffinityGroupsAntiAffinityMultipleNewNodesOnSameHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId1 = 100L;
Long newNodeId2 = 101L;
Long sharedHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.lenient().when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode1 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode1.getHostId()).thenReturn(sharedHostId);
Mockito.lenient().when(newNode1.getInstanceName()).thenReturn("new-node-vm-1");
VMInstanceVO newNode2 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode2.getHostId()).thenReturn(sharedHostId);
Mockito.when(newNode2.getInstanceName()).thenReturn("new-node-vm-2");
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getName()).thenReturn("host-1");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId1)).thenReturn(newNode1);
Mockito.when(vmInstanceDao.findById(newNodeId2)).thenReturn(newNode2);
Mockito.when(hostDao.findById(sharedHostId)).thenReturn(host);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId1, newNodeId2), cluster);
}
@Test
public void testValidateNodeAffinityGroupsAntiAffinityMultipleNewNodesOnDifferentHosts() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.lenient().when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId1 = 100L;
Long newNodeId2 = 101L;
Long hostId1 = 1000L;
Long hostId2 = 1001L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.lenient().when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.lenient().when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode1 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode1.getHostId()).thenReturn(hostId1);
VMInstanceVO newNode2 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode2.getHostId()).thenReturn(hostId2);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId1)).thenReturn(newNode1);
Mockito.when(vmInstanceDao.findById(newNodeId2)).thenReturn(newNode2);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId1, newNodeId2), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
@Test
public void testValidateNodeAffinityGroupsNodeWithNullHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.lenient().when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.lenient().when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.lenient().when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(null);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(vmInstanceDao, Mockito.atLeastOnce()).findById(newNodeId);
}
@Test
public void testValidateNodeAffinityGroupsNullNode() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.lenient().when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.lenient().when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.lenient().when(affinityGroup.getName()).thenReturn("anti-affinity-group");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(null);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(vmInstanceDao, Mockito.atLeastOnce()).findById(newNodeId);
}
@Test
public void testValidateNodeAffinityGroupsAffinityNoExistingWorkers() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.lenient().when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long newNodeHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.lenient().when(affinityGroup.getType()).thenReturn("host affinity");
Mockito.lenient().when(affinityGroup.getName()).thenReturn("affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(newNodeHostId);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
}

View File

@ -17,6 +17,15 @@
package com.cloud.kubernetes.cluster;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.cloudstack.affinity.AffinityGroupVO;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
@ -24,11 +33,16 @@ import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.UserVmManager;
import com.cloud.vm.VmDetailConstants;
@RunWith(MockitoJUnitRunner.class)
public class KubernetesServiceHelperImplTest {
@ -36,6 +50,10 @@ public class KubernetesServiceHelperImplTest {
KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@Mock
KubernetesClusterDao kubernetesClusterDao;
@Mock
AffinityGroupDao affinityGroupDao;
@Mock
ServiceOfferingDao serviceOfferingDao;
@InjectMocks
KubernetesServiceHelperImpl kubernetesServiceHelper = new KubernetesServiceHelperImpl();
@ -84,4 +102,302 @@ public class KubernetesServiceHelperImplTest {
Mockito.when(kubernetesCluster.getClusterType()).thenReturn(KubernetesCluster.ClusterType.ExternalManaged);
kubernetesServiceHelper.checkVmCanBeDestroyed(vm);
}
@Test
public void testIsValidNodeTypeEmptyNodeType() {
Assert.assertFalse(kubernetesServiceHelper.isValidNodeType(null));
}
@Test
public void testIsValidNodeTypeInvalidNodeType() {
Assert.assertFalse(kubernetesServiceHelper.isValidNodeType("invalidNodeType"));
}
@Test
public void testIsValidNodeTypeValidNodeTypeLowercase() {
String nodeType = KubernetesClusterNodeType.WORKER.name().toLowerCase();
Assert.assertTrue(kubernetesServiceHelper.isValidNodeType(nodeType));
}
private Map<String, String> createServiceOfferingMapEntry(KubernetesClusterNodeType nodeType, String offeringUuid) {
Map<String, String> map = new HashMap<>();
map.put(VmDetailConstants.CKS_NODE_TYPE, nodeType.name().toLowerCase());
map.put(VmDetailConstants.OFFERING, offeringUuid);
return map;
}
@Test
public void testGetServiceOfferingNodeTypeMap() {
String workerOfferingUuid = UUID.randomUUID().toString();
String controlOfferingUuid = UUID.randomUUID().toString();
ServiceOfferingVO workerOffering = Mockito.mock(ServiceOfferingVO.class);
Mockito.when(workerOffering.getId()).thenReturn(1L);
Mockito.when(serviceOfferingDao.findByUuid(workerOfferingUuid)).thenReturn(workerOffering);
ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class);
Mockito.when(controlOffering.getId()).thenReturn(2L);
Mockito.when(serviceOfferingDao.findByUuid(controlOfferingUuid)).thenReturn(controlOffering);
Map<String, Map<String, String>> serviceOfferingNodeTypeMap = new HashMap<>();
serviceOfferingNodeTypeMap.put("map1", createServiceOfferingMapEntry(KubernetesClusterNodeType.WORKER, workerOfferingUuid));
serviceOfferingNodeTypeMap.put("map2", createServiceOfferingMapEntry(KubernetesClusterNodeType.CONTROL, controlOfferingUuid));
Map<String, Long> result = kubernetesServiceHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap);
Assert.assertNotNull(result);
Assert.assertEquals(2, result.size());
Assert.assertTrue(result.containsKey(KubernetesClusterNodeType.WORKER.name()));
Assert.assertTrue(result.containsKey(KubernetesClusterNodeType.CONTROL.name()));
Assert.assertEquals(Long.valueOf(1L), result.get(KubernetesClusterNodeType.WORKER.name()));
Assert.assertEquals(Long.valueOf(2L), result.get(KubernetesClusterNodeType.CONTROL.name()));
}
@Test
public void testGetServiceOfferingNodeTypeMapNullMap() {
Map<String, Long> result = kubernetesServiceHelper.getServiceOfferingNodeTypeMap(null);
Assert.assertTrue(result.isEmpty());
}
@Test
public void testGetServiceOfferingNodeTypeMapEtcdNodes() {
String etcdOfferingUuid = UUID.randomUUID().toString();
ServiceOfferingVO etcdOffering = Mockito.mock(ServiceOfferingVO.class);
Mockito.when(etcdOffering.getId()).thenReturn(3L);
Mockito.when(serviceOfferingDao.findByUuid(etcdOfferingUuid)).thenReturn(etcdOffering);
Map<String, Map<String, String>> serviceOfferingNodeTypeMap = new HashMap<>();
serviceOfferingNodeTypeMap.put("map1", createServiceOfferingMapEntry(KubernetesClusterNodeType.ETCD, etcdOfferingUuid));
Map<String, Long> result = kubernetesServiceHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap);
Assert.assertNotNull(result);
Assert.assertEquals(1, result.size());
Assert.assertTrue(result.containsKey(KubernetesClusterNodeType.ETCD.name()));
Assert.assertEquals(Long.valueOf(3L), result.get(KubernetesClusterNodeType.ETCD.name()));
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeOfferingEntryCompletenessInvalidParameters() {
kubernetesServiceHelper.checkNodeTypeOfferingEntryCompleteness(KubernetesClusterNodeType.WORKER.name(), null);
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeOfferingEntryValuesInvalidNodeType() {
ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class);
kubernetesServiceHelper.checkNodeTypeOfferingEntryValues("invalidNodeTypeName", offering, "some-uuid");
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeOfferingEntryValuesEmptyOffering() {
kubernetesServiceHelper.checkNodeTypeOfferingEntryValues(KubernetesClusterNodeType.WORKER.name(), null, "some-uuid");
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeAffinityGroupEntryCompletenessBlankNodeType() {
kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryCompleteness("", "affinity-group-uuid");
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeAffinityGroupEntryCompletenessBlankAffinityGroupUuid() {
kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryCompleteness("control", "");
}
@Test
public void testCheckNodeTypeAffinityGroupEntryCompletenessValid() {
kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryCompleteness("control", "affinity-group-uuid");
}
@Test(expected = InvalidParameterValueException.class)
public void testCheckNodeTypeAffinityGroupEntryNodeTypeInvalid() {
kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryNodeType("invalid-node-type");
}
@Test
public void testCheckNodeTypeAffinityGroupEntryNodeTypeValid() {
kubernetesServiceHelper.checkNodeTypeAffinityGroupEntryNodeType("control");
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateAffinityGroupUuidAndGetIdBlank() {
kubernetesServiceHelper.validateAffinityGroupUuidAndGetId("");
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateAffinityGroupUuidAndGetIdNotFound() {
Mockito.when(affinityGroupDao.findByUuid("non-existent-uuid")).thenReturn(null);
kubernetesServiceHelper.validateAffinityGroupUuidAndGetId("non-existent-uuid");
}
@Test
public void testValidateAffinityGroupUuidAndGetIdValid() {
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getId()).thenReturn(100L);
Mockito.when(affinityGroupDao.findByUuid("valid-uuid")).thenReturn(affinityGroup);
Long result = kubernetesServiceHelper.validateAffinityGroupUuidAndGetId("valid-uuid");
Assert.assertEquals(Long.valueOf(100L), result);
}
@Test
public void testValidateAndGetAffinityGroupIdsSingleUuid() {
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getId()).thenReturn(1L);
Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup);
List<Long> result = kubernetesServiceHelper.validateAndGetAffinityGroupIds("uuid1");
Assert.assertEquals(1, result.size());
Assert.assertEquals(Long.valueOf(1L), result.get(0));
}
@Test
public void testValidateAndGetAffinityGroupIdsMultipleUuids() {
AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class);
AffinityGroupVO affinityGroup2 = Mockito.mock(AffinityGroupVO.class);
AffinityGroupVO affinityGroup3 = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup1.getId()).thenReturn(1L);
Mockito.when(affinityGroup2.getId()).thenReturn(2L);
Mockito.when(affinityGroup3.getId()).thenReturn(3L);
Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1);
Mockito.when(affinityGroupDao.findByUuid("uuid2")).thenReturn(affinityGroup2);
Mockito.when(affinityGroupDao.findByUuid("uuid3")).thenReturn(affinityGroup3);
List<Long> result = kubernetesServiceHelper.validateAndGetAffinityGroupIds("uuid1,uuid2,uuid3");
Assert.assertEquals(3, result.size());
Assert.assertEquals(Arrays.asList(1L, 2L, 3L), result);
}
@Test
public void testValidateAndGetAffinityGroupIdsWithSpaces() {
AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class);
AffinityGroupVO affinityGroup2 = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup1.getId()).thenReturn(1L);
Mockito.when(affinityGroup2.getId()).thenReturn(2L);
Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1);
Mockito.when(affinityGroupDao.findByUuid("uuid2")).thenReturn(affinityGroup2);
List<Long> result = kubernetesServiceHelper.validateAndGetAffinityGroupIds(" uuid1 , uuid2 ");
Assert.assertEquals(2, result.size());
Assert.assertEquals(Arrays.asList(1L, 2L), result);
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateAndGetAffinityGroupIdsOneInvalid() {
AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1);
Mockito.when(affinityGroupDao.findByUuid("invalid-uuid")).thenReturn(null);
kubernetesServiceHelper.validateAndGetAffinityGroupIds("uuid1,invalid-uuid");
}
@Test
public void testAddNodeTypeAffinityGroupEntry() {
Map<String, List<Long>> mapping = new HashMap<>();
kubernetesServiceHelper.addNodeTypeAffinityGroupEntry("control", Arrays.asList(1L, 2L), mapping);
Assert.assertEquals(1, mapping.size());
Assert.assertEquals(Arrays.asList(1L, 2L), mapping.get("CONTROL"));
}
@Test
public void testProcessNodeTypeAffinityGroupEntryAndAddToMappingIfValidEmptyEntry() {
Map<String, List<Long>> mapping = new HashMap<>();
kubernetesServiceHelper.processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(new HashMap<>(), mapping);
Assert.assertTrue(mapping.isEmpty());
}
@Test
public void testProcessNodeTypeAffinityGroupEntryAndAddToMappingIfValidValidEntry() {
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getId()).thenReturn(100L);
Mockito.when(affinityGroupDao.findByUuid("affinity-group-uuid")).thenReturn(affinityGroup);
Map<String, String> entry = new HashMap<>();
entry.put(VmDetailConstants.CKS_NODE_TYPE, "control");
entry.put(VmDetailConstants.AFFINITY_GROUP, "affinity-group-uuid");
Map<String, List<Long>> mapping = new HashMap<>();
kubernetesServiceHelper.processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(entry, mapping);
Assert.assertEquals(1, mapping.size());
Assert.assertEquals(Arrays.asList(100L), mapping.get("CONTROL"));
}
@Test
public void testProcessNodeTypeAffinityGroupEntryAndAddToMappingIfValidMultipleUuids() {
AffinityGroupVO affinityGroup1 = Mockito.mock(AffinityGroupVO.class);
AffinityGroupVO affinityGroup2 = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup1.getId()).thenReturn(1L);
Mockito.when(affinityGroup2.getId()).thenReturn(2L);
Mockito.when(affinityGroupDao.findByUuid("uuid1")).thenReturn(affinityGroup1);
Mockito.when(affinityGroupDao.findByUuid("uuid2")).thenReturn(affinityGroup2);
Map<String, String> entry = new HashMap<>();
entry.put(VmDetailConstants.CKS_NODE_TYPE, "worker");
entry.put(VmDetailConstants.AFFINITY_GROUP, "uuid1,uuid2");
Map<String, List<Long>> mapping = new HashMap<>();
kubernetesServiceHelper.processNodeTypeAffinityGroupEntryAndAddToMappingIfValid(entry, mapping);
Assert.assertEquals(1, mapping.size());
Assert.assertEquals(Arrays.asList(1L, 2L), mapping.get("WORKER"));
}
@Test
public void testGetAffinityGroupNodeTypeMapEmptyMap() {
Map<String, List<Long>> result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(null);
Assert.assertTrue(result.isEmpty());
result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(new HashMap<>());
Assert.assertTrue(result.isEmpty());
}
@Test
public void testGetAffinityGroupNodeTypeMapValidEntries() {
AffinityGroupVO controlAffinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(controlAffinityGroup.getId()).thenReturn(100L);
Mockito.when(affinityGroupDao.findByUuid("control-affinity-uuid")).thenReturn(controlAffinityGroup);
AffinityGroupVO workerAffinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(workerAffinityGroup.getId()).thenReturn(200L);
Mockito.when(affinityGroupDao.findByUuid("worker-affinity-uuid")).thenReturn(workerAffinityGroup);
Map<String, Map<String, String>> affinityGroupNodeTypeMap = new HashMap<>();
Map<String, String> controlEntry = new HashMap<>();
controlEntry.put(VmDetailConstants.CKS_NODE_TYPE, "control");
controlEntry.put(VmDetailConstants.AFFINITY_GROUP, "control-affinity-uuid");
affinityGroupNodeTypeMap.put("0", controlEntry);
Map<String, String> workerEntry = new HashMap<>();
workerEntry.put(VmDetailConstants.CKS_NODE_TYPE, "worker");
workerEntry.put(VmDetailConstants.AFFINITY_GROUP, "worker-affinity-uuid");
affinityGroupNodeTypeMap.put("1", workerEntry);
Map<String, List<Long>> result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(affinityGroupNodeTypeMap);
Assert.assertEquals(2, result.size());
Assert.assertEquals(Arrays.asList(100L), result.get("CONTROL"));
Assert.assertEquals(Arrays.asList(200L), result.get("WORKER"));
}
@Test
public void testGetAffinityGroupNodeTypeMapMultipleIdsPerNodeType() {
AffinityGroupVO ag1 = Mockito.mock(AffinityGroupVO.class);
AffinityGroupVO ag2 = Mockito.mock(AffinityGroupVO.class);
AffinityGroupVO ag3 = Mockito.mock(AffinityGroupVO.class);
Mockito.when(ag1.getId()).thenReturn(1L);
Mockito.when(ag2.getId()).thenReturn(2L);
Mockito.when(ag3.getId()).thenReturn(3L);
Mockito.when(affinityGroupDao.findByUuid("ag1")).thenReturn(ag1);
Mockito.when(affinityGroupDao.findByUuid("ag2")).thenReturn(ag2);
Mockito.when(affinityGroupDao.findByUuid("ag3")).thenReturn(ag3);
Map<String, Map<String, String>> affinityGroupNodeTypeMap = new HashMap<>();
Map<String, String> controlEntry = new HashMap<>();
controlEntry.put(VmDetailConstants.CKS_NODE_TYPE, "control");
controlEntry.put(VmDetailConstants.AFFINITY_GROUP, "ag1,ag2,ag3");
affinityGroupNodeTypeMap.put("0", controlEntry);
Map<String, List<Long>> result = kubernetesServiceHelper.getAffinityGroupNodeTypeMap(affinityGroupNodeTypeMap);
Assert.assertEquals(1, result.size());
Assert.assertEquals(Arrays.asList(1L, 2L, 3L), result.get("CONTROL"));
}
}

View File

@ -16,8 +16,14 @@
// under the License.
package com.cloud.kubernetes.cluster.actionworkers;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.apache.cloudstack.affinity.AffinityGroupVO;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.api.ApiConstants;
import org.junit.Assert;
import org.junit.Before;
@ -30,6 +36,8 @@ import org.mockito.junit.MockitoJUnitRunner;
import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterAffinityGroupMapDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
@ -60,6 +68,12 @@ public class KubernetesClusterActionWorkerTest {
@Mock
IPAddressDao ipAddressDao;
@Mock
KubernetesClusterAffinityGroupMapDao kubernetesClusterAffinityGroupMapDao;
@Mock
AffinityGroupDao affinityGroupDao;
KubernetesClusterActionWorker actionWorker = null;
final static Long DEFAULT_ID = 1L;
@ -70,10 +84,12 @@ public class KubernetesClusterActionWorkerTest {
kubernetesClusterManager.kubernetesSupportedVersionDao = kubernetesSupportedVersionDao;
kubernetesClusterManager.kubernetesClusterDetailsDao = kubernetesClusterDetailsDao;
kubernetesClusterManager.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao;
kubernetesClusterManager.kubernetesClusterAffinityGroupMapDao = kubernetesClusterAffinityGroupMapDao;
KubernetesCluster kubernetesCluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(kubernetesCluster.getId()).thenReturn(DEFAULT_ID);
actionWorker = new KubernetesClusterActionWorker(kubernetesCluster, kubernetesClusterManager);
actionWorker.ipAddressDao = ipAddressDao;
actionWorker.affinityGroupDao = affinityGroupDao;
}
@Test
@ -130,4 +146,87 @@ public class KubernetesClusterActionWorkerTest {
IpAddress result = actionWorker.getVpcTierKubernetesPublicIp(mockNetworkForGetVpcTierKubernetesPublicIpTest());
Assert.assertNotNull(result);
}
@Test
public void testGetAffinityGroupIdsForNodeTypeReturnsIds() {
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL"))
.thenReturn(Arrays.asList(1L, 2L));
List<Long> result = actionWorker.getAffinityGroupIdsForNodeType(KubernetesClusterNodeType.CONTROL);
Assert.assertEquals(2, result.size());
Assert.assertTrue(result.containsAll(Arrays.asList(1L, 2L)));
}
@Test
public void testGetAffinityGroupIdsForNodeTypeReturnsEmptyList() {
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "WORKER"))
.thenReturn(Collections.emptyList());
List<Long> result = actionWorker.getAffinityGroupIdsForNodeType(KubernetesClusterNodeType.WORKER);
Assert.assertTrue(result.isEmpty());
}
@Test
public void testGetMergedAffinityGroupIdsWithExplicitDedication() {
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL"))
.thenReturn(new ArrayList<>(Arrays.asList(1L)));
AffinityGroupVO explicitGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(explicitGroup.getId()).thenReturn(99L);
Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.eq("ExplicitDedication")))
.thenReturn(explicitGroup);
List<Long> result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.CONTROL, 1L, 1L);
Assert.assertEquals(2, result.size());
Assert.assertTrue(result.contains(1L));
Assert.assertTrue(result.contains(99L));
}
@Test
public void testGetMergedAffinityGroupIdsNoExplicitDedication() {
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "WORKER"))
.thenReturn(new ArrayList<>(Arrays.asList(1L, 2L)));
Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.eq("ExplicitDedication")))
.thenReturn(null);
Mockito.when(affinityGroupDao.findDomainLevelGroupByType(Mockito.anyLong(), Mockito.eq("ExplicitDedication")))
.thenReturn(null);
List<Long> result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.WORKER, 1L, 1L);
Assert.assertEquals(2, result.size());
}
@Test
public void testGetMergedAffinityGroupIdsReturnsNullWhenEmpty() {
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "ETCD"))
.thenReturn(new ArrayList<>());
Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.anyString()))
.thenReturn(null);
Mockito.when(affinityGroupDao.findDomainLevelGroupByType(Mockito.anyLong(), Mockito.anyString()))
.thenReturn(null);
List<Long> result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.ETCD, 1L, 1L);
Assert.assertNull(result);
}
@Test
public void testGetMergedAffinityGroupIdsExplicitDedicationAlreadyInList() {
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(DEFAULT_ID, "CONTROL"))
.thenReturn(new ArrayList<>(Arrays.asList(99L, 2L)));
AffinityGroupVO explicitGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(explicitGroup.getId()).thenReturn(99L);
Mockito.when(affinityGroupDao.findByAccountAndType(Mockito.anyLong(), Mockito.eq("ExplicitDedication")))
.thenReturn(explicitGroup);
List<Long> result = actionWorker.getMergedAffinityGroupIds(KubernetesClusterNodeType.CONTROL, 1L, 1L);
Assert.assertEquals(2, result.size());
Assert.assertTrue(result.contains(99L));
Assert.assertTrue(result.contains(2L));
}
}

View File

@ -16,10 +16,15 @@
// under the License.
package com.cloud.kubernetes.version;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -32,6 +37,9 @@ import org.springframework.test.util.ReflectionTestUtils;
import com.cloud.api.query.dao.TemplateJoinDao;
import com.cloud.api.query.vo.TemplateJoinVO;
import com.cloud.cpu.CPU;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.exception.InvalidParameterValueException;
@RunWith(MockitoJUnitRunner.class)
public class KubernetesVersionManagerImplTest {
@ -39,6 +47,12 @@ public class KubernetesVersionManagerImplTest {
@Mock
TemplateJoinDao templateJoinDao;
@Mock
ImageStoreDao imageStoreDao;
@Mock
DataCenterDao dataCenterDao;
@InjectMocks
KubernetesVersionManagerImpl kubernetesVersionManager = new KubernetesVersionManagerImpl();
@ -48,7 +62,7 @@ public class KubernetesVersionManagerImplTest {
Mockito.when(kubernetesSupportedVersion.getIsoId()).thenReturn(1L);
KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse();
kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion,
response);
response, true);
Assert.assertNull(ReflectionTestUtils.getField(response, "isoId"));
}
@ -63,13 +77,71 @@ public class KubernetesVersionManagerImplTest {
Mockito.when(templateJoinVO.getUuid()).thenReturn(uuid);
Mockito.when(templateJoinDao.findById(1L)).thenReturn(templateJoinVO);
kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion,
response);
response, true);
Assert.assertEquals(uuid, ReflectionTestUtils.getField(response, "isoId"));
Assert.assertNull(ReflectionTestUtils.getField(response, "isoState"));
ObjectInDataStoreStateMachine.State state = ObjectInDataStoreStateMachine.State.Ready;
Mockito.when(templateJoinVO.getState()).thenReturn(state);
kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion,
response);
response, true);
Assert.assertEquals(state.toString(), ReflectionTestUtils.getField(response, "isoState"));
}
@Test
public void testValidateImageStoreForZoneWithDirectDownload() {
ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", 1L, true);
}
@Test
public void testValidateImageStoreForZoneWithValidZone() {
Long zoneId = 1L;
List<ImageStoreVO> imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class));
Mockito.when(imageStoreDao.listStoresByZoneId(zoneId)).thenReturn(imageStores);
ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", zoneId, false);
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateImageStoreForZoneWithNoImageStore() {
Long zoneId = 1L;
DataCenterVO zone = Mockito.mock(DataCenterVO.class);
Mockito.when(zone.getName()).thenReturn("test-zone");
Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(zone);
Mockito.when(imageStoreDao.listStoresByZoneId(zoneId)).thenReturn(Collections.emptyList());
ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", zoneId, false);
}
@Test
public void testValidateImageStoreForAllZonesWithAllValid() {
DataCenterVO zone1 = Mockito.mock(DataCenterVO.class);
Mockito.when(zone1.getId()).thenReturn(1L);
DataCenterVO zone2 = Mockito.mock(DataCenterVO.class);
Mockito.when(zone2.getId()).thenReturn(2L);
List<DataCenterVO> zones = Arrays.asList(zone1, zone2);
Mockito.when(dataCenterDao.listAllZones()).thenReturn(zones);
List<ImageStoreVO> imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class));
Mockito.when(imageStoreDao.listStoresByZoneId(1L)).thenReturn(imageStores);
Mockito.when(imageStoreDao.listStoresByZoneId(2L)).thenReturn(imageStores);
ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", (Long) null, false);
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateImageStoreForAllZonesWithSomeMissingStorage() {
DataCenterVO zone1 = Mockito.mock(DataCenterVO.class);
Mockito.when(zone1.getId()).thenReturn(1L);
DataCenterVO zone2 = Mockito.mock(DataCenterVO.class);
Mockito.when(zone2.getId()).thenReturn(2L);
Mockito.when(zone2.getName()).thenReturn("zone-without-storage");
List<DataCenterVO> zones = Arrays.asList(zone1, zone2);
Mockito.when(dataCenterDao.listAllZones()).thenReturn(zones);
List<ImageStoreVO> imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class));
Mockito.when(imageStoreDao.listStoresByZoneId(1L)).thenReturn(imageStores);
Mockito.when(imageStoreDao.listStoresByZoneId(2L)).thenReturn(Collections.emptyList());
ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", (Long) null, false);
}
}

View File

@ -17,6 +17,9 @@
package com.cloud.kubernetes.version;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.when;
import java.lang.reflect.Field;
@ -25,6 +28,11 @@ import java.util.List;
import java.util.UUID;
import com.cloud.cpu.CPU;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountVO;
import com.cloud.user.User;
import com.cloud.user.UserVO;
import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd;
import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd;
import org.apache.cloudstack.api.command.admin.kubernetes.version.UpdateKubernetesSupportedVersionCmd;
@ -63,11 +71,6 @@ import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.template.TemplateApiService;
import com.cloud.template.VirtualMachineTemplate;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountVO;
import com.cloud.user.User;
import com.cloud.user.UserVO;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.db.Filter;
@ -75,6 +78,9 @@ import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
@RunWith(MockitoJUnitRunner.class)
public class KubernetesVersionServiceTest {
@ -94,7 +100,11 @@ public class KubernetesVersionServiceTest {
@Mock
private DataCenterDao dataCenterDao;
@Mock
private ImageStoreDao imageStoreDao;
@Mock
private TemplateApiService templateService;
@Mock
private Account accountMock;
AutoCloseable closeable;
@ -123,7 +133,12 @@ public class KubernetesVersionServiceTest {
DataCenterVO zone = Mockito.mock(DataCenterVO.class);
when(dataCenterDao.findById(Mockito.anyLong())).thenReturn(zone);
List<ImageStoreVO> imageStores = new ArrayList<>();
imageStores.add(Mockito.mock(ImageStoreVO.class));
when(imageStoreDao.listStoresByZoneId(Mockito.anyLong())).thenReturn(imageStores);
TemplateJoinVO templateJoinVO = Mockito.mock(TemplateJoinVO.class);
when(templateJoinVO.getUrl()).thenReturn("https://download.cloudstack.com");
when(templateJoinVO.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready);
when(templateJoinVO.getArch()).thenReturn(CPU.CPUArch.getDefault());
when(templateJoinDao.findById(Mockito.anyLong())).thenReturn(templateJoinVO);
@ -140,20 +155,67 @@ public class KubernetesVersionServiceTest {
@Test
public void listKubernetesSupportedVersionsTest() {
CallContext callContextMock = Mockito.mock(CallContext.class);
try (MockedStatic<CallContext> callContextMockedStatic = Mockito.mockStatic(CallContext.class)) {
callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock);
final SearchCriteria<KubernetesSupportedVersionVO> versionSearchCriteria = Mockito.mock(SearchCriteria.class);
when(callContextMock.getCallingAccount()).thenReturn(accountMock);
ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class);
List<KubernetesSupportedVersionVO> versionVOs = new ArrayList<>();
KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class);
when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION);
versionVOs.add(versionVO);
when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO);
when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(SearchCriteria.class),
Mockito.any(Filter.class))).thenReturn(new Pair<>(versionVOs, versionVOs.size()));
when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class)))
.thenReturn(new Pair<>(versionVOs, versionVOs.size()));
ListResponse<KubernetesSupportedVersionResponse> versionsResponse =
kubernetesVersionService.listKubernetesSupportedVersions(cmd);
Assert.assertEquals(versionVOs.size(), versionsResponse.getCount().intValue());
Assert.assertTrue(CollectionUtils.isNotEmpty(versionsResponse.getResponses()));
Assert.assertEquals(versionVOs.size(), versionsResponse.getResponses().size());
}
}
@Test
public void listKubernetesSupportedVersionsTestWhenAdmin() {
CallContext callContextMock = Mockito.mock(CallContext.class);
try (MockedStatic<CallContext> callContextMockedStatic = Mockito.mockStatic(CallContext.class)) {
callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock);
ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class);
List<KubernetesSupportedVersionVO> versionVOs = new ArrayList<>();
KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class);
when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION);
versionVOs.add(versionVO);
when(callContextMock.getCallingAccount()).thenReturn(accountMock);
when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO);
when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class)))
.thenReturn(new Pair<>(versionVOs, versionVOs.size()));
when(accountManager.isRootAdmin(anyLong())).thenReturn(true);
ListResponse<KubernetesSupportedVersionResponse> response = kubernetesVersionService.listKubernetesSupportedVersions(cmd);
assertNotNull(response.getResponses().get(0).getIsoUrl());
}
}
@Test
public void listKubernetesSupportedVersionsTestWhenOtherUser() {
CallContext callContextMock = Mockito.mock(CallContext.class);
try (MockedStatic<CallContext> callContextMockedStatic = Mockito.mockStatic(CallContext.class)) {
callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock);
ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class);
List<KubernetesSupportedVersionVO> versionVOs = new ArrayList<>();
KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class);
when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION);
versionVOs.add(versionVO);
when(callContextMock.getCallingAccount()).thenReturn(accountMock);
when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO);
when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class)))
.thenReturn(new Pair<>(versionVOs, versionVOs.size()));
when(accountManager.isRootAdmin(anyLong())).thenReturn(false);
when(accountMock.getId()).thenReturn(2L);
ListResponse<KubernetesSupportedVersionResponse> response = kubernetesVersionService.listKubernetesSupportedVersions(cmd);
assertNull(response.getResponses().get(0).getIsoUrl());
}
}
@Test(expected = InvalidParameterValueException.class)
public void addKubernetesSupportedVersionLowerUnsupportedTest() {
@ -224,7 +286,6 @@ public class KubernetesVersionServiceTest {
mockedComponentContext.when(() -> ComponentContext.inject(Mockito.any(RegisterIsoCmd.class))).thenReturn(
new RegisterIsoCmd());
mockedCallContext.when(CallContext::current).thenReturn(callContext);
when(templateService.registerIso(Mockito.any(RegisterIsoCmd.class))).thenReturn(
Mockito.mock(VirtualMachineTemplate.class));
VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class);

View File

@ -1801,14 +1801,18 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
protected String getStoragePoolNonDestroyedVolumesLog(long storagePoolId) {
StringBuilder sb = new StringBuilder();
List<VolumeVO> nonDestroyedVols = volumeDao.findByPoolId(storagePoolId, null).stream().filter(vol -> vol.getState() != Volume.State.Destroy).collect(Collectors.toList());
List<VolumeVO> nonDestroyedVols = volumeDao.findByPoolId(storagePoolId, null);
VMInstanceVO volInstance;
List<String> logMessageInfo = new ArrayList<>();
sb.append("[");
for (VolumeVO vol : nonDestroyedVols) {
volInstance = _vmInstanceDao.findById(vol.getInstanceId());
if (volInstance != null) {
logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid()));
} else {
logMessageInfo.add(String.format("Volume [%s]", vol.getUuid()));
}
}
sb.append(String.join(", ", logMessageInfo));
sb.append("]");

View File

@ -27,6 +27,7 @@ import com.cloud.exception.StorageConflictException;
import com.cloud.storage.StorageManager;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.Profiler;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
@ -200,12 +201,13 @@ public class StoragePoolMonitor implements Listener {
}
@Override
public synchronized boolean processDisconnect(long agentId, Status state) {
public boolean processDisconnect(long agentId, Status state) {
return processDisconnect(agentId, null, null, state);
}
@Override
public synchronized boolean processDisconnect(long agentId, String uuid, String name, Status state) {
public boolean processDisconnect(long agentId, String uuid, String name, Status state) {
logger.debug("Starting disconnect for Agent [id: {}, uuid: {}, name: {}]", agentId, uuid, name);
Host host = _storageManager.getHost(agentId);
if (host == null) {
logger.warn("Agent [id: {}, uuid: {}, name: {}] not found, not disconnecting pools", agentId, uuid, name);
@ -213,38 +215,52 @@ public class StoragePoolMonitor implements Listener {
}
if (host.getType() != Host.Type.Routing) {
logger.debug("Host [id: {}, uuid: {}, name: {}] is not of type {}, skip", agentId, uuid, name, Host.Type.Routing);
return false;
}
logger.debug("Looking for connected Storage Pools for Host [id: {}, uuid: {}, name: {}]", agentId, uuid, name);
List<StoragePoolHostVO> storagePoolHosts = _storageManager.findStoragePoolsConnectedToHost(host.getId());
if (storagePoolHosts == null) {
if (logger.isTraceEnabled()) {
logger.trace("No pools to disconnect for host: {}", host);
}
logger.debug("No pools to disconnect for host: {}", host);
return true;
}
logger.debug("Found {} pools to disconnect for host: {}", storagePoolHosts.size(), host);
boolean disconnectResult = true;
for (StoragePoolHostVO storagePoolHost : storagePoolHosts) {
int storagePoolHostsSize = storagePoolHosts.size();
for (int i = 0; i < storagePoolHostsSize; i++) {
StoragePoolHostVO storagePoolHost = storagePoolHosts.get(i);
logger.debug("Processing disconnect from Storage Pool {} ({} of {}) for host: {}", storagePoolHost.getPoolId(), i, storagePoolHostsSize, host);
StoragePoolVO pool = _poolDao.findById(storagePoolHost.getPoolId());
if (pool == null) {
logger.debug("No Storage Pool found with id {} ({} of {}) for host: {}", storagePoolHost.getPoolId(), i, storagePoolHostsSize, host);
continue;
}
if (!pool.isShared()) {
logger.debug("Storage Pool {} ({}) ({} of {}) is not shared for host: {}, ignore disconnect", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host);
continue;
}
// Handle only PowerFlex pool for now, not to impact other pools behavior
if (pool.getPoolType() != StoragePoolType.PowerFlex) {
logger.debug("Storage Pool {} ({}) ({} of {}) is not of type {} for host: {}, ignore disconnect", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, pool.getPoolType(), host);
continue;
}
logger.debug("Sending disconnect to Storage Pool {} ({}) ({} of {}) for host: {}", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host);
Profiler disconnectProfiler = new Profiler();
try {
disconnectProfiler.start();
_storageManager.disconnectHostFromSharedPool(host, pool);
} catch (Exception e) {
logger.error("Unable to disconnect host {} from storage pool {} due to {}", host, pool, e.toString());
disconnectResult = false;
} finally {
disconnectProfiler.stop();
long disconnectDuration = disconnectProfiler.getDurationInMillis() / 1000;
logger.debug("Finished disconnect with result {} from Storage Pool {} ({}) ({} of {}) for host: {}, duration: {} secs", disconnectResult, pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host, disconnectDuration);
}
}

View File

@ -1229,8 +1229,10 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar
if (dc.getDns2() != null) {
buf.append(" dns2=").append(dc.getDns2());
}
String nfsVersion = imageStoreDetailsUtil != null ? imageStoreDetailsUtil.getNfsVersion(secStores.get(0).getId()) : null;
String nfsVersion = imageStoreDetailsUtil.getNfsVersion(secStores.get(0).getId());
if (StringUtils.isNotBlank(nfsVersion)) {
buf.append(" nfsVersion=").append(nfsVersion);
}
buf.append(" keystore_password=").append(VirtualMachineGuru.getEncodedString(PasswordGenerator.generateRandomPassword(16)));
if (SystemVmEnableUserData.valueIn(dc.getId())) {

View File

@ -0,0 +1,940 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Tests for Kubernetes cluster affinity groups feature"""
import unittest
from marvin.cloudstackTestCase import cloudstackTestCase
from marvin.cloudstackAPI import (listInfrastructure,
listKubernetesSupportedVersions,
addKubernetesSupportedVersion,
deleteKubernetesSupportedVersion,
listKubernetesClusters,
createKubernetesCluster,
stopKubernetesCluster,
startKubernetesCluster,
deleteKubernetesCluster,
scaleKubernetesCluster,
destroyVirtualMachine,
deleteNetwork)
from marvin.cloudstackException import CloudstackAPIException
from marvin.lib.base import (ServiceOffering,
Account,
AffinityGroup,
Configurations)
from marvin.lib.utils import (cleanup_resources,
random_gen)
from marvin.lib.common import (get_zone,
get_domain)
from marvin.sshClient import SshClient
from nose.plugins.attrib import attr
from marvin.lib.decoratorGenerators import skipTestIf
import time
_multiprocess_shared_ = True
RAND_SUFFIX = random_gen()
class TestKubernetesClusterAffinityGroups(cloudstackTestCase):
"""
Tests for CKS Affinity Groups feature (since 4.23.0)
This feature allows specifying different affinity groups for each
Kubernetes node type (CONTROL, WORKER, ETCD).
"""
@classmethod
def setUpClass(cls):
testClient = super(TestKubernetesClusterAffinityGroups, cls).getClsTestClient()
if testClient is None:
raise unittest.SkipTest("Marvin test client not available - check marvin configuration")
cls.apiclient = testClient.getApiClient()
cls.services = testClient.getParsedTestDataConfig()
cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
cls.hypervisor = testClient.getHypervisorInfo()
cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__
cls.hypervisorNotSupported = False
if cls.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]:
cls.hypervisorNotSupported = True
cls.setup_failed = False
cls._cleanup = []
cls.kubernetes_version_ids = []
cls.initial_configuration_cks_enabled = None
cls.k8s_version = cls.services.get("cks_kubernetes_version_upgrade_to",
cls.services.get("cks_kubernetes_version_upgrade_from"))
if cls.hypervisorNotSupported == False:
cls.endpoint_url = Configurations.list(cls.apiclient, name="endpoint.url")[0].value
if "localhost" in cls.endpoint_url:
endpoint_url = "http://%s:%d/client/api" % (cls.mgtSvrDetails["mgtSvrIp"], cls.mgtSvrDetails["port"])
cls.debug("Setting endpoint.url to %s" % endpoint_url)
Configurations.update(cls.apiclient, "endpoint.url", endpoint_url)
cls.initial_configuration_cks_enabled = Configurations.list(
cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value
if cls.initial_configuration_cks_enabled not in ["true", True]:
cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server")
Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "true")
cls.restartServer()
cls.cks_service_offering = None
if cls.setup_failed == False:
try:
cls.kubernetes_version = cls.addKubernetesSupportedVersion(
cls.services["cks_kubernetes_versions"][cls.k8s_version])
cls.kubernetes_version_ids.append(cls.kubernetes_version.id)
except Exception as e:
cls.setup_failed = True
cls.debug("Failed to get Kubernetes version ISO in ready state: %s" % e)
if cls.setup_failed == False:
cks_offering_data = cls.services["cks_service_offering"]
cks_offering_data["name"] = 'CKS-Instance-' + random_gen()
cls.cks_service_offering = ServiceOffering.create(
cls.apiclient,
cks_offering_data
)
cls._cleanup.append(cls.cks_service_offering)
cls.domain = get_domain(cls.apiclient)
cls.account = Account.create(
cls.apiclient,
cls.services["account"],
domainid=cls.domain.id
)
cls._cleanup.append(cls.account)
cls.default_network = None
return
@classmethod
def tearDownClass(cls):
# Delete added Kubernetes supported version
for version_id in cls.kubernetes_version_ids:
try:
cls.deleteKubernetesSupportedVersion(version_id)
except Exception as e:
cls.debug("Error during cleanup for Kubernetes versions: %s" % e)
try:
# Restore CKS enabled
if cls.initial_configuration_cks_enabled not in ["true", True]:
cls.debug("Restoring Kubernetes Service enabled value")
Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "false")
cls.restartServer()
cleanup_resources(cls.apiclient, cls._cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
@classmethod
def restartServer(cls):
"""Restart management server"""
cls.debug("Restarting management server")
sshClient = SshClient(
cls.mgtSvrDetails["mgtSvrIp"],
22,
cls.mgtSvrDetails["user"],
cls.mgtSvrDetails["passwd"]
)
command = "service cloudstack-management stop"
sshClient.execute(command)
command = "service cloudstack-management start"
sshClient.execute(command)
# Wait for management to come up in 5 mins
timeout = time.time() + 300
while time.time() < timeout:
if cls.isManagementUp() is True:
return
time.sleep(5)
cls.setup_failed = True
cls.debug("Management server did not come up, failing")
return
@classmethod
def isManagementUp(cls):
try:
cls.apiclient.listInfrastructure(listInfrastructure.listInfrastructureCmd())
return True
except Exception:
return False
@classmethod
def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=30, interval=60):
"""Check if Kubernetes supported version ISO is in Ready state"""
while retries > 0:
time.sleep(interval)
list_versions_response = cls.listKubernetesSupportedVersion(version_id)
if not hasattr(list_versions_response, 'isostate') or not list_versions_response or not list_versions_response.isostate:
retries = retries - 1
continue
if 'Ready' == list_versions_response.isostate:
return
elif 'Failed' == list_versions_response.isostate:
raise Exception("Failed to download template: status - %s" % list_versions_response.isostate)
retries = retries - 1
raise Exception("Kubernetes supported version Ready state timed out")
@classmethod
def listKubernetesSupportedVersion(cls, version_id):
listKubernetesSupportedVersionsCmd = listKubernetesSupportedVersions.listKubernetesSupportedVersionsCmd()
listKubernetesSupportedVersionsCmd.id = version_id
versionResponse = cls.apiclient.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd)
return versionResponse[0]
@classmethod
def addKubernetesSupportedVersion(cls, version_service):
addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd()
addKubernetesSupportedVersionCmd.semanticversion = version_service["semanticversion"]
addKubernetesSupportedVersionCmd.name = 'v' + version_service["semanticversion"] + '-' + random_gen()
addKubernetesSupportedVersionCmd.url = version_service["url"]
addKubernetesSupportedVersionCmd.mincpunumber = version_service["mincpunumber"]
addKubernetesSupportedVersionCmd.minmemory = version_service["minmemory"]
kubernetes_version = cls.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd)
cls.debug("Waiting for Kubernetes version with ID %s to be ready" % kubernetes_version.id)
cls.waitForKubernetesSupportedVersionIsoReadyState(kubernetes_version.id)
kubernetes_version = cls.listKubernetesSupportedVersion(kubernetes_version.id)
return kubernetes_version
@classmethod
def deleteKubernetesSupportedVersion(cls, version_id):
deleteKubernetesSupportedVersionCmd = deleteKubernetesSupportedVersion.deleteKubernetesSupportedVersionCmd()
deleteKubernetesSupportedVersionCmd.id = version_id
cls.apiclient.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd)
@classmethod
def listKubernetesCluster(cls, cluster_id=None, cluster_name=None):
listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd()
listKubernetesClustersCmd.listall = True
if cluster_id is not None:
listKubernetesClustersCmd.id = cluster_id
if cluster_name is not None:
listKubernetesClustersCmd.name = cluster_name
clusterResponse = cls.apiclient.listKubernetesClusters(listKubernetesClustersCmd)
if (cluster_id is not None or cluster_name is not None) and clusterResponse is not None:
return clusterResponse[0]
return clusterResponse
@classmethod
def deleteKubernetesCluster(cls, cluster_id):
deleteKubernetesClusterCmd = deleteKubernetesCluster.deleteKubernetesClusterCmd()
deleteKubernetesClusterCmd.id = cluster_id
response = cls.apiclient.deleteKubernetesCluster(deleteKubernetesClusterCmd)
return response
@classmethod
def stopKubernetesCluster(cls, cluster_id):
stopKubernetesClusterCmd = stopKubernetesCluster.stopKubernetesClusterCmd()
stopKubernetesClusterCmd.id = cluster_id
response = cls.apiclient.stopKubernetesCluster(stopKubernetesClusterCmd)
return response
def setUp(self):
self.services = self.testClient.getParsedTestDataConfig()
self.apiclient = self.testClient.getApiClient()
self.dbclient = self.testClient.getDbConnection()
self.cleanup = []
self.aff_grp = []
return
def tearDown(self):
try:
cleanup_resources(self.apiclient, self.cleanup)
except Exception as e:
raise Exception("Warning: Exception during cleanup : %s" % e)
return
def deleteKubernetesClusterAndVerify(self, cluster_id, verify=True, forced=False):
"""Delete Kubernetes cluster and check if it is really deleted"""
delete_response = {}
forceDeleted = False
try:
delete_response = self.deleteKubernetesCluster(cluster_id)
except Exception as e:
if forced:
cluster = self.listKubernetesCluster(cluster_id)
if cluster is not None:
if cluster.state in ['Starting', 'Running', 'Upgrading', 'Scaling']:
self.stopKubernetesCluster(cluster_id)
self.deleteKubernetesCluster(cluster_id)
else:
forceDeleted = True
for cluster_vm in cluster.virtualmachines:
cmd = destroyVirtualMachine.destroyVirtualMachineCmd()
cmd.id = cluster_vm.id
cmd.expunge = True
self.apiclient.destroyVirtualMachine(cmd)
cmd = deleteNetwork.deleteNetworkCmd()
cmd.id = cluster.networkid
cmd.forced = True
self.apiclient.deleteNetwork(cmd)
self.dbclient.execute(
"update kubernetes_cluster set state='Destroyed', removed=now() where uuid = '%s';" % cluster.id)
else:
raise Exception("Error: Exception during delete cluster : %s" % e)
if verify and not forceDeleted:
self.assertEqual(
delete_response.success,
True,
"Check KubernetesCluster delete response {}, {}".format(delete_response.success, True)
)
db_cluster_removed = \
self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % cluster_id)[0][0]
self.assertNotEqual(
db_cluster_removed,
None,
"KubernetesCluster not removed in DB, {}".format(db_cluster_removed)
)
def create_aff_grp(self, aff_grp_name=None, aff_grp_type="host anti-affinity"):
"""Create an affinity group"""
if aff_grp_name is None:
aff_grp_name = "aff_grp_" + random_gen(size=6)
aff_grp_data = {
"name": aff_grp_name,
"type": aff_grp_type
}
aff_grp = AffinityGroup.create(
self.apiclient,
aff_grp_data,
self.account.name,
self.domain.id
)
self.aff_grp.append(aff_grp)
self.cleanup.append(aff_grp)
return aff_grp
def createKubernetesCluster(self, name, version_id, size=1, control_nodes=1, etcd_nodes=0,
control_aff_grp=None, worker_aff_grp=None, etcd_aff_grp=None):
"""Create a Kubernetes cluster with optional affinity groups for each node type"""
createKubernetesClusterCmd = createKubernetesCluster.createKubernetesClusterCmd()
createKubernetesClusterCmd.name = name
createKubernetesClusterCmd.description = name + "-description"
createKubernetesClusterCmd.kubernetesversionid = version_id
createKubernetesClusterCmd.size = size
createKubernetesClusterCmd.controlnodes = control_nodes
createKubernetesClusterCmd.serviceofferingid = self.cks_service_offering.id
createKubernetesClusterCmd.zoneid = self.zone.id
createKubernetesClusterCmd.noderootdisksize = 10
createKubernetesClusterCmd.account = self.account.name
createKubernetesClusterCmd.domainid = self.domain.id
if etcd_nodes > 0:
createKubernetesClusterCmd.etcdnodes = etcd_nodes
# Set affinity groups for node types using the nodeaffinitygroups parameter
# Format: list of {node: "<NODE_TYPE>", affinitygroup: "<UUID>"}
if control_aff_grp is not None:
if not hasattr(createKubernetesClusterCmd, 'nodeaffinitygroups'):
createKubernetesClusterCmd.nodeaffinitygroups = []
createKubernetesClusterCmd.nodeaffinitygroups.append({
"node": "CONTROL",
"affinitygroup": control_aff_grp.id
})
if worker_aff_grp is not None:
if not hasattr(createKubernetesClusterCmd, 'nodeaffinitygroups'):
createKubernetesClusterCmd.nodeaffinitygroups = []
createKubernetesClusterCmd.nodeaffinitygroups.append({
"node": "WORKER",
"affinitygroup": worker_aff_grp.id
})
if etcd_aff_grp is not None:
if not hasattr(createKubernetesClusterCmd, 'nodeaffinitygroups'):
createKubernetesClusterCmd.nodeaffinitygroups = []
createKubernetesClusterCmd.nodeaffinitygroups.append({
"node": "ETCD",
"affinitygroup": etcd_aff_grp.id
})
if self.default_network:
createKubernetesClusterCmd.networkid = self.default_network.id
clusterResponse = self.apiclient.createKubernetesCluster(createKubernetesClusterCmd)
return clusterResponse
def startKubernetesCluster(self, cluster_id):
startKubernetesClusterCmd = startKubernetesCluster.startKubernetesClusterCmd()
startKubernetesClusterCmd.id = cluster_id
response = self.apiclient.startKubernetesCluster(startKubernetesClusterCmd)
return response
def scaleKubernetesCluster(self, cluster_id, size):
scaleKubernetesClusterCmd = scaleKubernetesCluster.scaleKubernetesClusterCmd()
scaleKubernetesClusterCmd.id = cluster_id
scaleKubernetesClusterCmd.size = size
response = self.apiclient.scaleKubernetesCluster(scaleKubernetesClusterCmd)
return response
def verifyKubernetesClusterState(self, cluster_response, state):
"""Check if Kubernetes cluster state matches expected state"""
self.assertEqual(
cluster_response.state,
state,
"Check KubernetesCluster state {}, expected {}".format(cluster_response.state, state)
)
def verifyKubernetesClusterAffinityGroups(self, cluster, control_aff_grp=None,
worker_aff_grp=None, etcd_aff_grp=None):
"""Verify affinity groups are correctly assigned to the cluster"""
if control_aff_grp is not None:
self.assertEqual(
cluster.controlnodeaffinitygroupid,
control_aff_grp.id,
"Control node affinity group ID mismatch. Expected: {}, Got: {}".format(
control_aff_grp.id, cluster.controlnodeaffinitygroupid)
)
self.assertEqual(
cluster.controlnodeaffinitygroupname,
control_aff_grp.name,
"Control node affinity group name mismatch. Expected: {}, Got: {}".format(
control_aff_grp.name, cluster.controlnodeaffinitygroupname)
)
else:
self.assertTrue(
not hasattr(cluster, 'controlnodeaffinitygroupid') or cluster.controlnodeaffinitygroupid is None,
"Control node affinity group should be None"
)
if worker_aff_grp is not None:
self.assertEqual(
cluster.workernodeaffinitygroupid,
worker_aff_grp.id,
"Worker node affinity group ID mismatch. Expected: {}, Got: {}".format(
worker_aff_grp.id, cluster.workernodeaffinitygroupid)
)
self.assertEqual(
cluster.workernodeaffinitygroupname,
worker_aff_grp.name,
"Worker node affinity group name mismatch. Expected: {}, Got: {}".format(
worker_aff_grp.name, cluster.workernodeaffinitygroupname)
)
else:
self.assertTrue(
not hasattr(cluster, 'workernodeaffinitygroupid') or cluster.workernodeaffinitygroupid is None,
"Worker node affinity group should be None"
)
if etcd_aff_grp is not None:
self.assertEqual(
cluster.etcdnodeaffinitygroupid,
etcd_aff_grp.id,
"ETCD node affinity group ID mismatch. Expected: {}, Got: {}".format(
etcd_aff_grp.id, cluster.etcdnodeaffinitygroupid)
)
self.assertEqual(
cluster.etcdnodeaffinitygroupname,
etcd_aff_grp.name,
"ETCD node affinity group name mismatch. Expected: {}, Got: {}".format(
etcd_aff_grp.name, cluster.etcdnodeaffinitygroupname)
)
else:
self.assertTrue(
not hasattr(cluster, 'etcdnodeaffinitygroupid') or cluster.etcdnodeaffinitygroupid is None,
"ETCD node affinity group should be None"
)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_01_create_cluster_with_control_node_affinity_group(self):
"""Test creating a Kubernetes cluster with affinity group for control nodes only
# Validate the following:
# 1. Create an affinity group
# 2. Create a Kubernetes cluster with the affinity group for control nodes
# 3. Verify cluster is created successfully and affinity group is assigned
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating affinity group for control nodes")
control_aff_grp = self.create_aff_grp(aff_grp_name="control-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with control node affinity group")
cluster_name = "cks-aff-grp-control-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
control_aff_grp=control_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
self.verifyKubernetesClusterAffinityGroups(
cluster,
control_aff_grp=control_aff_grp,
worker_aff_grp=None,
etcd_aff_grp=None
)
self.debug("Kubernetes cluster with control node affinity group created successfully")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_02_create_cluster_with_worker_node_affinity_group(self):
"""Test creating a Kubernetes cluster with affinity group for worker nodes only
# Validate the following:
# 1. Create an affinity group
# 2. Create a Kubernetes cluster with the affinity group for worker nodes
# 3. Verify cluster is created successfully and affinity group is assigned
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating affinity group for worker nodes")
worker_aff_grp = self.create_aff_grp(aff_grp_name="worker-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with worker node affinity group")
cluster_name = "cks-aff-grp-worker-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
worker_aff_grp=worker_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
self.verifyKubernetesClusterAffinityGroups(
cluster,
control_aff_grp=None,
worker_aff_grp=worker_aff_grp,
etcd_aff_grp=None
)
self.debug("Kubernetes cluster with worker node affinity group created successfully")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_03_create_cluster_with_all_node_type_affinity_groups(self):
"""Test creating a Kubernetes cluster with different affinity groups for all node types
# Validate the following:
# 1. Create separate affinity groups for control, worker, and etcd nodes
# 2. Create a Kubernetes cluster with affinity groups for all node types
# 3. Verify cluster is created successfully and all affinity groups are assigned
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating affinity groups for all node types")
control_aff_grp = self.create_aff_grp(aff_grp_name="control-aff-grp-" + random_gen())
worker_aff_grp = self.create_aff_grp(aff_grp_name="worker-aff-grp-" + random_gen())
etcd_aff_grp = self.create_aff_grp(aff_grp_name="etcd-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with affinity groups for all node types")
cluster_name = "cks-aff-grp-all-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
etcd_nodes=1,
control_aff_grp=control_aff_grp,
worker_aff_grp=worker_aff_grp,
etcd_aff_grp=etcd_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
self.verifyKubernetesClusterAffinityGroups(
cluster,
control_aff_grp=control_aff_grp,
worker_aff_grp=worker_aff_grp,
etcd_aff_grp=etcd_aff_grp
)
self.debug("Kubernetes cluster with all node type affinity groups created successfully")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_04_create_cluster_with_same_affinity_group_all_node_types(self):
"""Test creating a Kubernetes cluster with the same affinity group for all node types
# Validate the following:
# 1. Create a single affinity group
# 2. Create a Kubernetes cluster using the same affinity group for all node types
# 3. Verify cluster is created successfully
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating a single affinity group for all node types")
shared_aff_grp = self.create_aff_grp(aff_grp_name="shared-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with same affinity group for all node types")
cluster_name = "cks-aff-grp-shared-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
control_aff_grp=shared_aff_grp,
worker_aff_grp=shared_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
self.verifyKubernetesClusterAffinityGroups(
cluster,
control_aff_grp=shared_aff_grp,
worker_aff_grp=shared_aff_grp,
etcd_aff_grp=None
)
self.debug("Kubernetes cluster with shared affinity group created successfully")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_05_scale_cluster_respects_worker_affinity_group(self):
"""Test that scaling a cluster respects the worker node affinity group
# Validate the following:
# 1. Create a cluster with worker node affinity group
# 2. Scale up the cluster
# 3. Verify affinity group assignments are preserved after scaling
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating affinity group for worker nodes")
worker_aff_grp = self.create_aff_grp(aff_grp_name="worker-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with worker node affinity group")
cluster_name = "cks-aff-grp-scale-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
worker_aff_grp=worker_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
self.debug("Scaling up Kubernetes cluster from 1 to 2 worker nodes")
cluster = self.scaleKubernetesCluster(cluster.id, 2)
self.verifyKubernetesClusterState(cluster, 'Running')
self.assertEqual(
cluster.size,
2,
"Cluster size should be 2 after scaling up, got {}".format(cluster.size)
)
# Verify affinity group is still assigned after scaling
self.verifyKubernetesClusterAffinityGroups(
cluster,
control_aff_grp=None,
worker_aff_grp=worker_aff_grp,
etcd_aff_grp=None
)
self.debug("Kubernetes cluster scaled successfully with affinity group preserved")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_06_stop_start_cluster_preserves_affinity_groups(self):
"""Test that stopping and starting a cluster preserves affinity groups
# Validate the following:
# 1. Create a cluster with affinity groups
# 2. Stop the cluster
# 3. Start the cluster
# 4. Verify affinity groups are preserved
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating affinity groups")
control_aff_grp = self.create_aff_grp(aff_grp_name="control-aff-grp-" + random_gen())
worker_aff_grp = self.create_aff_grp(aff_grp_name="worker-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with affinity groups")
cluster_name = "cks-aff-grp-lifecycle-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
control_aff_grp=control_aff_grp,
worker_aff_grp=worker_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
self.debug("Stopping Kubernetes cluster")
self.stopKubernetesCluster(cluster.id)
cluster = self.listKubernetesCluster(cluster.id)
self.assertEqual(
cluster.state,
'Stopped',
"Cluster should be in Stopped state, got {}".format(cluster.state)
)
self.debug("Starting Kubernetes cluster")
cluster = self.startKubernetesCluster(cluster.id)
self.verifyKubernetesClusterState(cluster, 'Running')
# Verify affinity groups are preserved after stop/start
self.verifyKubernetesClusterAffinityGroups(
cluster,
control_aff_grp=control_aff_grp,
worker_aff_grp=worker_aff_grp,
etcd_aff_grp=None
)
self.debug("Kubernetes cluster stop/start completed with affinity groups preserved")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_07_create_cluster_with_invalid_affinity_group_id(self):
"""Test creating a cluster with an invalid affinity group ID fails
# Validate the following:
# 1. Attempt to create a cluster with a non-existent affinity group ID
# 2. Verify the operation fails with appropriate error
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating Kubernetes cluster with invalid affinity group ID")
cluster_name = "cks-aff-grp-invalid-" + random_gen()
# Create a fake affinity group object with invalid ID
class FakeAffinityGroup:
def __init__(self):
self.id = "invalid-uuid-12345"
self.name = "fake-group"
fake_aff_grp = FakeAffinityGroup()
with self.assertRaises(Exception) as context:
self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
control_aff_grp=fake_aff_grp
)
self.debug("Expected error when creating cluster with invalid affinity group: %s" % context.exception)
# Clean up any partially created cluster
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_08_cluster_response_includes_affinity_group_details(self):
"""Test that cluster list response includes affinity group details
# Validate the following:
# 1. Create a cluster with affinity groups
# 2. List the cluster
# 3. Verify response includes affinity group IDs and names
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating affinity groups")
control_aff_grp = self.create_aff_grp(aff_grp_name="control-aff-grp-" + random_gen())
worker_aff_grp = self.create_aff_grp(aff_grp_name="worker-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with affinity groups")
cluster_name = "cks-aff-grp-response-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
control_aff_grp=control_aff_grp,
worker_aff_grp=worker_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
# List the cluster and verify response
listed_cluster = self.listKubernetesCluster(cluster.id)
self.assertIsNotNone(listed_cluster, "Cluster should be listed")
# Verify control node affinity group in response
self.assertTrue(
hasattr(listed_cluster, 'controlnodeaffinitygroupid'),
"Response should include controlnodeaffinitygroupid"
)
self.assertTrue(
hasattr(listed_cluster, 'controlnodeaffinitygroupname'),
"Response should include controlnodeaffinitygroupname"
)
# Verify worker node affinity group in response
self.assertTrue(
hasattr(listed_cluster, 'workernodeaffinitygroupid'),
"Response should include workernodeaffinitygroupid"
)
self.assertTrue(
hasattr(listed_cluster, 'workernodeaffinitygroupname'),
"Response should include workernodeaffinitygroupname"
)
# Verify the values match
self.verifyKubernetesClusterAffinityGroups(
listed_cluster,
control_aff_grp=control_aff_grp,
worker_aff_grp=worker_aff_grp,
etcd_aff_grp=None
)
self.debug("Cluster response includes correct affinity group details")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_09_create_cluster_without_affinity_groups(self):
"""Test creating a cluster without any affinity groups
# Validate the following:
# 1. Create a cluster without specifying any affinity groups
# 2. Verify cluster is created and affinity group fields are null/empty
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating Kubernetes cluster without affinity groups")
cluster_name = "cks-no-aff-grp-" + random_gen()
try:
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1
)
self.verifyKubernetesClusterState(cluster, 'Running')
# Verify no affinity groups are assigned
self.verifyKubernetesClusterAffinityGroups(
cluster,
control_aff_grp=None,
worker_aff_grp=None,
etcd_aff_grp=None
)
self.debug("Kubernetes cluster created successfully without affinity groups")
finally:
cluster = self.listKubernetesCluster(cluster_name=cluster_name)
if cluster is not None:
self.deleteKubernetesClusterAndVerify(cluster.id, False, True)
@attr(tags=["advanced", "smoke"], required_hardware="true")
@skipTestIf("hypervisorNotSupported")
def test_10_delete_cluster_with_affinity_groups(self):
"""Test that deleting a cluster with affinity groups works correctly
# Validate the following:
# 1. Create a cluster with affinity groups
# 2. Delete the cluster
# 3. Verify cluster is deleted and affinity groups still exist
"""
if self.setup_failed:
self.fail("Setup incomplete")
self.debug("Creating affinity groups")
control_aff_grp = self.create_aff_grp(aff_grp_name="control-aff-grp-" + random_gen())
worker_aff_grp = self.create_aff_grp(aff_grp_name="worker-aff-grp-" + random_gen())
self.debug("Creating Kubernetes cluster with affinity groups")
cluster_name = "cks-aff-grp-delete-" + random_gen()
cluster = self.createKubernetesCluster(
cluster_name,
self.kubernetes_version.id,
size=1,
control_nodes=1,
control_aff_grp=control_aff_grp,
worker_aff_grp=worker_aff_grp
)
self.verifyKubernetesClusterState(cluster, 'Running')
cluster_id = cluster.id
self.debug("Deleting Kubernetes cluster")
self.deleteKubernetesClusterAndVerify(cluster_id)
# Verify cluster is deleted
deleted_cluster = self.listKubernetesCluster(cluster_id)
self.assertIsNone(deleted_cluster, "Cluster should be deleted")
# Verify affinity groups still exist
control_aff_grp_list = AffinityGroup.list(self.apiclient, id=control_aff_grp.id)
self.assertIsNotNone(control_aff_grp_list, "Control affinity group should still exist")
worker_aff_grp_list = AffinityGroup.list(self.apiclient, id=worker_aff_grp.id)
self.assertIsNotNone(worker_aff_grp_list, "Worker affinity group should still exist")
self.debug("Kubernetes cluster deleted successfully, affinity groups preserved")

View File

@ -556,6 +556,9 @@
"label.cks.cluster.size": "Cluster size (Worker nodes)",
"label.cks.cluster.worker.nodes.offeringid": "Service Offering for Worker Nodes",
"label.cks.cluster.worker.nodes.templateid": "Template for Worker Nodes",
"label.cks.cluster.control.nodes.affinitygroupid": "Affinity Groups for Control Nodes",
"label.cks.cluster.worker.nodes.affinitygroupid": "Affinity Groups for Worker Nodes",
"label.cks.cluster.etcd.nodes.affinitygroupid": "Affinity Groups for ETCD Nodes",
"label.cleanup": "Clean up",
"label.clear": "Clear",
"label.clear.all": "Clear all",
@ -1382,6 +1385,7 @@
"label.isoname": "Attached ISO",
"label.isos": "ISOs",
"label.isostate": "ISO state",
"label.isourl": "ISO URL",
"label.ispersistent": "Persistent ",
"label.ispublic": "Public",
"label.isready": "Ready",
@ -2885,6 +2889,9 @@
"label.edgecluster": "Edge Cluster",
"label.encryption": "Encryption",
"label.etcdnodes": "Number of etcd nodes",
"label.controlaffinitygroupnames": "Control Affinity Groups",
"label.workeraffinitygroupnames": "Worker Affinity Groups",
"label.etcdaffinitygroupnames": "ETCD Affinity Groups",
"label.versioning": "Versioning",
"label.objectlocking": "Object Lock",
"label.bucket.policy": "Bucket Policy",

View File

@ -881,6 +881,7 @@
"label.isoname": "Imagem ISO plugada",
"label.isos": "ISOs",
"label.isostate": "Estado da ISO",
"label.isourl": "URL da ISO",
"label.ispersistent": "Persistente",
"label.ispublic": "P\u00fablico",
"label.isready": "Pronto",

View File

@ -574,7 +574,7 @@ export default {
const filters = ['cloud.managed', 'external.managed']
return filters
},
details: ['name', 'description', 'zonename', 'kubernetesversionname', 'autoscalingenabled', 'csienabled', 'minsize', 'maxsize', 'size', 'controlnodes', 'etcdnodes', 'cpunumber', 'memory', 'keypair', 'cniconfigname', 'associatednetworkname', 'account', 'domain', 'zonename', 'clustertype', 'created'],
details: ['name', 'description', 'zonename', 'kubernetesversionname', 'autoscalingenabled', 'csienabled', 'minsize', 'maxsize', 'size', 'controlnodes', 'controlaffinitygroupnames', 'etcdnodes', 'etcdaffinitygroupnames', 'workeraffinitygroupnames', 'cpunumber', 'memory', 'keypair', 'cniconfigname', 'associatednetworkname', 'account', 'domain', 'zonename', 'clustertype', 'created'],
tabs: [
{
name: 'k8s',

View File

@ -61,9 +61,9 @@ export default {
details: () => {
var fields = ['name', 'id', 'displaytext', 'checksum', 'hypervisor', 'arch', 'format', 'externalprovisioner', 'ostypename', 'size', 'physicalsize', 'isready', 'passwordenabled',
'crossZones', 'templatetype', 'directdownload', 'deployasis', 'ispublic', 'isfeatured', 'isextractable', 'isdynamicallyscalable', 'crosszones', 'type',
'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'forcks']
'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'url', 'forcks']
if (['Admin'].includes(store.getters.userInfo.roletype)) {
fields.push('templatetag', 'templatetype', 'url')
fields.push('templatetag', 'templatetype')
}
return fields
},
@ -373,7 +373,7 @@ export default {
permission: ['listKubernetesSupportedVersions'],
searchFilters: ['zoneid', 'minimumsemanticversion', 'arch'],
columns: ['name', 'state', 'semanticversion', 'isostate', 'mincpunumber', 'minmemory', 'arch', 'zonename'],
details: ['name', 'semanticversion', 'supportsautoscaling', 'zoneid', 'zonename', 'isoid', 'isoname', 'isostate', 'arch', 'mincpunumber', 'minmemory', 'supportsha', 'state', 'created'],
details: ['name', 'semanticversion', 'supportsautoscaling', 'zoneid', 'zonename', 'isoid', 'isoname', 'isostate', 'arch', 'mincpunumber', 'minmemory', 'supportsha', 'state', 'created', 'isourl'],
tabs: [
{
name: 'details',

View File

@ -162,7 +162,7 @@ export default {
postAPI('forgotPassword', loginParams)
.finally(() => {
this.$message.success(this.$t('message.forgot.password.success'))
this.$router.push({ path: '/login' }).catch(() => {})
this.$router.replace({ path: '/user/login' })
})
}).catch(error => {
this.formRef.value.scrollToField(error.errorFields[0].name)

View File

@ -316,7 +316,7 @@
</a-select-option>
</a-select>
</a-form-item>
<a-form-item v-if="form.advancedmode && form?.etcdnodes > 0" name="etcdtemplateid" ref="etcdtemplateid">
<a-form-item v-if="form.advancedmode && form.etcdnodes && form.etcdnodes > 0" name="etcdtemplateid" ref="etcdtemplateid">
<template #label>
<tooltip-label :title="$t('label.cks.cluster.etcd.nodes.templateid')" :tooltip="$t('label.cks.cluster.etcd.nodes.templateid')"/>
</template>
@ -335,6 +335,57 @@
</a-select-option>
</a-select>
</a-form-item>
<a-form-item v-if="form.advancedmode" name="controlaffinitygroupids" ref="controlaffinitygroupids">
<template #label>
<tooltip-label :title="$t('label.cks.cluster.control.nodes.affinitygroupid')" :tooltip="$t('label.cks.cluster.control.nodes.affinitygroupid')"/>
</template>
<a-select
v-model:value="controlAffinityGroups"
mode="multiple"
showSearch
optionFilterProp="label"
:filterOption="(input, option) => option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0"
:loading="affinityGroupLoading"
:placeholder="$t('label.cks.cluster.control.nodes.affinitygroupid')">
<a-select-option v-for="opt in affinityGroups" :key="opt.id" :label="opt.name">
{{ opt.name }}
</a-select-option>
</a-select>
</a-form-item>
<a-form-item v-if="form.advancedmode" name="workeraffinitygroupids" ref="workeraffinitygroupids">
<template #label>
<tooltip-label :title="$t('label.cks.cluster.worker.nodes.affinitygroupid')" :tooltip="$t('label.cks.cluster.worker.nodes.affinitygroupid')"/>
</template>
<a-select
v-model:value="workerAffinityGroups"
mode="multiple"
showSearch
optionFilterProp="label"
:filterOption="(input, option) => option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0"
:loading="affinityGroupLoading"
:placeholder="$t('label.cks.cluster.worker.nodes.affinitygroupid')">
<a-select-option v-for="opt in affinityGroups" :key="opt.id" :label="opt.name">
{{ opt.name }}
</a-select-option>
</a-select>
</a-form-item>
<a-form-item v-if="form.advancedmode && form.etcdnodes && form.etcdnodes > 0" name="etcdaffinitygroupids" ref="etcdaffinitygroupids">
<template #label>
<tooltip-label :title="$t('label.cks.cluster.etcd.nodes.affinitygroupid')" :tooltip="$t('label.cks.cluster.etcd.nodes.affinitygroupid')"/>
</template>
<a-select
v-model:value="etcdAffinityGroups"
mode="multiple"
showSearch
optionFilterProp="label"
:filterOption="(input, option) => option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0"
:loading="affinityGroupLoading"
:placeholder="$t('label.cks.cluster.etcd.nodes.affinitygroupid')">
<a-select-option v-for="opt in affinityGroups" :key="opt.id" :label="opt.name">
{{ opt.name }}
</a-select-option>
</a-select>
</a-form-item>
<a-form-item v-if="form.advancedmode && isASNumberRequired() && !form.networkid" name="asnumber" ref="asnumber">
<template #label>
<tooltip-label :title="$t('label.asnumber')" :tooltip="apiParams.asnumber.description"/>
@ -490,7 +541,12 @@ export default {
cksNetworkOffering: null,
asNumbersZone: [],
asNumberLoading: false,
selectedAsNumber: 0
selectedAsNumber: 0,
affinityGroups: [],
affinityGroupLoading: false,
controlAffinityGroups: [],
workerAffinityGroups: [],
etcdAffinityGroups: []
}
},
beforeCreate () {
@ -560,6 +616,22 @@ export default {
this.fetchCKSNetworkOfferingName()
this.fetchCniConfigurations()
},
fetchAffinityGroups () {
this.affinityGroups = []
const params = {}
if (!this.isObjectEmpty(this.selectedZone)) {
params.zoneid = this.selectedZone.id
}
this.affinityGroupLoading = true
getAPI('listAffinityGroups', params).then(json => {
const groups = json.listaffinitygroupsresponse.affinitygroup
if (this.arrayHasItems(groups)) {
this.affinityGroups = groups
}
}).finally(() => {
this.affinityGroupLoading = false
})
},
isValidValueForKey (obj, key) {
return key in obj && obj[key] != null
},
@ -600,6 +672,7 @@ export default {
this.fetchNetworkData()
this.fetchZoneHypervisors()
this.fetchZoneASNumbers()
this.fetchAffinityGroups()
},
handleASNumberChange (selectedIndex) {
this.selectedAsNumber = this.asNumbersZone[selectedIndex].asnumber
@ -881,6 +954,21 @@ export default {
advancedTemplates++
}
}
if (values.advancedmode) {
let affinityIndex = 0
const addAffinityGroups = (nodeType, groups) => {
if (groups && groups.length > 0) {
params[`nodeaffinitygroups[${affinityIndex}].node`] = nodeType
params[`nodeaffinitygroups[${affinityIndex}].affinitygroup`] = groups.join(',')
affinityIndex++
}
}
addAffinityGroups('control', this.controlAffinityGroups)
addAffinityGroups('worker', this.workerAffinityGroups)
if (values.etcdnodes > 0) {
addAffinityGroups('etcd', this.etcdAffinityGroups)
}
}
if (this.isValidValueForKey(values, 'noderootdisksize') && values.noderootdisksize > 0) {
params.noderootdisksize = values.noderootdisksize
}