implement node affinity group validation method

This commit is contained in:
Daman Arora 2026-01-12 09:16:03 -05:00
parent d27b2f45be
commit cd37b81147
2 changed files with 417 additions and 0 deletions

View File

@ -2357,6 +2357,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
if (validNodeIds.isEmpty()) {
throw new CloudRuntimeException("No valid nodes found to be added to the Kubernetes cluster");
}
validateNodeAffinityGroups(validNodeIds, kubernetesCluster);
KubernetesClusterAddWorker addWorker = new KubernetesClusterAddWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
addWorker = ComponentContext.inject(addWorker);
return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr(), cmd.isManualUpgrade());
@ -2416,6 +2417,83 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
return validNodeIds;
}
protected void validateNodeAffinityGroups(List<Long> nodeIds, KubernetesCluster cluster) {
List<Long> workerAffinityGroupIds = kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(
cluster.getId(), WORKER.name());
if (CollectionUtils.isEmpty(workerAffinityGroupIds)) {
return;
}
List<KubernetesClusterVmMapVO> existingWorkerVms = kubernetesClusterVmMapDao.listByClusterIdAndVmType(cluster.getId(), WORKER);
Set<Long> existingWorkerHostIds = new HashSet<>();
for (KubernetesClusterVmMapVO workerVmMap : existingWorkerVms) {
VMInstanceVO workerVm = vmInstanceDao.findById(workerVmMap.getVmId());
if (workerVm != null && workerVm.getHostId() != null) {
existingWorkerHostIds.add(workerVm.getHostId());
}
}
for (Long affinityGroupId : workerAffinityGroupIds) {
AffinityGroupVO affinityGroup = affinityGroupDao.findById(affinityGroupId);
if (affinityGroup == null) {
continue;
}
String affinityGroupType = affinityGroup.getType();
for (Long nodeId : nodeIds) {
VMInstanceVO node = vmInstanceDao.findById(nodeId);
if (node == null || node.getHostId() == null) {
continue;
}
Long nodeHostId = node.getHostId();
HostVO nodeHost = hostDao.findById(nodeHostId);
String nodeHostName = nodeHost != null ? nodeHost.getName() : String.valueOf(nodeHostId);
if ("host anti-affinity".equalsIgnoreCase(affinityGroupType)) {
if (existingWorkerHostIds.contains(nodeHostId)) {
throw new InvalidParameterValueException(String.format(
"Cannot add VM %s to cluster %s. VM is running on host %s which violates the cluster's " +
"host anti-affinity rule (affinity group: %s). Existing worker VMs are already running on this host.",
node.getInstanceName(), cluster.getName(), nodeHostName, affinityGroup.getName()));
}
} else if ("host affinity".equalsIgnoreCase(affinityGroupType)) {
if (!existingWorkerHostIds.isEmpty() && !existingWorkerHostIds.contains(nodeHostId)) {
List<String> existingHostNames = new ArrayList<>();
for (Long hostId : existingWorkerHostIds) {
HostVO host = hostDao.findById(hostId);
existingHostNames.add(host != null ? host.getName() : String.valueOf(hostId));
}
throw new InvalidParameterValueException(String.format(
"Cannot add VM %s to cluster %s. VM is running on host %s which violates the cluster's " +
"host affinity rule (affinity group: %s). All worker VMs must run on the same host. " +
"Existing workers are on host(s): %s.",
node.getInstanceName(), cluster.getName(), nodeHostName, affinityGroup.getName(),
String.join(", ", existingHostNames)));
}
}
}
if ("host anti-affinity".equalsIgnoreCase(affinityGroupType)) {
Set<Long> newNodeHostIds = new HashSet<>();
for (Long nodeId : nodeIds) {
VMInstanceVO node = vmInstanceDao.findById(nodeId);
if (node != null && node.getHostId() != null) {
Long nodeHostId = node.getHostId();
if (newNodeHostIds.contains(nodeHostId)) {
HostVO nodeHost = hostDao.findById(nodeHostId);
String nodeHostName = nodeHost != null ? nodeHost.getName() : String.valueOf(nodeHostId);
throw new InvalidParameterValueException(String.format(
"Cannot add VM %s to cluster %s. Multiple VMs being added are running on the same host %s, " +
"which violates the cluster's host anti-affinity rule (affinity group: %s).",
node.getInstanceName(), cluster.getName(), nodeHostName, affinityGroup.getName()));
}
newNodeHostIds.add(nodeHostId);
}
}
}
}
}
@Override
public List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd) {
if (!KubernetesServiceEnabled.value()) {

View File

@ -47,6 +47,8 @@ import com.cloud.utils.Pair;
import com.cloud.utils.net.NetUtils;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import org.apache.cloudstack.affinity.AffinityGroupVO;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.api.BaseCmd;
@ -113,6 +115,9 @@ public class KubernetesClusterManagerImplTest {
@Mock
private AffinityGroupDao affinityGroupDao;
@Mock
private HostDao hostDao;
@Spy
@InjectMocks
KubernetesClusterManagerImpl kubernetesClusterManager;
@ -575,4 +580,338 @@ public class KubernetesClusterManagerImplTest {
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(clusterId, ETCD.name());
}
@Test
public void testValidateNodeAffinityGroupsNoAffinityGroups() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
List<Long> nodeIds = Arrays.asList(100L, 101L);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Collections.emptyList());
kubernetesClusterManager.validateNodeAffinityGroups(nodeIds, cluster);
Mockito.verify(kubernetesClusterVmMapDao, Mockito.never()).listByClusterIdAndVmType(Mockito.anyLong(), Mockito.any());
}
@Test
public void testValidateNodeAffinityGroupsNullAffinityGroups() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
List<Long> nodeIds = Arrays.asList(100L, 101L);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(null);
kubernetesClusterManager.validateNodeAffinityGroups(nodeIds, cluster);
Mockito.verify(kubernetesClusterVmMapDao, Mockito.never()).listByClusterIdAndVmType(Mockito.anyLong(), Mockito.any());
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateNodeAffinityGroupsAntiAffinityNewNodeOnExistingHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long sharedHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(sharedHostId);
Mockito.when(newNode.getInstanceName()).thenReturn("new-node-vm");
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(sharedHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getName()).thenReturn("host-1");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
Mockito.when(hostDao.findById(sharedHostId)).thenReturn(host);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
}
@Test
public void testValidateNodeAffinityGroupsAntiAffinityNewNodeOnDifferentHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long existingHostId = 1000L;
Long newNodeHostId = 1001L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(newNodeHostId);
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(existingHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
@Test
public void testValidateNodeAffinityGroupsAffinityNewNodeOnSameHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long sharedHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host affinity");
Mockito.when(affinityGroup.getName()).thenReturn("affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(sharedHostId);
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(sharedHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateNodeAffinityGroupsAffinityNewNodeOnDifferentHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long existingWorkerVmId = 200L;
Long existingHostId = 1000L;
Long newNodeHostId = 1001L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host affinity");
Mockito.when(affinityGroup.getName()).thenReturn("affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(newNodeHostId);
Mockito.when(newNode.getInstanceName()).thenReturn("new-node-vm");
VMInstanceVO existingWorkerVm = Mockito.mock(VMInstanceVO.class);
Mockito.when(existingWorkerVm.getHostId()).thenReturn(existingHostId);
KubernetesClusterVmMapVO workerVmMap = Mockito.mock(KubernetesClusterVmMapVO.class);
Mockito.when(workerVmMap.getVmId()).thenReturn(existingWorkerVmId);
HostVO newHost = Mockito.mock(HostVO.class);
Mockito.when(newHost.getName()).thenReturn("host-2");
HostVO existingHost = Mockito.mock(HostVO.class);
Mockito.when(existingHost.getName()).thenReturn("host-1");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Arrays.asList(workerVmMap));
Mockito.when(vmInstanceDao.findById(existingWorkerVmId)).thenReturn(existingWorkerVm);
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
Mockito.when(hostDao.findById(newNodeHostId)).thenReturn(newHost);
Mockito.when(hostDao.findById(existingHostId)).thenReturn(existingHost);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
}
@Test(expected = InvalidParameterValueException.class)
public void testValidateNodeAffinityGroupsAntiAffinityMultipleNewNodesOnSameHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId1 = 100L;
Long newNodeId2 = 101L;
Long sharedHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode1 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode1.getHostId()).thenReturn(sharedHostId);
Mockito.when(newNode1.getInstanceName()).thenReturn("new-node-vm-1");
VMInstanceVO newNode2 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode2.getHostId()).thenReturn(sharedHostId);
Mockito.when(newNode2.getInstanceName()).thenReturn("new-node-vm-2");
HostVO host = Mockito.mock(HostVO.class);
Mockito.when(host.getName()).thenReturn("host-1");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId1)).thenReturn(newNode1);
Mockito.when(vmInstanceDao.findById(newNodeId2)).thenReturn(newNode2);
Mockito.when(hostDao.findById(sharedHostId)).thenReturn(host);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId1, newNodeId2), cluster);
}
@Test
public void testValidateNodeAffinityGroupsAntiAffinityMultipleNewNodesOnDifferentHosts() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId1 = 100L;
Long newNodeId2 = 101L;
Long hostId1 = 1000L;
Long hostId2 = 1001L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode1 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode1.getHostId()).thenReturn(hostId1);
VMInstanceVO newNode2 = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode2.getHostId()).thenReturn(hostId2);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId1)).thenReturn(newNode1);
Mockito.when(vmInstanceDao.findById(newNodeId2)).thenReturn(newNode2);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId1, newNodeId2), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
@Test
public void testValidateNodeAffinityGroupsNodeWithNullHost() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(null);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(vmInstanceDao).findById(newNodeId);
}
@Test
public void testValidateNodeAffinityGroupsNullNode() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host anti-affinity");
Mockito.when(affinityGroup.getName()).thenReturn("anti-affinity-group");
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(null);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(vmInstanceDao).findById(newNodeId);
}
@Test
public void testValidateNodeAffinityGroupsAffinityNoExistingWorkers() {
KubernetesCluster cluster = Mockito.mock(KubernetesCluster.class);
Mockito.when(cluster.getId()).thenReturn(1L);
Mockito.when(cluster.getName()).thenReturn("test-cluster");
Long newNodeId = 100L;
Long newNodeHostId = 1000L;
AffinityGroupVO affinityGroup = Mockito.mock(AffinityGroupVO.class);
Mockito.when(affinityGroup.getType()).thenReturn("host affinity");
Mockito.when(affinityGroup.getName()).thenReturn("affinity-group");
VMInstanceVO newNode = Mockito.mock(VMInstanceVO.class);
Mockito.when(newNode.getHostId()).thenReturn(newNodeHostId);
Mockito.when(kubernetesClusterAffinityGroupMapDao.listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name()))
.thenReturn(Arrays.asList(10L));
Mockito.when(affinityGroupDao.findById(10L)).thenReturn(affinityGroup);
Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(1L, WORKER))
.thenReturn(Collections.emptyList());
Mockito.when(vmInstanceDao.findById(newNodeId)).thenReturn(newNode);
kubernetesClusterManager.validateNodeAffinityGroups(Arrays.asList(newNodeId), cluster);
Mockito.verify(kubernetesClusterAffinityGroupMapDao).listAffinityGroupIdsByClusterIdAndNodeType(1L, WORKER.name());
}
}