etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NSX: Cleanup NSX resources during k8s cluster cleanup

This commit is contained in:
Pearl Dsilva 2024-01-17 13:01:34 -05:00
parent a4f63053b6
commit d7b877aeba
7 changed files with 66 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
View File

@ -22,4 +22,5 @@ import org.apache.cloudstack.acl.ControlledEntity;
public interface KubernetesClusterHelper extends Adapter {
ControlledEntity findByUuid(String uuid);
ControlledEntity findByVmId(long vmId);
}

13
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
View File

@ -17,6 +17,7 @@
package com.cloud.kubernetes.cluster;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.utils.component.AdapterBase;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.framework.config.ConfigKey;
@ -24,18 +25,30 @@ import org.apache.cloudstack.framework.config.Configurable;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.Objects;
@Component
public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable {
@Inject
private KubernetesClusterDao kubernetesClusterDao;
@Inject
private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@Override
public ControlledEntity findByUuid(String uuid) {
return kubernetesClusterDao.findByUuid(uuid);
}
@Override
public ControlledEntity findByVmId(long vmId) {
KubernetesClusterVmMapVO clusterVmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId);
if (Objects.isNull(clusterVmMapVO)) {
return null;
}
return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId());
}
@Override
public String getConfigComponentName() {
return KubernetesClusterHelper.class.getSimpleName();

2
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
View File

@ -23,6 +23,8 @@ import java.util.List;
public interface KubernetesClusterVmMapDao extends GenericDao<KubernetesClusterVmMapVO, Long> {
public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId);
public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId);
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
int removeByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);

12
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
View File

@ -31,12 +31,17 @@ import com.cloud.utils.db.SearchCriteria;
public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClusterVmMapVO, Long> implements KubernetesClusterVmMapDao {
private final SearchBuilder<KubernetesClusterVmMapVO> clusterIdSearch;
private final SearchBuilder<KubernetesClusterVmMapVO> vmIdSearch;
public KubernetesClusterVmMapDaoImpl() {
clusterIdSearch = createSearchBuilder();
clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(), SearchCriteria.Op.IN);
clusterIdSearch.done();
vmIdSearch = createSearchBuilder();
vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
vmIdSearch.done();
}
@Override
@ -47,6 +52,13 @@ public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClus
return listBy(sc, filter);
}
@Override
public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) {
SearchCriteria<KubernetesClusterVmMapVO> sc = vmIdSearch.create();
sc.setParameters("vmId", vmId);
return findOneBy(sc);
}
@Override
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds) {
SearchCriteria<KubernetesClusterVmMapVO> sc = clusterIdSearch.create();

21
plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
View File

@ -505,10 +505,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
if (!canHandle(network, Network.Service.PortForwarding)) {
return false;
}
boolean result = true;
for (PortForwardingRule rule : rules) {
IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId());
UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId());
if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) {
if ((vm == null && (rule.getState() != FirewallRule.State.Revoke)) ||
(vm != null && networkModel.getNicInNetwork(vm.getId(), network.getId()) == null)) {
continue;
}
NsxOpObject nsxObject = getNsxOpObject(network);
@ -523,8 +525,8 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setNetworkResourceId(nsxObject.getNetworkResourceId())
.setNetworkResourceName(nsxObject.getNetworkResourceName())
.setVpcResource(nsxObject.isVpcResource())
.setVmId(vm.getId())
.setVmIp(vm.getPrivateIpAddress())
.setVmId(Objects.nonNull(vm) ? vm.getId() : 0)
.setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() : null)
.setPublicIp(publicIp.getAddress().addr())
.setPrivatePort(privatePort)
.setPublicPort(publicPort)
@ -532,12 +534,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT))
.build();
if (rule.getState() == FirewallRule.State.Add) {
return nsxService.createPortForwardRule(networkRule);
result &= nsxService.createPortForwardRule(networkRule);
} else if (rule.getState() == FirewallRule.State.Revoke) {
return nsxService.deletePortForwardRule(networkRule);
result &= nsxService.deletePortForwardRule(networkRule);
}
}
return true;
return result;
}
public Pair<VpcVO, NetworkVO> getVpcOrNetwork(Long vpcId, long networkId) {
@ -613,6 +615,7 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
@Override
public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) throws ResourceUnavailableException {
boolean result = true;
for (LoadBalancingRule loadBalancingRule : rules) {
if (loadBalancingRule.getState() == FirewallRule.State.Active) {
continue;
@ -638,12 +641,12 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns
.setAlgorithm(loadBalancingRule.getAlgorithm())
.build();
if (loadBalancingRule.getState() == FirewallRule.State.Add) {
return nsxService.createLbRule(networkRule);
result &= nsxService.createLbRule(networkRule);
} else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) {
return nsxService.deleteLbRule(networkRule);
result &= nsxService.deleteLbRule(networkRule);
}
}
return true;
return result;
}
@Override

28
server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
View File

@ -51,6 +51,9 @@ import javax.naming.ConfigurationException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
import com.cloud.network.dao.NsxProviderDao;
import com.cloud.network.element.NsxProviderVO;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@ -589,6 +592,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
@Inject
VMScheduleManager vmScheduleManager;
@Inject
NsxProviderDao nsxProviderDao;
private ScheduledExecutorService _executor = null;
private ScheduledExecutorService _vmIpFetchExecutor = null;
@ -597,6 +602,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private boolean _dailyOrHourly = false;
private int capacityReleaseInterval;
private ExecutorService _vmIpFetchThreadExecutor;
private List<KubernetesClusterHelper> kubernetesClusterHelpers;
private String _instance;
@ -610,6 +616,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
private static final int NUM_OF_2K_BLOCKS = 512;
private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES;
public List<KubernetesClusterHelper> getKubernetesClusterHelpers() {
return kubernetesClusterHelpers;
}
public void setKubernetesClusterHelpers(final List<KubernetesClusterHelper> kubernetesClusterHelpers) {
this.kubernetesClusterHelpers = kubernetesClusterHelpers;
}
@Inject
private OrchestrationService _orchSrvc;
@ -2528,11 +2542,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
}
// cleanup port forwarding rules
if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
} else {
success = false;
s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
NsxProviderVO nsx = nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId());
if (Objects.isNull(nsx) || Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) {
if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
} else {
success = false;
s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
}
}
// cleanup load balancer rules

5
server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
View File

@ -106,8 +106,9 @@
<bean id="configurationServerImpl" class="com.cloud.server.ConfigurationServerImpl" />
<bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl" />
<bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl">
<property name="kubernetesClusterHelpers" value="#{kubernetesClusterHelperRegistry.registered}" />
</bean>
<bean id="consoleProxyManagerImpl" class="com.cloud.consoleproxy.ConsoleProxyManagerImpl">
<property name="consoleProxyAllocators"