NSX: Cleanup NSX resources during k8s cluster cleanup (#8528)

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java

@@ -22,4 +22,5 @@ import org.apache.cloudstack.acl.ControlledEntity;
 public interface KubernetesClusterHelper extends Adapter {
 
     ControlledEntity findByUuid(String uuid);
+    ControlledEntity findByVmId(long vmId);
 }

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java

@@ -17,6 +17,7 @@
 package com.cloud.kubernetes.cluster;
 
 import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
 import com.cloud.utils.component.AdapterBase;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.framework.config.ConfigKey;
@@ -24,18 +25,30 @@ import org.apache.cloudstack.framework.config.Configurable;
 import org.springframework.stereotype.Component;
 
 import javax.inject.Inject;
+import java.util.Objects;
 
 @Component
 public class KubernetesClusterHelperImpl extends AdapterBase implements KubernetesClusterHelper, Configurable {
 
     @Inject
     private KubernetesClusterDao kubernetesClusterDao;
+    @Inject
+    private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
 
     @Override
     public ControlledEntity findByUuid(String uuid) {
         return kubernetesClusterDao.findByUuid(uuid);
     }
 
+    @Override
+    public ControlledEntity findByVmId(long vmId) {
+        KubernetesClusterVmMapVO clusterVmMapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId);
+        if (Objects.isNull(clusterVmMapVO)) {
+            return null;
+        }
+        return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId());
+    }
+
     @Override
     public String getConfigComponentName() {
         return KubernetesClusterHelper.class.getSimpleName();

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java

@@ -23,6 +23,8 @@ import java.util.List;
 
 public interface KubernetesClusterVmMapDao extends GenericDao<KubernetesClusterVmMapVO, Long> {
     public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId);
+
+    public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId);
     public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
 
     int removeByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java

@@ -31,12 +31,17 @@ import com.cloud.utils.db.SearchCriteria;
 public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClusterVmMapVO, Long> implements KubernetesClusterVmMapDao {
 
     private final SearchBuilder<KubernetesClusterVmMapVO> clusterIdSearch;
+    private final SearchBuilder<KubernetesClusterVmMapVO> vmIdSearch;
 
     public KubernetesClusterVmMapDaoImpl() {
         clusterIdSearch = createSearchBuilder();
         clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
         clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(), SearchCriteria.Op.IN);
         clusterIdSearch.done();
+
+        vmIdSearch = createSearchBuilder();
+        vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(), SearchCriteria.Op.EQ);
+        vmIdSearch.done();
     }
 
     @Override
@@ -47,6 +52,13 @@ public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase<KubernetesClus
         return listBy(sc, filter);
     }
 
+    @Override
+    public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) {
+        SearchCriteria<KubernetesClusterVmMapVO> sc = vmIdSearch.create();
+        sc.setParameters("vmId", vmId);
+        return findOneBy(sc);
+    }
+
     @Override
     public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds) {
         SearchCriteria<KubernetesClusterVmMapVO> sc = clusterIdSearch.create();

plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java

@@ -505,10 +505,12 @@ public class NsxElement extends AdapterBase implements  DhcpServiceProvider, Dns
         if (!canHandle(network, Network.Service.PortForwarding)) {
             return false;
         }
+        boolean result = true;
         for (PortForwardingRule rule : rules) {
             IPAddressVO publicIp = ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId());
             UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId());
-            if (vm == null || networkModel.getNicInNetwork(vm.getId(), network.getId()) == null) {
+            if ((vm == null && (rule.getState() != FirewallRule.State.Revoke)) ||
+                    (vm != null && networkModel.getNicInNetwork(vm.getId(), network.getId()) == null)) {
                 continue;
             }
             NsxOpObject nsxObject = getNsxOpObject(network);
@@ -523,8 +525,8 @@ public class NsxElement extends AdapterBase implements  DhcpServiceProvider, Dns
                     .setNetworkResourceId(nsxObject.getNetworkResourceId())
                     .setNetworkResourceName(nsxObject.getNetworkResourceName())
                     .setVpcResource(nsxObject.isVpcResource())
-                    .setVmId(vm.getId())
+                    .setVmId(Objects.nonNull(vm) ? vm.getId() : 0)
-                    .setVmIp(vm.getPrivateIpAddress())
+                    .setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() : null)
                     .setPublicIp(publicIp.getAddress().addr())
                     .setPrivatePort(privatePort)
                     .setPublicPort(publicPort)
@@ -532,12 +534,12 @@ public class NsxElement extends AdapterBase implements  DhcpServiceProvider, Dns
                     .setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT))
                     .build();
             if (rule.getState() == FirewallRule.State.Add) {
-                return nsxService.createPortForwardRule(networkRule);
+                result &= nsxService.createPortForwardRule(networkRule);
             } else if (rule.getState() == FirewallRule.State.Revoke) {
-                return nsxService.deletePortForwardRule(networkRule);
+                result &= nsxService.deletePortForwardRule(networkRule);
             }
         }
-        return true;
+        return result;
     }
 
     public Pair<VpcVO, NetworkVO> getVpcOrNetwork(Long vpcId, long networkId) {
@@ -613,6 +615,7 @@ public class NsxElement extends AdapterBase implements  DhcpServiceProvider, Dns
 
     @Override
     public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) throws ResourceUnavailableException {
+        boolean result = true;
         for (LoadBalancingRule loadBalancingRule : rules) {
             if (loadBalancingRule.getState() == FirewallRule.State.Active) {
                 continue;
@@ -638,12 +641,12 @@ public class NsxElement extends AdapterBase implements  DhcpServiceProvider, Dns
                     .setAlgorithm(loadBalancingRule.getAlgorithm())
                     .build();
             if (loadBalancingRule.getState() == FirewallRule.State.Add) {
-                return nsxService.createLbRule(networkRule);
+                result &= nsxService.createLbRule(networkRule);
             } else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) {
-                return nsxService.deleteLbRule(networkRule);
+                result &= nsxService.deleteLbRule(networkRule);
             }
         }
-        return true;
+        return result;
     }
 
     @Override

server/src/main/java/com/cloud/vm/UserVmManagerImpl.java

@@ -51,6 +51,9 @@ import javax.naming.ConfigurationException;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.ParserConfigurationException;
 
+import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
+import com.cloud.network.dao.NsxProviderDao;
+import com.cloud.network.element.NsxProviderVO;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.acl.ControlledEntity.ACLType;
 import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -589,6 +592,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
 
     @Inject
     VMScheduleManager vmScheduleManager;
+    @Inject
+    NsxProviderDao nsxProviderDao;
 
     private ScheduledExecutorService _executor = null;
     private ScheduledExecutorService _vmIpFetchExecutor = null;
@@ -597,6 +602,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private boolean _dailyOrHourly = false;
     private int capacityReleaseInterval;
     private ExecutorService _vmIpFetchThreadExecutor;
+    private List<KubernetesClusterHelper> kubernetesClusterHelpers;
 
 
     private String _instance;
@@ -610,6 +616,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     private static final int NUM_OF_2K_BLOCKS = 512;
     private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES;
 
+    public List<KubernetesClusterHelper> getKubernetesClusterHelpers() {
+        return kubernetesClusterHelpers;
+    }
+
+    public void setKubernetesClusterHelpers(final List<KubernetesClusterHelper> kubernetesClusterHelpers) {
+        this.kubernetesClusterHelpers = kubernetesClusterHelpers;
+    }
+
     @Inject
     private OrchestrationService _orchSrvc;
 
@@ -2528,11 +2542,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         }
 
         // cleanup port forwarding rules
-        if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
+        VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
-            s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
+        NsxProviderVO nsx = nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId());
-        } else {
+        if (Objects.isNull(nsx) || Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) {
-            success = false;
+            if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
-            s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+                s_logger.debug("Port forwarding rules are removed successfully as a part of vm id=" + vmId + " expunge");
+            } else {
+                success = false;
+                s_logger.warn("Fail to remove port forwarding rules as a part of vm id=" + vmId + " expunge");
+            }
         }
 
         // cleanup load balancer rules

server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml

@@ -106,8 +106,9 @@
 
     <bean id="configurationServerImpl" class="com.cloud.server.ConfigurationServerImpl" />
 
-
+    <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl">
-    <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl" />
+        <property name="kubernetesClusterHelpers" value="#{kubernetesClusterHelperRegistry.registered}" />
+    </bean>
 
     <bean id="consoleProxyManagerImpl" class="com.cloud.consoleproxy.ConsoleProxyManagerImpl">
         <property name="consoleProxyAllocators"