cks: create separate service account in project

A separate service account will be created and added in the project, if
not exist already, when a Kubernetes cluster is deployed in a project.
This account will have a role with limited API access.

Cleanup clusters on owner account cleanup, delete service account
if needed

When the owner account of k8s clusters is deleted, while its node VMs
get expunged, the cluster entry in DB remain present. This fixes the
issue by cleaning up all clusters for the account deleted.

Project k8s service account will be deleted on account cleanup or when
there is no active k8s cluster remaining

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java

@@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster;
 
 import org.apache.cloudstack.acl.ControlledEntity;
 
+import com.cloud.user.Account;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.Adapter;
 
@@ -25,4 +26,5 @@ public interface KubernetesServiceHelper extends Adapter {
 
     ControlledEntity findByUuid(String uuid);
     void checkVmCanBeDestroyed(UserVm userVm);
+    void cleanupForAccount(Account account);
 }

engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java

@@ -16,6 +16,9 @@
 // under the License.
 package com.cloud.user.dao;
 
+import java.util.Date;
+import java.util.List;
+
 import com.cloud.user.Account;
 import com.cloud.user.AccountVO;
 import com.cloud.user.User;
@@ -23,9 +26,6 @@ import com.cloud.utils.Pair;
 import com.cloud.utils.db.Filter;
 import com.cloud.utils.db.GenericDao;
 
-import java.util.Date;
-import java.util.List;
-
 public interface AccountDao extends GenericDao<AccountVO, Long> {
     Pair<User, Account> findUserAccountByApiKey(String apiKey);
 
@@ -33,6 +33,8 @@ public interface AccountDao extends GenericDao<AccountVO, Long> {
 
     Pair<List<AccountVO>, Integer> findAccountsLike(String accountName, Filter filter);
 
+    List<AccountVO> findAccountsByName(String accountName);
+
     List<AccountVO> findActiveAccounts(Long maxAccountId, Filter filter);
 
     List<AccountVO> findRecentlyDeletedAccounts(Long maxAccountId, Date earliestRemovedDate, Filter filter);

engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java

@@ -16,6 +16,15 @@
 // under the License.
 package com.cloud.user.dao;
 
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
+
 import com.cloud.user.Account;
 import com.cloud.user.Account.State;
 import com.cloud.user.AccountVO;
@@ -30,15 +39,7 @@ import com.cloud.utils.db.SearchBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.SearchCriteria.Func;
 import com.cloud.utils.db.SearchCriteria.Op;
-import org.apache.commons.lang3.StringUtils;
 import com.cloud.utils.db.TransactionLegacy;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.util.Date;
-import java.util.List;
 
 @Component
 public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements AccountDao {
@@ -180,6 +181,16 @@ public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements A
         return searchAndCount(sc, filter);
     }
 
+    @Override
+    public List<AccountVO> findAccountsByName(String accountName) {
+        SearchBuilder<AccountVO> sb = createSearchBuilder();
+        sb.and("accountName", sb.entity().getAccountName(), SearchCriteria.Op.EQ);
+        sb.done();
+        SearchCriteria<AccountVO> sc = sb.create();
+        sc.setParameters("accountName", accountName);
+        return search(sc, null);
+    }
+
     @Override
     public Account findEnabledAccount(String accountName, Long domainId) {
         SearchCriteria<AccountVO> sc = AllFieldsSearch.create("accountName", accountName);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java

@@ -41,6 +41,11 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
 import org.apache.cloudstack.acl.ControlledEntity;
+import org.apache.cloudstack.acl.Role;
+import org.apache.cloudstack.acl.RolePermissionEntity;
+import org.apache.cloudstack.acl.RoleService;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.acl.Rule;
 import org.apache.cloudstack.acl.SecurityChecker;
 import org.apache.cloudstack.annotation.AnnotationService;
 import org.apache.cloudstack.annotation.dao.AnnotationDao;
@@ -49,6 +54,14 @@ import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.ApiConstants.VMDetails;
 import org.apache.cloudstack.api.BaseCmd;
 import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd;
+import org.apache.cloudstack.api.command.user.address.DisassociateIPAddrCmd;
+import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd;
+import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.firewall.DeleteFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.firewall.ListFirewallRulesCmd;
+import org.apache.cloudstack.api.command.user.firewall.UpdateFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@@ -59,6 +72,18 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetes
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd;
+import org.apache.cloudstack.api.command.user.loadbalancer.AssignToLoadBalancerRuleCmd;
+import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd;
+import org.apache.cloudstack.api.command.user.loadbalancer.DeleteLoadBalancerRuleCmd;
+import org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRuleInstancesCmd;
+import org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRulesCmd;
+import org.apache.cloudstack.api.command.user.loadbalancer.RemoveFromLoadBalancerRuleCmd;
+import org.apache.cloudstack.api.command.user.loadbalancer.UpdateLoadBalancerRuleCmd;
+import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
+import org.apache.cloudstack.api.command.user.network.ListNetworksCmd;
+import org.apache.cloudstack.api.command.user.vm.ListVMsCmd;
 import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse;
 import org.apache.cloudstack.api.response.KubernetesClusterResponse;
 import org.apache.cloudstack.api.response.ListResponse;
@@ -145,6 +170,8 @@ import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
 import com.cloud.org.Cluster;
 import com.cloud.org.Grouping;
 import com.cloud.projects.Project;
+import com.cloud.projects.ProjectAccount;
+import com.cloud.projects.ProjectManager;
 import com.cloud.resource.ResourceManager;
 import com.cloud.service.ServiceOfferingVO;
 import com.cloud.service.dao.ServiceOfferingDao;
@@ -153,15 +180,18 @@ import com.cloud.storage.dao.VMTemplateDao;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
 import com.cloud.user.AccountService;
+import com.cloud.user.AccountVO;
 import com.cloud.user.SSHKeyPairVO;
 import com.cloud.user.User;
 import com.cloud.user.UserAccount;
 import com.cloud.user.UserVO;
+import com.cloud.user.dao.AccountDao;
 import com.cloud.user.dao.SSHKeyPairDao;
 import com.cloud.user.dao.UserDao;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
+import com.cloud.utils.UuidUtils;
 import com.cloud.utils.component.ComponentContext;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -186,6 +216,33 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
 
     private static final Logger LOGGER = Logger.getLogger(KubernetesClusterManagerImpl.class);
     private static final String DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNetworkOfferingforKubernetesService";
+    private static final List<Class<?>> PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList(
+            QueryAsyncJobResultCmd.class,
+            ListVMsCmd.class,
+            ListNetworksCmd.class,
+            ListPublicIpAddressesCmd.class,
+            AssociateIPAddrCmd.class,
+            DisassociateIPAddrCmd.class,
+            ListLoadBalancerRulesCmd.class,
+            CreateLoadBalancerRuleCmd.class,
+            UpdateLoadBalancerRuleCmd.class,
+            DeleteLoadBalancerRuleCmd.class,
+            AssignToLoadBalancerRuleCmd.class,
+            RemoveFromLoadBalancerRuleCmd.class,
+            ListLoadBalancerRuleInstancesCmd.class,
+            ListFirewallRulesCmd.class,
+            CreateFirewallRuleCmd.class,
+            UpdateFirewallRuleCmd.class,
+            DeleteFirewallRuleCmd.class,
+            ListNetworkACLsCmd.class,
+            CreateNetworkACLCmd.class,
+            DeleteNetworkACLCmd.class,
+            ListKubernetesClustersCmd.class,
+            ScaleKubernetesClusterCmd.class
+    );
+    private static final String PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME = "Kubernetes";
+    private static final String PROJECT_KUBERNETES_ACCOUNT_LAST_NAME = "Service User";
+
 
     protected StateMachine2<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> _stateMachine = KubernetesCluster.State.getStateMachine();
 
@@ -258,9 +315,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
     public SecurityGroupService securityGroupService;
     @Inject
     public NetworkHelper networkHelper;
-
     @Inject
     private UserVmService userVmService;
+    @Inject
+    public AccountDao accountDao;
+    @Inject
+    public ProjectManager projectManager;
+    @Inject
+    RoleService roleService;
 
     private void logMessage(final Level logLevel, final String message, final Exception e) {
         if (logLevel == Level.WARN) {
@@ -1371,23 +1433,31 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         }
     }
 
-    private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster) {
+    protected String[] createUserApiKeyAndSecretKey(long userId) {
-        Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
+        CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
-        if (owner == null || owner.getType() == Account.Type.PROJECT) {
+        try {
-            owner = CallContext.current().getCallingAccount();
+            return accountService.createApiKeyAndSecretKey(userId);
+        } finally {
+            CallContext.unregister();
+        }
+    }
+
+    protected String[] getServiceUserKeys(Account owner) {
+        String username = owner.getAccountName();
+        if (!username.startsWith(KUBEADMIN_ACCOUNT_NAME + "-")) {
+            username += "-" + KUBEADMIN_ACCOUNT_NAME;
         }
-        String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME;
         UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId());
-        String[] keys = null;
+        String[] keys;
         if (kubeadmin == null) {
             User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(),
-                KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
+                    KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
-            keys = accountService.createApiKeyAndSecretKey(kube.getId());
+            keys = createUserApiKeyAndSecretKey(kube.getId());
         } else {
             String apiKey = kubeadmin.getApiKey();
             String secretKey = kubeadmin.getSecretKey();
             if (StringUtils.isAnyEmpty(apiKey, secretKey)) {
-                keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId());
+                keys = createUserApiKeyAndSecretKey(kubeadmin.getId());
             } else {
                 keys = new String[]{apiKey, secretKey};
             }
@@ -1395,6 +1465,76 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         return keys;
     }
 
+    protected Role createProjectKubernetesAccountRole() {
+        Role role = roleService.createRole(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, RoleType.User,
+                PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, false);
+        for (Class<?> allowedApi : PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS) {
+            final String apiName = BaseCmd.getCommandNameByClass(allowedApi);
+            roleService.createRolePermission(role, new Rule(apiName), RolePermissionEntity.Permission.ALLOW,
+                    String.format("Allow %s", apiName));
+        }
+        roleService.createRolePermission(role, new Rule("*"), RolePermissionEntity.Permission.DENY,
+                "Deny all");
+        LOGGER.debug(String.format("Created default role for Kubernetes service account in projects: %s", role));
+        return role;
+    }
+
+    public Role getProjectKubernetesAccountRole() {
+        List<Role> roles = roleService.findRolesByName(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME);
+        if (CollectionUtils.isNotEmpty(roles)) {
+            Role role = roles.get(0);
+            LOGGER.debug(String.format("Found default role for Kubernetes service account in projects: %s", role));
+            return role;
+        }
+        return createProjectKubernetesAccountRole();
+    }
+
+    protected Account createProjectKubernetesAccount(final Project project, final String accountName) {
+        CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+        try {
+            Role role = getProjectKubernetesAccountRole();
+            UserAccount userAccount = accountService.createUserAccount(accountName,
+                    UuidUtils.first(UUID.randomUUID().toString()), PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME,
+                    PROJECT_KUBERNETES_ACCOUNT_LAST_NAME, null, null, accountName, Account.Type.NORMAL, role.getId(),
+                    project.getDomainId(), null, null, null, null, User.Source.NATIVE);
+            projectManager.assignAccountToProject(project, userAccount.getAccountId(), ProjectAccount.Role.Regular,
+                    userAccount.getId(), null);
+            Account account = accountService.getAccount(userAccount.getAccountId());
+            LOGGER.debug(String.format("Created Kubernetes service account in project %s: %s", project, account));
+            return account;
+        } finally {
+            CallContext.unregister();
+        }
+    }
+
+    protected Account getProjectKubernetesAccount(final Account callerAccount, final boolean create) {
+        Project project = ApiDBUtils.findProjectByProjectAccountId(callerAccount.getId());
+        final String accountName = String.format("%s-%s", KUBEADMIN_ACCOUNT_NAME, UuidUtils.first(project.getUuid()));
+        List<AccountVO> accounts = accountDao.findAccountsByName(accountName);
+        for (AccountVO account : accounts) {
+            if (projectManager.canAccessProjectAccount(account, project.getProjectAccountId())) {
+                LOGGER.debug(String.format("Created Kubernetes service account in project %s: %s", project, account));
+                return account;
+            }
+        }
+        return create ? createProjectKubernetesAccount(project, accountName) : null;
+    }
+
+    protected Account getProjectKubernetesAccount(final Account callerAccount) {
+        return getProjectKubernetesAccount(callerAccount, true);
+    }
+
+    private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster) {
+        Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
+        if (owner == null) {
+            owner = CallContext.current().getCallingAccount();
+        }
+        if (owner.getType() == Account.Type.PROJECT) {
+            owner = getProjectKubernetesAccount(owner);
+        }
+        return getServiceUserKeys(owner);
+    }
+
     @Override
     @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_STOP,
             eventDescription = "stopping Kubernetes cluster", async = true)
@@ -1439,15 +1579,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
             logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
         }
         Long kubernetesClusterId = cmd.getId();
-        KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId);
+        final KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId);
         if (cluster == null) {
             throw new InvalidParameterValueException("Invalid cluster id specified");
         }
         accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, cluster);
         if (cluster.getClusterType() == KubernetesCluster.ClusterType.CloudManaged) {
-            KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(cluster, this);
+            return destroyKubernetesCluster(cluster);
-            destroyWorker = ComponentContext.inject(destroyWorker);
-            return destroyWorker.destroy();
         } else {
             boolean cleanup = cmd.getCleanup();
             boolean expunge = cmd.getExpunge();
@@ -1470,13 +1608,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
                     }
                 }
             }
-            return Transaction.execute(new TransactionCallback<Boolean>() {
+            return Transaction.execute((TransactionCallback<Boolean>) status -> {
-                @Override
+                kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
-                public Boolean doInTransaction(TransactionStatus status) {
+                kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
-                    kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
+                if (kubernetesClusterDao.remove(kubernetesClusterId)) {
-                    kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
+                    deleteProjectKubernetesAccountIfNeeded(cluster);
-                    return kubernetesClusterDao.remove(kubernetesClusterId);
+                    return true;
                 }
+                return false;
             });
         }
     }
@@ -1730,6 +1869,66 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
         return responseList;
     }
 
+    protected void deleteProjectKubernetesAccount(Account projectAccount) {
+        CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+        try {
+            Account serviceAccount = getProjectKubernetesAccount(projectAccount, false);
+            if (serviceAccount != null) {
+                accountManager.deleteAccount(accountDao.findById(serviceAccount.getId()), User.UID_SYSTEM,
+                        accountService.getSystemAccount());
+            }
+        } finally {
+            CallContext.unregister();
+        }
+    }
+
+    protected void deleteProjectKubernetesAccountIfNeeded(final KubernetesCluster kubernetesCluster) {
+        Account owner = accountService.getAccount(kubernetesCluster.getAccountId());
+        if (owner == null) {
+            return;
+        }
+        if (Account.Type.PROJECT.equals(owner.getType()) &&
+                kubernetesClusterDao.countNotForGCByAccount(owner.getAccountId()) == 0) {
+            deleteProjectKubernetesAccount(owner);
+        }
+    }
+
+    protected boolean destroyKubernetesCluster(KubernetesCluster kubernetesCluster, boolean deleteProjectAccount) {
+        KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster,
+                KubernetesClusterManagerImpl.this);
+        destroyWorker = ComponentContext.inject(destroyWorker);
+        boolean result = destroyWorker.destroy();
+        if (deleteProjectAccount) {
+            deleteProjectKubernetesAccountIfNeeded(kubernetesCluster);
+        }
+        return result;
+    }
+
+    protected boolean destroyKubernetesCluster(KubernetesCluster kubernetesCluster) {
+        return destroyKubernetesCluster(kubernetesCluster, true);
+    }
+
+    @Override
+    public void cleanupForAccount(Account account) {
+        List<KubernetesClusterVO> clusters = kubernetesClusterDao.listForCleanupByAccount(account.getId());
+        if (CollectionUtils.isEmpty(clusters)) {
+            return;
+        }
+        LOGGER.debug(String.format("Cleaning up %d Kubernetes cluster for %s", clusters.size(), account));
+        for (KubernetesClusterVO cluster : clusters) {
+            try {
+                destroyKubernetesCluster(cluster, false);
+            } catch (CloudRuntimeException e) {
+                LOGGER.warn(String.format("Failed to destroy Kubernetes cluster: %s during cleanup for %s",
+                        cluster.getName(), account), e);
+            }
+        }
+        if (!Account.Type.PROJECT.equals(account.getType())) {
+            return;
+        }
+        deleteProjectKubernetesAccount(account);
+    }
+
     @Override
     public List<Class<?>> getCommands() {
         List<Class<?>> cmdList = new ArrayList<Class<?>>();
@@ -1781,9 +1980,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
                         LOGGER.info(String.format("Running Kubernetes cluster garbage collector on Kubernetes cluster : %s", kubernetesCluster.getName()));
                     }
                     try {
-                        KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
+                        if (destroyKubernetesCluster(kubernetesCluster)) {
-                        destroyWorker = ComponentContext.inject(destroyWorker);
-                        if (destroyWorker.destroy()) {
                             if (LOGGER.isInfoEnabled()) {
                                 LOGGER.info(String.format("Garbage collection complete for Kubernetes cluster : %s", kubernetesCluster.getName()));
                             }
@@ -1909,9 +2106,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
                             LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster : %s for state: %s", kubernetesCluster.getName(), KubernetesCluster.State.Destroying.toString()));
                         }
                         try {
-                            KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
+                            destroyKubernetesCluster(kubernetesCluster);
-                            destroyWorker = ComponentContext.inject(destroyWorker);
-                            destroyWorker.destroy();
                         } catch (Exception e) {
                             LOGGER.warn(String.format("Failed to run Kubernetes cluster Destroying state scanner on Kubernetes cluster : %s status scanner", kubernetesCluster.getName()), e);
                         }
@@ -1925,7 +2120,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
     }
 
     // checks if Kubernetes cluster is in desired state
-    boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) {
+    private boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) {
         List<KubernetesClusterVmMapVO> clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
 
         // check cluster is running at desired capacity include control nodes as well
@@ -1985,6 +2180,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
             LOGGER.trace("Added service for the network offering: " + offService);
         }
 
+        getProjectKubernetesAccountRole();
+
         _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS);
         _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS);
 

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java

@@ -16,6 +16,8 @@
 // under the License.
 package com.cloud.kubernetes.cluster;
 
+import java.util.List;
+
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
 import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@@ -33,16 +35,16 @@ import org.apache.cloudstack.api.response.RemoveVirtualMachinesFromKubernetesClu
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 
+import com.cloud.user.Account;
 import com.cloud.utils.component.PluggableService;
 import com.cloud.utils.exception.CloudRuntimeException;
 
-import java.util.List;
-
 public interface KubernetesClusterService extends PluggableService, Configurable {
     static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
     static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
     static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
     static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";
+    String PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME = "Project Kubernetes Service Role";
 
     static final ConfigKey<Boolean> KubernetesServiceEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
             "cloud.kubernetes.service.enabled",
@@ -122,4 +124,6 @@ public interface KubernetesClusterService extends PluggableService, Configurable
     boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd);
 
     List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd);
+
+    void cleanupForAccount(Account account);
 }

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java

@@ -34,6 +34,7 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
 import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
 import com.cloud.kubernetes.version.KubernetesSupportedVersion;
 import com.cloud.kubernetes.version.KubernetesVersionEventTypes;
+import com.cloud.user.Account;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.AdapterBase;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -47,6 +48,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet
     private KubernetesClusterDao kubernetesClusterDao;
     @Inject
     private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
+    @Inject
+    KubernetesClusterService kubernetesClusterService;
 
     protected void setEventTypeEntityDetails(Class<?> eventTypeDefinedClass, Class<?> entityClass) {
         Field[] declaredFields = eventTypeDefinedClass.getDeclaredFields();
@@ -94,6 +97,11 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet
         throw new CloudRuntimeException(msg);
     }
 
+    @Override
+    public void cleanupForAccount(Account account) {
+        kubernetesClusterService.cleanupForAccount(account);
+    }
+
     @Override
     public String getConfigComponentName() {
         return KubernetesServiceHelper.class.getSimpleName();

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java

@@ -25,8 +25,8 @@ import com.cloud.utils.fsm.StateDao;
 
 public interface KubernetesClusterDao extends GenericDao<KubernetesClusterVO, Long>,
         StateDao<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> {
-
+    List<KubernetesClusterVO> listForCleanupByAccount(long accountId);
-    List<KubernetesClusterVO> listByAccount(long accountId);
+    int countNotForGCByAccount(long accountId);
     List<KubernetesClusterVO> findKubernetesClustersToGarbageCollect();
     List<KubernetesClusterVO> findManagedKubernetesClustersInState(KubernetesCluster.State state);
     List<KubernetesClusterVO> listByNetworkId(long networkId);

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java

@@ -30,16 +30,25 @@ import com.cloud.utils.db.TransactionLegacy;
 @Component
 public class KubernetesClusterDaoImpl extends GenericDaoBase<KubernetesClusterVO, Long> implements KubernetesClusterDao {
 
-    private final SearchBuilder<KubernetesClusterVO> AccountIdSearch;
+    private final SearchBuilder<KubernetesClusterVO> CleanupAccountIdSearch;
+    private final SearchBuilder<KubernetesClusterVO> NotForGCByAccountIDCount;
     private final SearchBuilder<KubernetesClusterVO> GarbageCollectedSearch;
     private final SearchBuilder<KubernetesClusterVO> ManagedStateSearch;
     private final SearchBuilder<KubernetesClusterVO> SameNetworkSearch;
     private final SearchBuilder<KubernetesClusterVO> KubernetesVersionSearch;
 
     public KubernetesClusterDaoImpl() {
-        AccountIdSearch = createSearchBuilder();
+        CleanupAccountIdSearch = createSearchBuilder();
-        AccountIdSearch.and("account", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+        CleanupAccountIdSearch.and("account", CleanupAccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
-        AccountIdSearch.done();
+        CleanupAccountIdSearch.and("cluster_type", CleanupAccountIdSearch.entity().getClusterType(), SearchCriteria.Op.EQ);
+        CleanupAccountIdSearch.done();
+
+        NotForGCByAccountIDCount = createSearchBuilder();
+        NotForGCByAccountIDCount.and("gc", NotForGCByAccountIDCount.entity().isCheckForGc(), SearchCriteria.Op.EQ);
+        NotForGCByAccountIDCount.and("account", NotForGCByAccountIDCount.entity().getAccountId(), SearchCriteria.Op.EQ);
+        NotForGCByAccountIDCount.and("cluster_type", NotForGCByAccountIDCount.entity().getClusterType(), SearchCriteria.Op.EQ);
+        NotForGCByAccountIDCount.select(null, SearchCriteria.Func.COUNT, null);
+        NotForGCByAccountIDCount.done();
 
         GarbageCollectedSearch = createSearchBuilder();
         GarbageCollectedSearch.and("gc", GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ);
@@ -62,10 +71,20 @@ public class KubernetesClusterDaoImpl extends GenericDaoBase<KubernetesClusterVO
     }
 
     @Override
-    public List<KubernetesClusterVO> listByAccount(long accountId) {
+    public List<KubernetesClusterVO> listForCleanupByAccount(long accountId) {
-        SearchCriteria<KubernetesClusterVO> sc = AccountIdSearch.create();
+        SearchCriteria<KubernetesClusterVO> sc = CleanupAccountIdSearch.create();
+        sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged);
         sc.setParameters("account", accountId);
-        return listBy(sc, null);
+        return listBy(sc);
+    }
+
+    @Override
+    public int countNotForGCByAccount(long accountId) {
+        SearchCriteria<KubernetesClusterVO> sc = NotForGCByAccountIDCount.create();
+        sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged);
+        sc.setParameters("account", accountId);
+        sc.setParameters("gc", false);
+        return getCount(sc);
     }
 
     @Override

server/src/main/java/com/cloud/user/AccountManagerImpl.java

@@ -85,6 +85,7 @@ import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
 
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.auth.SetupUserTwoFactorAuthenticationCmd;
@@ -117,6 +118,7 @@ import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.PermissionDeniedException;
 import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.kubernetes.cluster.KubernetesServiceHelper;
 import com.cloud.network.IpAddress;
 import com.cloud.network.IpAddressManager;
 import com.cloud.network.Network;
@@ -171,6 +173,7 @@ import com.cloud.utils.ConstantTimeComparator;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
+import com.cloud.utils.component.ComponentContext;
 import com.cloud.utils.component.Manager;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.component.PluggableService;
@@ -859,6 +862,16 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
         return cleanupAccount(account, callerUserId, caller);
     }
 
+    protected void cleanupPluginsResourcesIfNeeded(Account account) {
+        try {
+            KubernetesServiceHelper kubernetesServiceHelper =
+                    ComponentContext.getDelegateComponentOfType(KubernetesServiceHelper.class);
+            kubernetesServiceHelper.cleanupForAccount(account);
+        } catch (NoSuchBeanDefinitionException ignored) {
+            s_logger.debug("No KubernetesServiceHelper bean found");
+        }
+    }
+
     protected boolean cleanupAccount(AccountVO account, long callerUserId, Account caller) {
         long accountId = account.getId();
         boolean accountCleanupNeeded = false;
@@ -940,6 +953,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
                 }
             }
 
+            cleanupPluginsResourcesIfNeeded(account);
+
             // Destroy the account's VMs
             List<UserVmVO> vms = _userVmDao.listByAccountId(accountId);
             if (s_logger.isDebugEnabled()) {