diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java index a13c1b3a6a8..84e898528b5 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java @@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster; import org.apache.cloudstack.acl.ControlledEntity; +import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.utils.component.Adapter; @@ -26,4 +27,5 @@ public interface KubernetesServiceHelper extends Adapter { ControlledEntity findByUuid(String uuid); ControlledEntity findByVmId(long vmId); void checkVmCanBeDestroyed(UserVm userVm); + void cleanupForAccount(Account account); } diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java index 17b07496731..dae5f3a3467 100644 --- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java +++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.user.dao; +import java.util.Date; +import java.util.List; + import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.User; @@ -23,9 +26,6 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; -import java.util.Date; -import java.util.List; - public interface AccountDao extends GenericDao { Pair findUserAccountByApiKey(String apiKey); @@ -33,6 +33,8 @@ public interface AccountDao extends GenericDao { Pair, Integer> findAccountsLike(String accountName, Filter filter); + List findAccountsByName(String accountName); + List findActiveAccounts(Long maxAccountId, Filter filter); List findRecentlyDeletedAccounts(Long maxAccountId, Date earliestRemovedDate, Filter filter); diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java index 2654b22374f..f5f95d5da1f 100644 --- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java @@ -16,6 +16,14 @@ // under the License. package com.cloud.user.dao; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + import com.cloud.user.Account; import com.cloud.user.Account.State; import com.cloud.user.AccountVO; @@ -30,14 +38,7 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; import com.cloud.utils.db.SearchCriteria.Op; -import org.apache.commons.lang3.StringUtils; import com.cloud.utils.db.TransactionLegacy; -import org.springframework.stereotype.Component; - -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.util.Date; -import java.util.List; @Component public class AccountDaoImpl extends GenericDaoBase implements AccountDao { @@ -190,6 +191,16 @@ public class AccountDaoImpl extends GenericDaoBase implements A return searchAndCount(sc, filter); } + @Override + public List findAccountsByName(String accountName) { + SearchBuilder sb = createSearchBuilder(); + sb.and("accountName", sb.entity().getAccountName(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("accountName", accountName); + return search(sc, null); + } + @Override public Account findEnabledAccount(String accountName, Long domainId) { SearchCriteria sc = AllFieldsSearch.create("accountName", accountName); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 131d7b22606..5e86bb6daf0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -44,6 +44,11 @@ import javax.naming.ConfigurationException; import com.cloud.uservm.UserVm; import com.cloud.vm.UserVmService; import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.Role; +import org.apache.cloudstack.acl.RolePermissionEntity; +import org.apache.cloudstack.acl.RoleService; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.Rule; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -52,6 +57,14 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.VMDetails; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd; +import org.apache.cloudstack.api.command.user.address.DisassociateIPAddrCmd; +import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.firewall.DeleteFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.firewall.ListFirewallRulesCmd; +import org.apache.cloudstack.api.command.user.firewall.UpdateFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; @@ -62,6 +75,18 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetes import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.loadbalancer.AssignToLoadBalancerRuleCmd; +import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; +import org.apache.cloudstack.api.command.user.loadbalancer.DeleteLoadBalancerRuleCmd; +import org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRuleInstancesCmd; +import org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRulesCmd; +import org.apache.cloudstack.api.command.user.loadbalancer.RemoveFromLoadBalancerRuleCmd; +import org.apache.cloudstack.api.command.user.loadbalancer.UpdateLoadBalancerRuleCmd; +import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; +import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd; +import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd; +import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; +import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -147,6 +172,8 @@ import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; import com.cloud.org.Cluster; import com.cloud.org.Grouping; import com.cloud.projects.Project; +import com.cloud.projects.ProjectAccount; +import com.cloud.projects.ProjectManager; import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -155,14 +182,17 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; +import com.cloud.user.AccountVO; import com.cloud.user.SSHKeyPairVO; import com.cloud.user.User; import com.cloud.user.UserAccount; import com.cloud.user.UserVO; +import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; +import com.cloud.utils.UuidUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -186,12 +216,40 @@ import org.apache.logging.log4j.Level; public class KubernetesClusterManagerImpl extends ManagerBase implements KubernetesClusterService { private static final String DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNetworkOfferingforKubernetesService"; + private static final String DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network Offering used for CloudStack Kubernetes service"; private static final String DEFAULT_NSX_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNSXNetworkOfferingforKubernetesService"; private static final String DEFAULT_NSX_VPC_TIER_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNSXVPCNetworkOfferingforKubernetesService"; private static final String DEFAULT_NSX_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network Offering for NSX CloudStack Kubernetes Service"; private static final String DEFAULT_NSX_VPC_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network Offering for NSX CloudStack Kubernetes service on VPC"; + private static final List> PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList( + QueryAsyncJobResultCmd.class, + ListVMsCmd.class, + ListNetworksCmd.class, + ListPublicIpAddressesCmd.class, + AssociateIPAddrCmd.class, + DisassociateIPAddrCmd.class, + ListLoadBalancerRulesCmd.class, + CreateLoadBalancerRuleCmd.class, + UpdateLoadBalancerRuleCmd.class, + DeleteLoadBalancerRuleCmd.class, + AssignToLoadBalancerRuleCmd.class, + RemoveFromLoadBalancerRuleCmd.class, + ListLoadBalancerRuleInstancesCmd.class, + ListFirewallRulesCmd.class, + CreateFirewallRuleCmd.class, + UpdateFirewallRuleCmd.class, + DeleteFirewallRuleCmd.class, + ListNetworkACLsCmd.class, + CreateNetworkACLCmd.class, + DeleteNetworkACLCmd.class, + ListKubernetesClustersCmd.class, + ScaleKubernetesClusterCmd.class + ); + private static final String PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME = "Kubernetes"; + private static final String PROJECT_KUBERNETES_ACCOUNT_LAST_NAME = "Service User"; + protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); ScheduledExecutorService _gcExecutor; @@ -263,11 +321,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne public SecurityGroupService securityGroupService; @Inject public NetworkHelper networkHelper; - @Inject private UserVmService userVmService; @Inject RoutedIpv4Manager routedIpv4Manager; + @Inject + public AccountDao accountDao; + @Inject + public ProjectManager projectManager; + @Inject + RoleService roleService; private void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.WARN) { @@ -1380,23 +1443,31 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } - private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster) { - Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId()); - if (owner == null || owner.getType() == Account.Type.PROJECT) { - owner = CallContext.current().getCallingAccount(); + protected String[] createUserApiKeyAndSecretKey(long userId) { + CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM); + try { + return accountService.createApiKeyAndSecretKey(userId); + } finally { + CallContext.unregister(); + } + } + + protected String[] getServiceUserKeys(Account owner) { + String username = owner.getAccountName(); + if (!username.startsWith(KUBEADMIN_ACCOUNT_NAME + "-")) { + username += "-" + KUBEADMIN_ACCOUNT_NAME; } - String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME; UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId()); - String[] keys = null; + String[] keys; if (kubeadmin == null) { User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(), - KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN)); - keys = accountService.createApiKeyAndSecretKey(kube.getId()); + KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN)); + keys = createUserApiKeyAndSecretKey(kube.getId()); } else { String apiKey = kubeadmin.getApiKey(); String secretKey = kubeadmin.getSecretKey(); if (StringUtils.isAnyEmpty(apiKey, secretKey)) { - keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId()); + keys = createUserApiKeyAndSecretKey(kubeadmin.getId()); } else { keys = new String[]{apiKey, secretKey}; } @@ -1404,6 +1475,76 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return keys; } + protected Role createProjectKubernetesAccountRole() { + Role role = roleService.createRole(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, RoleType.User, + PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, false); + for (Class allowedApi : PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS) { + final String apiName = BaseCmd.getCommandNameByClass(allowedApi); + roleService.createRolePermission(role, new Rule(apiName), RolePermissionEntity.Permission.ALLOW, + String.format("Allow %s", apiName)); + } + roleService.createRolePermission(role, new Rule("*"), RolePermissionEntity.Permission.DENY, + "Deny all"); + logger.debug(String.format("Created default role for Kubernetes service account in projects: %s", role)); + return role; + } + + public Role getProjectKubernetesAccountRole() { + List roles = roleService.findRolesByName(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME); + if (CollectionUtils.isNotEmpty(roles)) { + Role role = roles.get(0); + logger.debug(String.format("Found default role for Kubernetes service account in projects: %s", role)); + return role; + } + return createProjectKubernetesAccountRole(); + } + + protected Account createProjectKubernetesAccount(final Project project, final String accountName) { + CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM); + try { + Role role = getProjectKubernetesAccountRole(); + UserAccount userAccount = accountService.createUserAccount(accountName, + UuidUtils.first(UUID.randomUUID().toString()), PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME, + PROJECT_KUBERNETES_ACCOUNT_LAST_NAME, null, null, accountName, Account.Type.NORMAL, role.getId(), + project.getDomainId(), null, null, null, null, User.Source.NATIVE); + projectManager.assignAccountToProject(project, userAccount.getAccountId(), ProjectAccount.Role.Regular, + userAccount.getId(), null); + Account account = accountService.getAccount(userAccount.getAccountId()); + logger.debug(String.format("Created Kubernetes service account in project %s: %s", project, account)); + return account; + } finally { + CallContext.unregister(); + } + } + + protected Account getProjectKubernetesAccount(final Account callerAccount, final boolean create) { + Project project = ApiDBUtils.findProjectByProjectAccountId(callerAccount.getId()); + final String accountName = String.format("%s-%s", KUBEADMIN_ACCOUNT_NAME, UuidUtils.first(project.getUuid())); + List accounts = accountDao.findAccountsByName(accountName); + for (AccountVO account : accounts) { + if (projectManager.canAccessProjectAccount(account, project.getProjectAccountId())) { + logger.debug(String.format("Created Kubernetes service account in project %s: %s", project, account)); + return account; + } + } + return create ? createProjectKubernetesAccount(project, accountName) : null; + } + + protected Account getProjectKubernetesAccount(final Account callerAccount) { + return getProjectKubernetesAccount(callerAccount, true); + } + + private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster) { + Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId()); + if (owner == null) { + owner = CallContext.current().getCallingAccount(); + } + if (owner.getType() == Account.Type.PROJECT) { + owner = getProjectKubernetesAccount(owner); + } + return getServiceUserKeys(owner); + } + @Override @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_STOP, eventDescription = "stopping Kubernetes cluster", async = true) @@ -1448,15 +1589,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } Long kubernetesClusterId = cmd.getId(); - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); + final KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); if (cluster == null) { throw new InvalidParameterValueException("Invalid cluster id specified"); } accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, cluster); if (cluster.getClusterType() == KubernetesCluster.ClusterType.CloudManaged) { - KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(cluster, this); - destroyWorker = ComponentContext.inject(destroyWorker); - return destroyWorker.destroy(); + return destroyKubernetesCluster(cluster); } else { boolean cleanup = cmd.getCleanup(); boolean expunge = cmd.getExpunge(); @@ -1483,13 +1622,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } } - return Transaction.execute(new TransactionCallback() { - @Override - public Boolean doInTransaction(TransactionStatus status) { - kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId); - kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId); - return kubernetesClusterDao.remove(kubernetesClusterId); + return Transaction.execute((TransactionCallback) status -> { + kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId); + kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId); + if (kubernetesClusterDao.remove(kubernetesClusterId)) { + deleteProjectKubernetesAccountIfNeeded(cluster); + return true; } + return false; }); } } @@ -1752,6 +1892,66 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return responseList; } + protected void deleteProjectKubernetesAccount(Account projectAccount) { + CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM); + try { + Account serviceAccount = getProjectKubernetesAccount(projectAccount, false); + if (serviceAccount != null) { + accountManager.deleteAccount(accountDao.findById(serviceAccount.getId()), User.UID_SYSTEM, + accountService.getSystemAccount()); + } + } finally { + CallContext.unregister(); + } + } + + protected void deleteProjectKubernetesAccountIfNeeded(final KubernetesCluster kubernetesCluster) { + Account owner = accountService.getAccount(kubernetesCluster.getAccountId()); + if (owner == null) { + return; + } + if (Account.Type.PROJECT.equals(owner.getType()) && + kubernetesClusterDao.countNotForGCByAccount(owner.getAccountId()) == 0) { + deleteProjectKubernetesAccount(owner); + } + } + + protected boolean destroyKubernetesCluster(KubernetesCluster kubernetesCluster, boolean deleteProjectAccount) { + KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, + KubernetesClusterManagerImpl.this); + destroyWorker = ComponentContext.inject(destroyWorker); + boolean result = destroyWorker.destroy(); + if (deleteProjectAccount) { + deleteProjectKubernetesAccountIfNeeded(kubernetesCluster); + } + return result; + } + + protected boolean destroyKubernetesCluster(KubernetesCluster kubernetesCluster) { + return destroyKubernetesCluster(kubernetesCluster, true); + } + + @Override + public void cleanupForAccount(Account account) { + List clusters = kubernetesClusterDao.listForCleanupByAccount(account.getId()); + if (CollectionUtils.isEmpty(clusters)) { + return; + } + logger.debug(String.format("Cleaning up %d Kubernetes cluster for %s", clusters.size(), account)); + for (KubernetesClusterVO cluster : clusters) { + try { + destroyKubernetesCluster(cluster, false); + } catch (CloudRuntimeException e) { + logger.warn(String.format("Failed to destroy Kubernetes cluster: %s during cleanup for %s", + cluster.getName(), account), e); + } + } + if (!Account.Type.PROJECT.equals(account.getType())) { + return; + } + deleteProjectKubernetesAccount(account); + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); @@ -1803,12 +2003,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logger.info("Running Kubernetes cluster garbage collector on Kubernetes cluster: {}", kubernetesCluster); } try { - KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); - destroyWorker = ComponentContext.inject(destroyWorker); - if (destroyWorker.destroy()) { - if (logger.isInfoEnabled()) { - logger.info("Garbage collection complete for Kubernetes cluster: {}", kubernetesCluster); - } + if (destroyKubernetesCluster(kubernetesCluster)) { + logger.info("Garbage collection complete for Kubernetes cluster: {}", kubernetesCluster); } else { logger.warn("Garbage collection failed for Kubernetes cluster : {}, it will be attempted to garbage collected in next run", kubernetesCluster); } @@ -1931,9 +2127,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logger.info("Running Kubernetes cluster state scanner on Kubernetes cluster: {} for state: {}", kubernetesCluster, KubernetesCluster.State.Destroying.toString()); } try { - KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); - destroyWorker = ComponentContext.inject(destroyWorker); - destroyWorker.destroy(); + destroyKubernetesCluster(kubernetesCluster); } catch (Exception e) { logger.warn("Failed to run Kubernetes cluster Destroying state scanner on Kubernetes cluster : {} status scanner", kubernetesCluster, e); } @@ -1947,7 +2141,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } // checks if Kubernetes cluster is in desired state - boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { + private boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); // check cluster is running at desired capacity include control nodes as well @@ -1985,6 +2179,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne createNetworkOfferingForKubernetes(DEFAULT_NSX_VPC_TIER_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME, DEFAULT_NSX_VPC_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT , true, true); + getProjectKubernetesAccountRole(); + _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 9d86c564de4..0c2338465de 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.kubernetes.cluster; +import java.util.List; + import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; @@ -34,16 +36,16 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import com.cloud.network.Network; +import com.cloud.user.Account; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; -import java.util.List; - public interface KubernetesClusterService extends PluggableService, Configurable { static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0"; static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2; static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048; static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin"; + String PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME = "Project Kubernetes Service Role"; static final ConfigKey KubernetesServiceEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.service.enabled", @@ -125,4 +127,6 @@ public interface KubernetesClusterService extends PluggableService, Configurable List removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd); boolean isDirectAccess(Network network); + + void cleanupForAccount(Account account); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java index bf49c2abb8d..d7e6f65ca05 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java @@ -32,6 +32,7 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesVersionEventTypes; +import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -50,6 +51,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet private KubernetesClusterDao kubernetesClusterDao; @Inject private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Inject + KubernetesClusterService kubernetesClusterService; protected void setEventTypeEntityDetails(Class eventTypeDefinedClass, Class entityClass) { Field[] declaredFields = eventTypeDefinedClass.getDeclaredFields(); @@ -106,6 +109,11 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet throw new CloudRuntimeException(msg); } + @Override + public void cleanupForAccount(Account account) { + kubernetesClusterService.cleanupForAccount(account); + } + @Override public String getConfigComponentName() { return KubernetesServiceHelper.class.getSimpleName(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java index 9341912012f..7df6a6b1dce 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java @@ -25,8 +25,8 @@ import com.cloud.utils.fsm.StateDao; public interface KubernetesClusterDao extends GenericDao, StateDao { - - List listByAccount(long accountId); + List listForCleanupByAccount(long accountId); + int countNotForGCByAccount(long accountId); List findKubernetesClustersToGarbageCollect(); List findManagedKubernetesClustersInState(KubernetesCluster.State state); List listByNetworkId(long networkId); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java index 63cca3563f7..7bec98d5d25 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java @@ -30,16 +30,25 @@ import com.cloud.utils.db.TransactionLegacy; @Component public class KubernetesClusterDaoImpl extends GenericDaoBase implements KubernetesClusterDao { - private final SearchBuilder AccountIdSearch; + private final SearchBuilder CleanupAccountIdSearch; + private final SearchBuilder NotForGCByAccountIDCount; private final SearchBuilder GarbageCollectedSearch; private final SearchBuilder ManagedStateSearch; private final SearchBuilder SameNetworkSearch; private final SearchBuilder KubernetesVersionSearch; public KubernetesClusterDaoImpl() { - AccountIdSearch = createSearchBuilder(); - AccountIdSearch.and("account", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); - AccountIdSearch.done(); + CleanupAccountIdSearch = createSearchBuilder(); + CleanupAccountIdSearch.and("account", CleanupAccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + CleanupAccountIdSearch.and("cluster_type", CleanupAccountIdSearch.entity().getClusterType(), SearchCriteria.Op.EQ); + CleanupAccountIdSearch.done(); + + NotForGCByAccountIDCount = createSearchBuilder(); + NotForGCByAccountIDCount.and("gc", NotForGCByAccountIDCount.entity().isCheckForGc(), SearchCriteria.Op.EQ); + NotForGCByAccountIDCount.and("account", NotForGCByAccountIDCount.entity().getAccountId(), SearchCriteria.Op.EQ); + NotForGCByAccountIDCount.and("cluster_type", NotForGCByAccountIDCount.entity().getClusterType(), SearchCriteria.Op.EQ); + NotForGCByAccountIDCount.select(null, SearchCriteria.Func.COUNT, null); + NotForGCByAccountIDCount.done(); GarbageCollectedSearch = createSearchBuilder(); GarbageCollectedSearch.and("gc", GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ); @@ -62,10 +71,20 @@ public class KubernetesClusterDaoImpl extends GenericDaoBase listByAccount(long accountId) { - SearchCriteria sc = AccountIdSearch.create(); + public List listForCleanupByAccount(long accountId) { + SearchCriteria sc = CleanupAccountIdSearch.create(); + sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged); sc.setParameters("account", accountId); - return listBy(sc, null); + return listBy(sc); + } + + @Override + public int countNotForGCByAccount(long accountId) { + SearchCriteria sc = NotForGCByAccountIDCount.create(); + sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged); + sc.setParameters("account", accountId); + sc.setParameters("gc", false); + return getCount(sc); } @Override diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index db2b5f32a7f..177f821d178 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -120,6 +120,7 @@ import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.dao.HostDao; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper; import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; import com.cloud.network.Network; @@ -887,6 +888,16 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M return cleanupAccount(account, callerUserId, caller); } + protected void cleanupPluginsResourcesIfNeeded(Account account) { + try { + KubernetesServiceHelper kubernetesServiceHelper = + ComponentContext.getDelegateComponentOfType(KubernetesServiceHelper.class); + kubernetesServiceHelper.cleanupForAccount(account); + } catch (NoSuchBeanDefinitionException ignored) { + logger.debug("No KubernetesServiceHelper bean found"); + } + } + protected boolean cleanupAccount(AccountVO account, long callerUserId, Account caller) { long accountId = account.getId(); boolean accountCleanupNeeded = false; @@ -968,6 +979,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } } + cleanupPluginsResourcesIfNeeded(account); + // Destroy the account's VMs List vms = _userVmDao.listByAccountId(accountId); if (logger.isDebugEnabled()) {