etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cks: create separate service account in project 

A separate service account will be created and added in the project, if
not exist already, when a Kubernetes cluster is deployed in a project.
This account will have a role with limited API access.

Cleanup clusters on owner account cleanup, delete service account
if needed

When the owner account of k8s clusters is deleted, while its node VMs
get expunged, the cluster entry in DB remain present. This fixes the
issue by cleaning up all clusters for the account deleted.

Project k8s service account will be deleted on account cleanup or when
there is no active k8s cluster remaining

Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com>
This commit is contained in:
Abhishek Kumar 2025-02-21 11:16:22 +05:30 committed by Pearl Dsilva
parent 9cf9966dfe
commit 19d6b979af
9 changed files with 306 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
View File

@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster;
import org.apache.cloudstack.acl.ControlledEntity;
import com.cloud.user.Account;
import com.cloud.uservm.UserVm;
import com.cloud.utils.component.Adapter;
@ -26,4 +27,5 @@ public interface KubernetesServiceHelper extends Adapter {
ControlledEntity findByUuid(String uuid);
ControlledEntity findByVmId(long vmId);
void checkVmCanBeDestroyed(UserVm userVm);
void cleanupForAccount(Account account);
}

8
engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
View File

@ -16,6 +16,9 @@
// under the License.
package com.cloud.user.dao;
import java.util.Date;
import java.util.List;
import com.cloud.user.Account;
import com.cloud.user.AccountVO;
import com.cloud.user.User;
@ -23,9 +26,6 @@ import com.cloud.utils.Pair;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDao;
import java.util.Date;
import java.util.List;
public interface AccountDao extends GenericDao<AccountVO, Long> {
Pair<User, Account> findUserAccountByApiKey(String apiKey);
@ -33,6 +33,8 @@ public interface AccountDao extends GenericDao<AccountVO, Long> {
Pair<List<AccountVO>, Integer> findAccountsLike(String accountName, Filter filter);
List<AccountVO> findAccountsByName(String accountName);
List<AccountVO> findActiveAccounts(Long maxAccountId, Filter filter);
List<AccountVO> findRecentlyDeletedAccounts(Long maxAccountId, Date earliestRemovedDate, Filter filter);

25
engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
View File

@ -16,6 +16,14 @@
// under the License.
package com.cloud.user.dao;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import com.cloud.user.Account;
import com.cloud.user.Account.State;
import com.cloud.user.AccountVO;
@ -30,14 +38,7 @@ import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Func;
import com.cloud.utils.db.SearchCriteria.Op;
import org.apache.commons.lang3.StringUtils;
import com.cloud.utils.db.TransactionLegacy;
import org.springframework.stereotype.Component;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.Date;
import java.util.List;
@Component
public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements AccountDao {
@ -190,6 +191,16 @@ public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements A
return searchAndCount(sc, filter);
}
@Override
public List<AccountVO> findAccountsByName(String accountName) {
SearchBuilder<AccountVO> sb = createSearchBuilder();
sb.and("accountName", sb.entity().getAccountName(), SearchCriteria.Op.EQ);
sb.done();
SearchCriteria<AccountVO> sc = sb.create();
sc.setParameters("accountName", accountName);
return search(sc, null);
}
@Override
public Account findEnabledAccount(String accountName, Long domainId) {
SearchCriteria<AccountVO> sc = AllFieldsSearch.create("accountName", accountName);

256
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View File

@ -44,6 +44,11 @@ import javax.naming.ConfigurationException;
import com.cloud.uservm.UserVm;
import com.cloud.vm.UserVmService;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.Role;
import org.apache.cloudstack.acl.RolePermissionEntity;
import org.apache.cloudstack.acl.RoleService;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.acl.Rule;
import org.apache.cloudstack.acl.SecurityChecker;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
@ -52,6 +57,14 @@ import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiConstants.VMDetails;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.ResponseObject.ResponseView;
import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd;
import org.apache.cloudstack.api.command.user.address.DisassociateIPAddrCmd;
import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd;
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
import org.apache.cloudstack.api.command.user.firewall.DeleteFirewallRuleCmd;
import org.apache.cloudstack.api.command.user.firewall.ListFirewallRulesCmd;
import org.apache.cloudstack.api.command.user.firewall.UpdateFirewallRuleCmd;
import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@ -62,6 +75,18 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetes
import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.loadbalancer.AssignToLoadBalancerRuleCmd;
import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd;
import org.apache.cloudstack.api.command.user.loadbalancer.DeleteLoadBalancerRuleCmd;
import org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRuleInstancesCmd;
import org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRulesCmd;
import org.apache.cloudstack.api.command.user.loadbalancer.RemoveFromLoadBalancerRuleCmd;
import org.apache.cloudstack.api.command.user.loadbalancer.UpdateLoadBalancerRuleCmd;
import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd;
import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
import org.apache.cloudstack.api.command.user.network.ListNetworksCmd;
import org.apache.cloudstack.api.command.user.vm.ListVMsCmd;
import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.ListResponse;
@ -147,6 +172,8 @@ import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.projects.Project;
import com.cloud.projects.ProjectAccount;
import com.cloud.projects.ProjectManager;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
@ -155,14 +182,17 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountService;
import com.cloud.user.AccountVO;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.user.User;
import com.cloud.user.UserAccount;
import com.cloud.user.UserVO;
import com.cloud.user.dao.AccountDao;
import com.cloud.user.dao.SSHKeyPairDao;
import com.cloud.user.dao.UserDao;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.UuidUtils;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
@ -186,12 +216,40 @@ import org.apache.logging.log4j.Level;
public class KubernetesClusterManagerImpl extends ManagerBase implements KubernetesClusterService {
private static final String DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNetworkOfferingforKubernetesService";
private static final String DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network Offering used for CloudStack Kubernetes service";
private static final String DEFAULT_NSX_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNSXNetworkOfferingforKubernetesService";
private static final String DEFAULT_NSX_VPC_TIER_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNSXVPCNetworkOfferingforKubernetesService";
private static final String DEFAULT_NSX_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network Offering for NSX CloudStack Kubernetes Service";
private static final String DEFAULT_NSX_VPC_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network Offering for NSX CloudStack Kubernetes service on VPC";
private static final List<Class<?>> PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList(
QueryAsyncJobResultCmd.class,
ListVMsCmd.class,
ListNetworksCmd.class,
ListPublicIpAddressesCmd.class,
AssociateIPAddrCmd.class,
DisassociateIPAddrCmd.class,
ListLoadBalancerRulesCmd.class,
CreateLoadBalancerRuleCmd.class,
UpdateLoadBalancerRuleCmd.class,
DeleteLoadBalancerRuleCmd.class,
AssignToLoadBalancerRuleCmd.class,
RemoveFromLoadBalancerRuleCmd.class,
ListLoadBalancerRuleInstancesCmd.class,
ListFirewallRulesCmd.class,
CreateFirewallRuleCmd.class,
UpdateFirewallRuleCmd.class,
DeleteFirewallRuleCmd.class,
ListNetworkACLsCmd.class,
CreateNetworkACLCmd.class,
DeleteNetworkACLCmd.class,
ListKubernetesClustersCmd.class,
ScaleKubernetesClusterCmd.class
);
private static final String PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME = "Kubernetes";
private static final String PROJECT_KUBERNETES_ACCOUNT_LAST_NAME = "Service User";
protected StateMachine2<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> _stateMachine = KubernetesCluster.State.getStateMachine();
ScheduledExecutorService _gcExecutor;
@ -263,11 +321,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
public SecurityGroupService securityGroupService;
@Inject
public NetworkHelper networkHelper;
@Inject
private UserVmService userVmService;
@Inject
RoutedIpv4Manager routedIpv4Manager;
@Inject
public AccountDao accountDao;
@Inject
public ProjectManager projectManager;
@Inject
RoleService roleService;
private void logMessage(final Level logLevel, final String message, final Exception e) {
if (logLevel == Level.WARN) {
@ -1380,23 +1443,31 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
}
private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster) {
Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
if (owner == null || owner.getType() == Account.Type.PROJECT) {
owner = CallContext.current().getCallingAccount();
protected String[] createUserApiKeyAndSecretKey(long userId) {
CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
try {
return accountService.createApiKeyAndSecretKey(userId);
} finally {
CallContext.unregister();
}
}
protected String[] getServiceUserKeys(Account owner) {
String username = owner.getAccountName();
if (!username.startsWith(KUBEADMIN_ACCOUNT_NAME + "-")) {
username += "-" + KUBEADMIN_ACCOUNT_NAME;
}
String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME;
UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId());
String[] keys = null;
String[] keys;
if (kubeadmin == null) {
User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(),
KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
keys = accountService.createApiKeyAndSecretKey(kube.getId());
KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
keys = createUserApiKeyAndSecretKey(kube.getId());
} else {
String apiKey = kubeadmin.getApiKey();
String secretKey = kubeadmin.getSecretKey();
if (StringUtils.isAnyEmpty(apiKey, secretKey)) {
keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId());
keys = createUserApiKeyAndSecretKey(kubeadmin.getId());
} else {
keys = new String[]{apiKey, secretKey};
}
@ -1404,6 +1475,76 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
return keys;
}
protected Role createProjectKubernetesAccountRole() {
Role role = roleService.createRole(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, RoleType.User,
PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, false);
for (Class<?> allowedApi : PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS) {
final String apiName = BaseCmd.getCommandNameByClass(allowedApi);
roleService.createRolePermission(role, new Rule(apiName), RolePermissionEntity.Permission.ALLOW,
String.format("Allow %s", apiName));
}
roleService.createRolePermission(role, new Rule("*"), RolePermissionEntity.Permission.DENY,
"Deny all");
logger.debug(String.format("Created default role for Kubernetes service account in projects: %s", role));
return role;
}
public Role getProjectKubernetesAccountRole() {
List<Role> roles = roleService.findRolesByName(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME);
if (CollectionUtils.isNotEmpty(roles)) {
Role role = roles.get(0);
logger.debug(String.format("Found default role for Kubernetes service account in projects: %s", role));
return role;
}
return createProjectKubernetesAccountRole();
}
protected Account createProjectKubernetesAccount(final Project project, final String accountName) {
CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
try {
Role role = getProjectKubernetesAccountRole();
UserAccount userAccount = accountService.createUserAccount(accountName,
UuidUtils.first(UUID.randomUUID().toString()), PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME,
PROJECT_KUBERNETES_ACCOUNT_LAST_NAME, null, null, accountName, Account.Type.NORMAL, role.getId(),
project.getDomainId(), null, null, null, null, User.Source.NATIVE);
projectManager.assignAccountToProject(project, userAccount.getAccountId(), ProjectAccount.Role.Regular,
userAccount.getId(), null);
Account account = accountService.getAccount(userAccount.getAccountId());
logger.debug(String.format("Created Kubernetes service account in project %s: %s", project, account));
return account;
} finally {
CallContext.unregister();
}
}
protected Account getProjectKubernetesAccount(final Account callerAccount, final boolean create) {
Project project = ApiDBUtils.findProjectByProjectAccountId(callerAccount.getId());
final String accountName = String.format("%s-%s", KUBEADMIN_ACCOUNT_NAME, UuidUtils.first(project.getUuid()));
List<AccountVO> accounts = accountDao.findAccountsByName(accountName);
for (AccountVO account : accounts) {
if (projectManager.canAccessProjectAccount(account, project.getProjectAccountId())) {
logger.debug(String.format("Created Kubernetes service account in project %s: %s", project, account));
return account;
}
}
return create ? createProjectKubernetesAccount(project, accountName) : null;
}
protected Account getProjectKubernetesAccount(final Account callerAccount) {
return getProjectKubernetesAccount(callerAccount, true);
}
private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster) {
Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
if (owner == null) {
owner = CallContext.current().getCallingAccount();
}
if (owner.getType() == Account.Type.PROJECT) {
owner = getProjectKubernetesAccount(owner);
}
return getServiceUserKeys(owner);
}
@Override
@ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_STOP,
eventDescription = "stopping Kubernetes cluster", async = true)
@ -1448,15 +1589,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
Long kubernetesClusterId = cmd.getId();
KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId);
final KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId);
if (cluster == null) {
throw new InvalidParameterValueException("Invalid cluster id specified");
}
accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, cluster);
if (cluster.getClusterType() == KubernetesCluster.ClusterType.CloudManaged) {
KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(cluster, this);
destroyWorker = ComponentContext.inject(destroyWorker);
return destroyWorker.destroy();
return destroyKubernetesCluster(cluster);
} else {
boolean cleanup = cmd.getCleanup();
boolean expunge = cmd.getExpunge();
@ -1483,13 +1622,14 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
}
}
return Transaction.execute(new TransactionCallback<Boolean>() {
@Override
public Boolean doInTransaction(TransactionStatus status) {
kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
return kubernetesClusterDao.remove(kubernetesClusterId);
return Transaction.execute((TransactionCallback<Boolean>) status -> {
kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
if (kubernetesClusterDao.remove(kubernetesClusterId)) {
deleteProjectKubernetesAccountIfNeeded(cluster);
return true;
}
return false;
});
}
}
@ -1752,6 +1892,66 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
return responseList;
}
protected void deleteProjectKubernetesAccount(Account projectAccount) {
CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
try {
Account serviceAccount = getProjectKubernetesAccount(projectAccount, false);
if (serviceAccount != null) {
accountManager.deleteAccount(accountDao.findById(serviceAccount.getId()), User.UID_SYSTEM,
accountService.getSystemAccount());
}
} finally {
CallContext.unregister();
}
}
protected void deleteProjectKubernetesAccountIfNeeded(final KubernetesCluster kubernetesCluster) {
Account owner = accountService.getAccount(kubernetesCluster.getAccountId());
if (owner == null) {
return;
}
if (Account.Type.PROJECT.equals(owner.getType()) &&
kubernetesClusterDao.countNotForGCByAccount(owner.getAccountId()) == 0) {
deleteProjectKubernetesAccount(owner);
}
}
protected boolean destroyKubernetesCluster(KubernetesCluster kubernetesCluster, boolean deleteProjectAccount) {
KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster,
KubernetesClusterManagerImpl.this);
destroyWorker = ComponentContext.inject(destroyWorker);
boolean result = destroyWorker.destroy();
if (deleteProjectAccount) {
deleteProjectKubernetesAccountIfNeeded(kubernetesCluster);
}
return result;
}
protected boolean destroyKubernetesCluster(KubernetesCluster kubernetesCluster) {
return destroyKubernetesCluster(kubernetesCluster, true);
}
@Override
public void cleanupForAccount(Account account) {
List<KubernetesClusterVO> clusters = kubernetesClusterDao.listForCleanupByAccount(account.getId());
if (CollectionUtils.isEmpty(clusters)) {
return;
}
logger.debug(String.format("Cleaning up %d Kubernetes cluster for %s", clusters.size(), account));
for (KubernetesClusterVO cluster : clusters) {
try {
destroyKubernetesCluster(cluster, false);
} catch (CloudRuntimeException e) {
logger.warn(String.format("Failed to destroy Kubernetes cluster: %s during cleanup for %s",
cluster.getName(), account), e);
}
}
if (!Account.Type.PROJECT.equals(account.getType())) {
return;
}
deleteProjectKubernetesAccount(account);
}
@Override
public List<Class<?>> getCommands() {
List<Class<?>> cmdList = new ArrayList<Class<?>>();
@ -1803,12 +2003,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
logger.info("Running Kubernetes cluster garbage collector on Kubernetes cluster: {}", kubernetesCluster);
}
try {
KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
destroyWorker = ComponentContext.inject(destroyWorker);
if (destroyWorker.destroy()) {
if (logger.isInfoEnabled()) {
logger.info("Garbage collection complete for Kubernetes cluster: {}", kubernetesCluster);
}
if (destroyKubernetesCluster(kubernetesCluster)) {
logger.info("Garbage collection complete for Kubernetes cluster: {}", kubernetesCluster);
} else {
logger.warn("Garbage collection failed for Kubernetes cluster : {}, it will be attempted to garbage collected in next run", kubernetesCluster);
}
@ -1931,9 +2127,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
logger.info("Running Kubernetes cluster state scanner on Kubernetes cluster: {} for state: {}", kubernetesCluster, KubernetesCluster.State.Destroying.toString());
}
try {
KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
destroyWorker = ComponentContext.inject(destroyWorker);
destroyWorker.destroy();
destroyKubernetesCluster(kubernetesCluster);
} catch (Exception e) {
logger.warn("Failed to run Kubernetes cluster Destroying state scanner on Kubernetes cluster : {} status scanner", kubernetesCluster, e);
}
@ -1947,7 +2141,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
// checks if Kubernetes cluster is in desired state
boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) {
private boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) {
List<KubernetesClusterVmMapVO> clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
// check cluster is running at desired capacity include control nodes as well
@ -1985,6 +2179,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
createNetworkOfferingForKubernetes(DEFAULT_NSX_VPC_TIER_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME,
DEFAULT_NSX_VPC_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT , true, true);
getProjectKubernetesAccountRole();
_gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS);
_stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS);

8
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
View File

@ -16,6 +16,8 @@
// under the License.
package com.cloud.kubernetes.cluster;
import java.util.List;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@ -34,16 +36,16 @@ import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import com.cloud.network.Network;
import com.cloud.user.Account;
import com.cloud.utils.component.PluggableService;
import com.cloud.utils.exception.CloudRuntimeException;
import java.util.List;
public interface KubernetesClusterService extends PluggableService, Configurable {
static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";
String PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME = "Project Kubernetes Service Role";
static final ConfigKey<Boolean> KubernetesServiceEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
"cloud.kubernetes.service.enabled",
@ -125,4 +127,6 @@ public interface KubernetesClusterService extends PluggableService, Configurable
List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd);
boolean isDirectAccess(Network network);
void cleanupForAccount(Account account);
}

8
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
View File

@ -32,6 +32,7 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.kubernetes.version.KubernetesSupportedVersion;
import com.cloud.kubernetes.version.KubernetesVersionEventTypes;
import com.cloud.user.Account;
import com.cloud.uservm.UserVm;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.exception.CloudRuntimeException;
@ -50,6 +51,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet
private KubernetesClusterDao kubernetesClusterDao;
@Inject
private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@Inject
KubernetesClusterService kubernetesClusterService;
protected void setEventTypeEntityDetails(Class<?> eventTypeDefinedClass, Class<?> entityClass) {
Field[] declaredFields = eventTypeDefinedClass.getDeclaredFields();
@ -106,6 +109,11 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet
throw new CloudRuntimeException(msg);
}
@Override
public void cleanupForAccount(Account account) {
kubernetesClusterService.cleanupForAccount(account);
}
@Override
public String getConfigComponentName() {
return KubernetesServiceHelper.class.getSimpleName();

4
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
View File

@ -25,8 +25,8 @@ import com.cloud.utils.fsm.StateDao;
public interface KubernetesClusterDao extends GenericDao<KubernetesClusterVO, Long>,
StateDao<KubernetesCluster.State, KubernetesCluster.Event, KubernetesCluster> {
List<KubernetesClusterVO> listByAccount(long accountId);
List<KubernetesClusterVO> listForCleanupByAccount(long accountId);
int countNotForGCByAccount(long accountId);
List<KubernetesClusterVO> findKubernetesClustersToGarbageCollect();
List<KubernetesClusterVO> findManagedKubernetesClustersInState(KubernetesCluster.State state);
List<KubernetesClusterVO> listByNetworkId(long networkId);

33
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
View File

@ -30,16 +30,25 @@ import com.cloud.utils.db.TransactionLegacy;
@Component
public class KubernetesClusterDaoImpl extends GenericDaoBase<KubernetesClusterVO, Long> implements KubernetesClusterDao {
private final SearchBuilder<KubernetesClusterVO> AccountIdSearch;
private final SearchBuilder<KubernetesClusterVO> CleanupAccountIdSearch;
private final SearchBuilder<KubernetesClusterVO> NotForGCByAccountIDCount;
private final SearchBuilder<KubernetesClusterVO> GarbageCollectedSearch;
private final SearchBuilder<KubernetesClusterVO> ManagedStateSearch;
private final SearchBuilder<KubernetesClusterVO> SameNetworkSearch;
private final SearchBuilder<KubernetesClusterVO> KubernetesVersionSearch;
public KubernetesClusterDaoImpl() {
AccountIdSearch = createSearchBuilder();
AccountIdSearch.and("account", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
AccountIdSearch.done();
CleanupAccountIdSearch = createSearchBuilder();
CleanupAccountIdSearch.and("account", CleanupAccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
CleanupAccountIdSearch.and("cluster_type", CleanupAccountIdSearch.entity().getClusterType(), SearchCriteria.Op.EQ);
CleanupAccountIdSearch.done();
NotForGCByAccountIDCount = createSearchBuilder();
NotForGCByAccountIDCount.and("gc", NotForGCByAccountIDCount.entity().isCheckForGc(), SearchCriteria.Op.EQ);
NotForGCByAccountIDCount.and("account", NotForGCByAccountIDCount.entity().getAccountId(), SearchCriteria.Op.EQ);
NotForGCByAccountIDCount.and("cluster_type", NotForGCByAccountIDCount.entity().getClusterType(), SearchCriteria.Op.EQ);
NotForGCByAccountIDCount.select(null, SearchCriteria.Func.COUNT, null);
NotForGCByAccountIDCount.done();
GarbageCollectedSearch = createSearchBuilder();
GarbageCollectedSearch.and("gc", GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ);
@ -62,10 +71,20 @@ public class KubernetesClusterDaoImpl extends GenericDaoBase<KubernetesClusterVO
}
@Override
public List<KubernetesClusterVO> listByAccount(long accountId) {
SearchCriteria<KubernetesClusterVO> sc = AccountIdSearch.create();
public List<KubernetesClusterVO> listForCleanupByAccount(long accountId) {
SearchCriteria<KubernetesClusterVO> sc = CleanupAccountIdSearch.create();
sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged);
sc.setParameters("account", accountId);
return listBy(sc, null);
return listBy(sc);
}
@Override
public int countNotForGCByAccount(long accountId) {
SearchCriteria<KubernetesClusterVO> sc = NotForGCByAccountIDCount.create();
sc.setParameters("cluster_type", KubernetesCluster.ClusterType.CloudManaged);
sc.setParameters("account", accountId);
sc.setParameters("gc", false);
return getCount(sc);
}
@Override

13
server/src/main/java/com/cloud/user/AccountManagerImpl.java
View File

@ -120,6 +120,7 @@ import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.dao.HostDao;
import com.cloud.kubernetes.cluster.KubernetesServiceHelper;
import com.cloud.network.IpAddress;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
@ -887,6 +888,16 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
return cleanupAccount(account, callerUserId, caller);
}
protected void cleanupPluginsResourcesIfNeeded(Account account) {
try {
KubernetesServiceHelper kubernetesServiceHelper =
ComponentContext.getDelegateComponentOfType(KubernetesServiceHelper.class);
kubernetesServiceHelper.cleanupForAccount(account);
} catch (NoSuchBeanDefinitionException ignored) {
logger.debug("No KubernetesServiceHelper bean found");
}
}
protected boolean cleanupAccount(AccountVO account, long callerUserId, Account caller) {
long accountId = account.getId();
boolean accountCleanupNeeded = false;
@ -968,6 +979,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M
}
}
cleanupPluginsResourcesIfNeeded(account);
// Destroy the account's VMs
List<UserVmVO> vms = _userVmDao.listByAccountId(accountId);
if (logger.isDebugEnabled()) {