etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support to deploy CKS cluster nodes on hosts dedicated to a domain 

---------

Co-authored-by: Pearl Dsilva <pearl1594@gmail.com>
This commit is contained in:
Nicolas Vazquez 2024-05-02 10:19:43 -03:00 committed by nvazquez
parent f103c43f09
commit b72a7cd020
No known key found for this signature in database
GPG Key ID: 656E1BCC8CB54F84
8 changed files with 202 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View File

@ -46,12 +46,16 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.dc.DedicatedResourceVO;
import com.cloud.dc.dao.DedicatedResourceDao;
import com.cloud.host.Host;
import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker;
import com.cloud.network.dao.NsxProviderDao;
import com.cloud.network.element.NsxProviderVO;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker;
import com.cloud.template.TemplateApiService;
import com.cloud.user.dao.AccountDao;
import com.cloud.uservm.UserVm;
import com.cloud.vm.NicVO;
import com.cloud.vm.UserVmService;
@ -59,6 +63,8 @@ import com.cloud.vm.dao.NicDao;
import com.cloud.vm.dao.UserVmDetailsDao;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.SecurityChecker;
import org.apache.cloudstack.affinity.AffinityGroupVO;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
import org.apache.cloudstack.api.ApiConstants;
@ -118,7 +124,6 @@ import com.cloud.exception.InsufficientServerCapacityException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.host.Host.Type;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
@ -236,12 +241,18 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
@Inject
protected HostDao hostDao;
@Inject
protected AffinityGroupDao affinityGroupDao;
@Inject
protected ServiceOfferingDao serviceOfferingDao;
@Inject
protected VMTemplateDao templateDao;
@Inject
protected TemplateJoinDao templateJoinDao;
@Inject
protected DedicatedResourceDao dedicatedResourceDao;
@Inject
protected AccountDao accountDao;
@Inject
protected AccountService accountService;
@Inject
protected AccountManager accountManager;
@ -531,16 +542,43 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
}
private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException {
public Long getExplicitAffinityGroup(Long domainId) {
AffinityGroupVO groupVO = affinityGroupDao.findDomainLevelGroupByType(domainId, "ExplicitDedication");
if (Objects.nonNull(groupVO)) {
return groupVO.getId();
}
return null;
}
private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering,
final Long domainId, final Long accountId) throws InsufficientServerCapacityException {
final int cpu_requested = offering.getCpu() * offering.getSpeed();
final long ram_requested = offering.getRamSize() * 1024L * 1024L;
List<HostVO> hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, zone.getId());
boolean useDedicatedHosts = false;
Long group = getExplicitAffinityGroup(domainId);
List<HostVO> hosts = new ArrayList<>();
if (Objects.nonNull(group)) {
List<DedicatedResourceVO> dedicatedHosts = new ArrayList<>();
if (Objects.nonNull(accountId)) {
dedicatedHosts = dedicatedResourceDao.listByAccountId(accountId);
} else if (Objects.nonNull(domainId)) {
dedicatedHosts = dedicatedResourceDao.listByDomainId(domainId);
}
for (DedicatedResourceVO dedicatedHost : dedicatedHosts) {
hosts.add(hostDao.findById(dedicatedHost.getHostId()));
useDedicatedHosts = true;
}
}
if (hosts.isEmpty()) {
hosts = resourceManager.listAllHostsInOneZoneByType(Host.Type.Routing, zone.getId());
}
final Map<String, Pair<HostVO, Integer>> hosts_with_resevered_capacity = new ConcurrentHashMap<String, Pair<HostVO, Integer>>();
for (HostVO h : hosts) {
hosts_with_resevered_capacity.put(h.getUuid(), new Pair<HostVO, Integer>(h, 0));
}
boolean suitable_host_found = false;
Cluster planCluster = null;
HostVO suitableHost = null;
for (int i = 1; i <= nodesCount; i++) {
suitable_host_found = false;
for (Map.Entry<String, Pair<HostVO, Integer>> hostEntry : hosts_with_resevered_capacity.entrySet()) {
@ -566,6 +604,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
hostEntry.setValue(new Pair<HostVO, Integer>(hostVO, reserved));
suitable_host_found = true;
suitableHost = hostVO;
planCluster = cluster;
break;
}
@ -581,6 +620,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
if (logger.isInfoEnabled()) {
logger.info(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid()));
}
if (useDedicatedHosts) {
planCluster = clusterDao.findById(suitableHost.getClusterId());
return new DeployDestination(zone, null, planCluster, suitableHost);
}
return new DeployDestination(zone, null, planCluster, null);
}
String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%s) with offering ID: %s",
@ -1349,7 +1392,16 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
Map<String, Long> serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap();
Long defaultServiceOfferingId = cmd.getServiceOfferingId();
Hypervisor.HypervisorType hypervisorType = getHypervisorTypeAndValidateNodeDeployments(serviceOfferingNodeTypeMap, defaultServiceOfferingId, nodeTypeCount, zone);
String accountName = cmd.getAccountName();
Long domainId = cmd.getDomainId();
Long accountId = null;
if (Objects.nonNull(accountName) && Objects.nonNull(domainId)) {
Account account = accountDao.findActiveAccount(accountName, domainId);
if (Objects.nonNull(account)) {
accountId = account.getId();
}
}
Hypervisor.HypervisorType hypervisorType = getHypervisorTypeAndValidateNodeDeployments(serviceOfferingNodeTypeMap, defaultServiceOfferingId, nodeTypeCount, zone, domainId, accountId);
SecurityGroup securityGroup = null;
if (zone.isSecurityGroupEnabled()) {
@ -1434,7 +1486,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
protected Hypervisor.HypervisorType getHypervisorTypeAndValidateNodeDeployments(Map<String, Long> serviceOfferingNodeTypeMap,
Long defaultServiceOfferingId,
Map<String, Long> nodeTypeCount, DataCenter zone) {
Map<String, Long> nodeTypeCount,
DataCenter zone, Long domainId, Long accountId) {
Hypervisor.HypervisorType hypervisorType = null;
for (String nodeType : CLUSTER_NODES_TYPES_LIST) {
if (!nodeTypeCount.containsKey(nodeType)) {
@ -1448,7 +1501,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
(!serviceOfferingNodeTypeMap.containsKey(ETCD.name()) || nodes == 0)) {
continue;
}
DeployDestination deployDestination = plan(nodes, zone, serviceOffering);
DeployDestination deployDestination = plan(nodes, zone, serviceOffering, domainId, accountId);
if (deployDestination.getCluster() == null) {
logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone : %s", zone.getName()));
}
@ -1491,14 +1544,17 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
* are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources
* are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just
* start the VM's (which can possibly implicitly start the network also).
*
* @param kubernetesClusterId
* @param domainId
* @param accountName
* @param onCreate
* @return
* @throws CloudRuntimeException
*/
@Override
public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws CloudRuntimeException {
public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException {
if (!KubernetesServiceEnabled.value()) {
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
@ -1526,6 +1582,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
if (zone == null) {
logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster : %s", kubernetesCluster.getName()));
}
Long accountId = null;
if (Objects.nonNull(accountName) && Objects.nonNull(domainId)) {
Account account = accountDao.findActiveAccount(accountName, domainId);
if (Objects.nonNull(account)) {
accountId = account.getId();
}
}
KubernetesClusterStartWorker startWorker =
new KubernetesClusterStartWorker(kubernetesCluster, this);
startWorker = ComponentContext.inject(startWorker);
@ -1533,10 +1596,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
// Start for Kubernetes cluster in 'Created' state
String[] keys = getServiceUserKeys(kubernetesCluster);
startWorker.setKeys(keys);
return startWorker.startKubernetesClusterOnCreate();
return startWorker.startKubernetesClusterOnCreate(domainId, accountId);
} else {
// Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started
return startWorker.startStoppedKubernetesCluster();
return startWorker.startStoppedKubernetesCluster(domainId, accountId);
}
}

2
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
View File

@ -136,7 +136,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable
KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException;
boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws CloudRuntimeException;
boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException;
boolean stopKubernetesCluster(StopKubernetesClusterCmd cmd) throws CloudRuntimeException;

18
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
View File

@ -59,6 +59,8 @@ import com.cloud.vm.NicVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.UserVmManager;
import org.apache.cloudstack.affinity.AffinityGroupVO;
import org.apache.cloudstack.affinity.dao.AffinityGroupDao;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
import org.apache.cloudstack.ca.CAManager;
@ -194,6 +196,8 @@ public class KubernetesClusterActionWorker {
protected FirewallService firewallService;
@Inject
private NicDao nicDao;
@Inject
protected AffinityGroupDao affinityGroupDao;
protected KubernetesClusterDao kubernetesClusterDao;
protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@ -983,4 +987,18 @@ public class KubernetesClusterActionWorker {
});
return vmIdPortMap;
}
public Long getExplicitAffinityGroup(Long domainId, Long accountId) {
AffinityGroupVO groupVO = null;
if (Objects.nonNull(accountId)) {
groupVO = affinityGroupDao.findByAccountAndType(accountId, "ExplicitDedication");
}
if (Objects.isNull(groupVO)) {
groupVO = affinityGroupDao.findDomainLevelGroupByType(domainId, "ExplicitDedication");
}
if (Objects.nonNull(groupVO)) {
return groupVO.getId();
}
return null;
}
}

163
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
View File

@ -27,6 +27,7 @@ import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -36,19 +37,18 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO;
import com.cloud.dc.DedicatedResourceVO;
import com.cloud.dc.dao.DedicatedResourceDao;
import com.cloud.deploy.DataCenterDeployment;
import com.cloud.deploy.DeploymentPlan;
import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType;
import com.cloud.kubernetes.cluster.KubernetesClusterService;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
import com.cloud.network.rules.FirewallManager;
import com.cloud.network.rules.RulesService;
import com.cloud.network.rules.dao.PortForwardingRulesDao;
import com.cloud.offering.NetworkOffering;
import com.cloud.offerings.dao.NetworkOfferingDao;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.utils.db.TransactionCallbackWithException;
import com.cloud.utils.net.Ip;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
@ -146,6 +146,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
@Inject
protected ResourceManager resourceManager;
@Inject
protected DedicatedResourceDao dedicatedResourceDao;
@Inject
protected LoadBalancerDao loadBalancerDao;
@Inject
protected VMInstanceDao vmInstanceDao;
@ -171,88 +173,34 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
kubernetesClusterNodeNamePrefix = getKubernetesClusterNodeNamePrefix();
}
private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
final String ejectIsoKey = "{{ k8s.eject.iso }}";
final String installWaitTime = "{{ k8s.install.wait.time }}";
final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
final Long waitTime = KubernetesClusterService.KubernetesWorkerNodeInstallAttemptWait.value();
final Long reattempts = KubernetesClusterService.KubernetesWorkerNodeInstallReattempts.value();
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
String sshKeyPair = kubernetesCluster.getKeyPair();
if (StringUtils.isNotEmpty(sshKeyPair)) {
SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair);
if (sshkp != null) {
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
}
}
k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey);
k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp);
k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
k8sNodeConfig = k8sNodeConfig.replace(installWaitTime, String.valueOf(waitTime));
k8sNodeConfig = k8sNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts));
k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig);
return k8sNodeConfig;
}
protected String updateKubeConfigWithRegistryDetails(String k8sConfig) {
/* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to
* use docker private registry */
String registryUsername = null;
String registryPassword = null;
String registryUrl = null;
List<KubernetesClusterDetailsVO> details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId());
for (KubernetesClusterDetailsVO detail : details) {
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) {
registryUsername = detail.getValue();
}
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) {
registryPassword = detail.getValue();
}
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) {
registryUrl = detail.getValue();
}
}
if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) {
// Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details
String runCmd = "- bash -x /opt/bin/setup-containerd";
String registryEp = registryUrl.split("://")[1];
k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd);
final String registryUrlKey = "{{registry.url}}";
final String registryUrlEpKey = "{{registry.url.endpoint}}";
final String registryAuthKey = "{{registry.token}}";
final String registryUname = "{{registry.username}}";
final String registryPsswd = "{{registry.password}}";
final String usernamePasswordKey = registryUsername + ":" + registryPassword;
String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl);
k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp);
k8sConfig = k8sConfig.replace(registryUname, registryUsername);
k8sConfig = k8sConfig.replace(registryPsswd, registryPassword);
k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth);
}
return k8sConfig;
}
protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException {
protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering,
final Long domainId, final Long accountId) throws InsufficientServerCapacityException {
final int cpu_requested = offering.getCpu() * offering.getSpeed();
final long ram_requested = offering.getRamSize() * 1024L * 1024L;
List<HostVO> hosts = resourceManager.listAllHostsInOneZoneByType(Host.Type.Routing, zone.getId());
boolean useDedicatedHosts = false;
List<HostVO> hosts = new ArrayList<>();
Long group = getExplicitAffinityGroup(domainId, accountId);
if (Objects.nonNull(group)) {
List<DedicatedResourceVO> dedicatedHosts = new ArrayList<>();
if (Objects.nonNull(accountId)) {
dedicatedHosts = dedicatedResourceDao.listByAccountId(accountId);
} else if (Objects.nonNull(domainId)) {
dedicatedHosts = dedicatedResourceDao.listByDomainId(domainId);
}
for (DedicatedResourceVO dedicatedHost : dedicatedHosts) {
hosts.add(hostDao.findById(dedicatedHost.getHostId()));
useDedicatedHosts = true;
}
}
if (hosts.isEmpty()) {
hosts = resourceManager.listAllHostsInOneZoneByType(Host.Type.Routing, zone.getId());
}
final Map<String, Pair<HostVO, Integer>> hosts_with_resevered_capacity = new ConcurrentHashMap<String, Pair<HostVO, Integer>>();
for (HostVO h : hosts) {
hosts_with_resevered_capacity.put(h.getUuid(), new Pair<HostVO, Integer>(h, 0));
}
boolean suitable_host_found = false;
HostVO suitableHost = null;
for (int i = 1; i <= nodesCount; i++) {
suitable_host_found = false;
for (Map.Entry<String, Pair<HostVO, Integer>> hostEntry : hosts_with_resevered_capacity.entrySet()) {
@ -281,6 +229,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
}
hostEntry.setValue(new Pair<HostVO, Integer>(h, reserved));
suitable_host_found = true;
suitableHost = h;
break;
}
}
@ -296,6 +245,9 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
if (logger.isInfoEnabled()) {
logger.info(String.format("Suitable hosts found in datacenter : %s, creating deployment destination", zone.getName()));
}
if (useDedicatedHosts) {
return new DeployDestination(zone, null, null, suitableHost);
}
return new DeployDestination(zone, null, null, null);
}
String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%s) with offering : %s and hypervisor: %s",
@ -305,13 +257,13 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId());
}
protected DeployDestination plan() throws InsufficientServerCapacityException {
protected DeployDestination plan(Long domainId, Long accountId) throws InsufficientServerCapacityException {
ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId());
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
if (logger.isDebugEnabled()) {
logger.debug(String.format("Checking deployment destination for Kubernetes cluster : %s in zone : %s", kubernetesCluster.getName(), zone.getName()));
}
return plan(kubernetesCluster.getTotalNodeCount(), zone, offering);
return plan(kubernetesCluster.getTotalNodeCount(), zone, offering, domainId, accountId);
}
protected void resizeNodeVolume(final UserVm vm) throws ManagementServerException {
@ -337,14 +289,33 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
}
}
protected void startKubernetesVM(final UserVm vm) throws ManagementServerException {
protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId) throws ManagementServerException {
try {
StartVMCmd startVm = new StartVMCmd();
startVm = ComponentContext.inject(startVm);
Field f = startVm.getClass().getDeclaredField("id");
f.setAccessible(true);
f.set(startVm, vm.getId());
itMgr.advanceStart(vm.getUuid(), null, null);
DeploymentPlan planner = null;
if (Objects.nonNull(domainId) && !listDedicatedHostsInDomain(domainId).isEmpty()) {
DeployDestination dest = null;
try {
dest = plan(domainId, accountId);
} catch (InsufficientCapacityException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
}
if (dest != null) {
planner = new DataCenterDeployment(
Objects.nonNull(dest.getDataCenter()) ? dest.getDataCenter().getId() : 0,
Objects.nonNull(dest.getPod()) ? dest.getPod().getId() : null,
Objects.nonNull(dest.getCluster()) ? dest.getCluster().getId() : null,
Objects.nonNull(dest.getHost()) ? dest.getHost().getId() : null,
null,
null);
}
}
itMgr.advanceStart(vm.getUuid(), null, planner, null);
if (logger.isInfoEnabled()) {
logger.info(String.format("Started VM : %s in the Kubernetes cluster : %s", vm.getDisplayName(), kubernetesCluster.getName()));
}
@ -358,16 +329,17 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
}
}
protected List<UserVm> provisionKubernetesClusterNodeVms(final long nodeCount, final int offset, final String publicIpAddress) throws ManagementServerException,
protected List<UserVm> provisionKubernetesClusterNodeVms(final long nodeCount, final int offset,
final String publicIpAddress, final Long domainId, final Long accountId) throws ManagementServerException,
ResourceUnavailableException, InsufficientCapacityException {
List<UserVm> nodes = new ArrayList<>();
for (int i = offset + 1; i <= nodeCount; i++) {
UserVm vm = createKubernetesNode(publicIpAddress);
UserVm vm = createKubernetesNode(publicIpAddress, domainId, accountId);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}
startKubernetesVM(vm);
startKubernetesVM(vm, domainId, accountId);
vm = userVmDao.findById(vm.getId());
if (vm == null) {
throw new ManagementServerException(String.format("Failed to provision worker VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@ -380,12 +352,12 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
return nodes;
}
protected List<UserVm> provisionKubernetesClusterNodeVms(final long nodeCount, final String publicIpAddress) throws ManagementServerException,
protected List<UserVm> provisionKubernetesClusterNodeVms(final long nodeCount, final String publicIpAddress, final Long domainId, final Long accountId) throws ManagementServerException,
ResourceUnavailableException, InsufficientCapacityException {
return provisionKubernetesClusterNodeVms(nodeCount, 0, publicIpAddress);
return provisionKubernetesClusterNodeVms(nodeCount, 0, publicIpAddress, domainId, accountId);
}
protected UserVm createKubernetesNode(String joinIp) throws ManagementServerException,
protected UserVm createKubernetesNode(String joinIp, Long domainId, Long accountId) throws ManagementServerException,
ResourceUnavailableException, InsufficientCapacityException {
UserVm nodeVm = null;
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
@ -416,18 +388,21 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) {
keypairs.add(kubernetesCluster.getKeyPair());
}
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
if (zone.isSecurityGroupEnabled()) {
List<Long> securityGroupIds = new ArrayList<>();
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
nodeVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, securityGroupIds, owner,
hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs,
null, addrs, null, null, null, customParameterMap, null, null, null,
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
null, true, null, UserVmManager.CKS_NODE);
} else {
nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, owner,
hostName, hostName, null, null, null,
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs,
null, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
}
if (logger.isInfoEnabled()) {
logger.info(String.format("Created node VM : %s, %s in the Kubernetes cluster : %s", hostName, nodeVm.getUuid(), kubernetesCluster.getName()));
@ -908,4 +883,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
updateLoginUserDetails(null);
}
}
protected List<DedicatedResourceVO> listDedicatedHostsInDomain(Long domainId) {
return dedicatedResourceDao.listByDomainId(domainId);
}
}

6
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java
View File

@ -333,9 +333,9 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
try {
if (originalState.equals(KubernetesCluster.State.Running)) {
plan(newVmRequiredCount, zone, clusterServiceOffering);
plan(newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId());
} else {
plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering);
plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId());
}
} catch (InsufficientCapacityException e) {
logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster : %s in zone : %s, insufficient capacity", kubernetesCluster.getName(), zone.getName()));
@ -437,7 +437,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
launchPermissionDao.persist(launchPermission);
}
try {
clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress);
clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId());
updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList()));
} catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) {
logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to provision node VM in the cluster", kubernetesCluster.getName()), e);

52
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
View File

@ -24,9 +24,11 @@ import java.net.URL;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.cloudstack.api.BaseCmd;
@ -188,7 +190,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
return k8sControlNodeConfig;
}
private UserVm createKubernetesControlNode(final Network network, String serverIp) throws ManagementServerException,
private UserVm createKubernetesControlNode(final Network network, String serverIp, Long domainId, Long accountId) throws ManagementServerException,
ResourceUnavailableException, InsufficientCapacityException {
UserVm controlVm = null;
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
@ -225,18 +227,21 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
keypairs.add(kubernetesCluster.getKeyPair());
}
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
if (zone.isSecurityGroupEnabled()) {
List<Long> securityGroupIds = new ArrayList<>();
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
controlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner,
hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs,
requestedIps, addrs, null, null, null, customParameterMap, null, null, null,
requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
null, true, null, UserVmManager.CKS_NODE);
} else {
controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner,
hostName, hostName, null, null, null,
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs,
requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
}
if (logger.isInfoEnabled()) {
logger.info(String.format("Created control VM ID: %s, %s in the Kubernetes cluster : %s", controlVm.getUuid(), hostName, kubernetesCluster.getName()));
@ -276,7 +281,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
return k8sControlNodeConfig;
}
private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance) throws ManagementServerException,
private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance,
final Long domainId, final Long accountId) throws ManagementServerException,
ResourceUnavailableException, InsufficientCapacityException {
UserVm additionalControlVm = null;
DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId());
@ -307,18 +313,21 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
keypairs.add(kubernetesCluster.getKeyPair());
}
Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId);
if (zone.isSecurityGroupEnabled()) {
List<Long> securityGroupIds = new ArrayList<>();
securityGroupIds.add(kubernetesCluster.getSecurityGroupId());
additionalControlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner,
hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs,
null, addrs, null, null, null, customParameterMap, null, null, null,
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null,
null, true, null, UserVmManager.CKS_NODE);
} else {
additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner,
hostName, hostName, null, null, null,
Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs,
null, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
null, addrs, null, null, Objects.nonNull(affinityGroupId) ?
Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null);
}
if (logger.isInfoEnabled()) {
@ -327,15 +336,16 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
return additionalControlVm;
}
private UserVm provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress) throws
private UserVm provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress,
final Long domainId, final Long accountId) throws
ManagementServerException, InsufficientCapacityException, ResourceUnavailableException {
UserVm k8sControlVM = null;
k8sControlVM = createKubernetesControlNode(network, publicIpAddress);
k8sControlVM = createKubernetesControlNode(network, publicIpAddress, domainId, accountId);
addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(k8sControlVM);
}
startKubernetesVM(k8sControlVM);
startKubernetesVM(k8sControlVM, domainId, accountId);
k8sControlVM = userVmDao.findById(k8sControlVM.getId());
if (k8sControlVM == null) {
throw new ManagementServerException(String.format("Failed to provision control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@ -346,18 +356,18 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
return k8sControlVM;
}
private List<UserVm> provisionKubernetesClusterAdditionalControlVms(final String publicIpAddress) throws
private List<UserVm> provisionKubernetesClusterAdditionalControlVms(final String publicIpAddress, final Long domainId, final Long accountId) throws
InsufficientCapacityException, ManagementServerException, ResourceUnavailableException {
List<UserVm> additionalControlVms = new ArrayList<>();
if (kubernetesCluster.getControlNodeCount() > 1) {
for (int i = 1; i < kubernetesCluster.getControlNodeCount(); i++) {
UserVm vm = null;
vm = createKubernetesAdditionalControlNode(publicIpAddress, i);
vm = createKubernetesAdditionalControlNode(publicIpAddress, i, domainId, accountId);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}
startKubernetesVM(vm);
startKubernetesVM(vm, domainId, accountId);
vm = userVmDao.findById(vm.getId());
if (vm == null) {
throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName()));
@ -418,7 +428,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, true);
}
private void startKubernetesClusterVMs() {
private void startKubernetesClusterVMs(Long domainId, Long accountId) {
List <UserVm> clusterVms = getKubernetesClusterVMs();
for (final UserVm vm : clusterVms) {
if (vm == null) {
@ -426,7 +436,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
}
try {
resizeNodeVolume(vm);
startKubernetesVM(vm);
startKubernetesVM(vm, domainId, accountId);
} catch (ManagementServerException ex) {
logger.warn(String.format("Failed to start VM : %s in Kubernetes cluster : %s due to ", vm.getDisplayName(), kubernetesCluster.getName()) + ex);
// don't bail out here. proceed further to stop the reset of the VM's
@ -480,7 +490,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO);
}
public boolean startKubernetesClusterOnCreate() {
public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId) {
init();
if (logger.isInfoEnabled()) {
logger.info(String.format("Starting Kubernetes cluster : %s", kubernetesCluster.getName()));
@ -489,7 +499,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested);
DeployDestination dest = null;
try {
dest = plan();
dest = plan(domainId, accountId);
} catch (InsufficientCapacityException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
}
@ -519,7 +529,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
List<UserVm> clusterVMs = new ArrayList<>();
UserVm k8sControlVM = null;
try {
k8sControlVM = provisionKubernetesClusterControlVm(network, publicIpAddress);
k8sControlVM = provisionKubernetesClusterControlVm(network, publicIpAddress, domainId, accountId);
} catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the control VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
}
@ -532,13 +542,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
}
}
try {
List<UserVm> additionalControlVMs = provisionKubernetesClusterAdditionalControlVms(publicIpAddress);
List<UserVm> additionalControlVMs = provisionKubernetesClusterAdditionalControlVms(publicIpAddress, domainId, accountId);
clusterVMs.addAll(additionalControlVMs);
} catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning additional control VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
}
try {
List<UserVm> nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster.getNodeCount(), publicIpAddress);
List<UserVm> nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster.getNodeCount(), publicIpAddress, domainId, accountId);
clusterVMs.addAll(nodeVMs);
} catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) {
logTransitStateAndThrow(Level.ERROR, String.format("Provisioning node VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e);
@ -590,14 +600,14 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
public boolean startStoppedKubernetesCluster() throws CloudRuntimeException {
public boolean startStoppedKubernetesCluster(Long domainId, Long accountId) throws CloudRuntimeException {
init();
if (logger.isInfoEnabled()) {
logger.info(String.format("Starting Kubernetes cluster : %s", kubernetesCluster.getName()));
}
final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000;
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested);
startKubernetesClusterVMs();
startKubernetesClusterVMs(domainId, accountId);
try {
InetAddress address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost());
} catch (MalformedURLException | UnknownHostException ex) {

5
plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java
View File

@ -112,7 +112,8 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
@ACL(accessType = AccessType.UseEntry)
@Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class,
description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.")
description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. " +
"Hosts dedicated to the specified domain will be used for deploying the cluster")
private Long domainId;
@ACL(accessType = AccessType.UseEntry)
@ -321,7 +322,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd {
public void execute() {
try {
if (KubernetesCluster.ClusterType.valueOf(getClusterType()) == KubernetesCluster.ClusterType.CloudManaged
&& !kubernetesClusterService.startKubernetesCluster(getEntityId(), true)) {
&& !kubernetesClusterService.startKubernetesCluster(getEntityId(), getDomainId(), getAccountName(), true)) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start Kubernetes cluster");
}
KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getEntityId());

4
plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java
View File

@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster;
import javax.inject.Inject;
import com.cloud.user.Account;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiConstants;
@ -108,7 +109,8 @@ public class StartKubernetesClusterCmd extends BaseAsyncCmd {
public void execute() throws ServerApiException, ConcurrentOperationException {
final KubernetesCluster kubernetesCluster = validateRequest();
try {
if (!kubernetesClusterService.startKubernetesCluster(kubernetesCluster.getId(), false)) {
Account account = _accountService.getAccount(kubernetesCluster.getAccountId());
if (!kubernetesClusterService.startKubernetesCluster(kubernetesCluster.getId(), kubernetesCluster.getDomainId(), account.getAccountName(), false)) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to start Kubernetes cluster ID: %d", getId()));
}
final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId());