diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index c31160ead84..4da80278f54 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -120,6 +120,7 @@ public class ApiConstants { public static final String ENCRYPT_FORMAT = "encryptformat"; public static final String ENCRYPT_ROOT = "encryptroot"; public static final String ENCRYPTION_SUPPORTED = "encryptionsupported"; + public static final String ETCD_IPS = "etcdips"; public static final String MIN_IOPS = "miniops"; public static final String MAX_IOPS = "maxiops"; public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve"; @@ -1055,6 +1056,7 @@ public class ApiConstants { public static final String ETCD_NODES = "etcdnodes"; public static final String EXTERNAL_NODES = "externalnodes"; public static final String IS_EXTERNAL_NODE = "isexternalnode"; + public static final String IS_ETCD_NODE = "isetcdnode"; public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java index 2285547ab77..a25d25d650f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java @@ -30,11 +30,15 @@ public class KubernetesUserVmResponse extends UserVmResponse { @Param(description = "If the VM is an externally added node") private boolean isExternalNode; - public boolean isExternalNode() { - return isExternalNode; - } + @SerializedName(ApiConstants.IS_ETCD_NODE) + @Param(description = "If the VM is an etcd node") + private boolean isEtcdNode; public void setExternalNode(boolean externalNode) { isExternalNode = externalNode; } + + public void setEtcdNode(boolean etcdNode) { + isEtcdNode = etcdNode; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 1272eb9e27c..8e2b72529df 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -42,18 +42,23 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.dc.DedicatedResourceVO; import com.cloud.dc.dao.DedicatedResourceDao; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.Host; import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.element.NsxProviderVO; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.template.TemplateApiService; import com.cloud.user.dao.AccountDao; import com.cloud.uservm.UserVm; @@ -304,6 +309,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne private UserVmService userVmService; @Inject private TemplateApiService templateService; + @Inject + private PortForwardingRulesDao pfRuleDao; private void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.WARN) { @@ -745,10 +752,23 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate zone metrics response"); } kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode()); + kubernetesUserVmResponse.setEtcdNode(vmMapVO.isEtcdNode()); vmResponses.add(kubernetesUserVmResponse); } } - response.setExternalNodes(vmList.stream().filter(KubernetesClusterVmMapVO::isEtcdNode).count()); + List etcdNodeIds = vmList.stream().filter(KubernetesClusterVmMapVO::isEtcdNode).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + List etcdIpIds = new ArrayList<>(); + Map etcdIps = new HashMap<>(); + int etcdNodeSshPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + etcdNodeIds.forEach(id -> { + etcdIpIds.addAll(pfRuleDao.listByVm(id).stream().filter(rule -> rule.getSourcePortStart() == etcdNodeSshPort) + .map(PortForwardingRuleVO::getSourceIpAddressId).collect(Collectors.toList())); + }); + etcdIpIds.forEach(id -> { + IPAddressVO ipAddress = ipAddressDao.findById(id); + etcdIps.put(ipAddress.getUuid(), ipAddress.getAddress().addr()); + }); + response.setEtcdIps(etcdIps); } response.setHasAnnotation(annotationDao.hasAnnotations(kubernetesCluster.getUuid(), AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), accountService.isRootAdmin(caller.getId()))); @@ -758,6 +778,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne response.setMaxSize(kubernetesCluster.getMaxSize()); response.setClusterType(kubernetesCluster.getClusterType()); response.setCreated(kubernetesCluster.getCreated()); + + + return response; } @@ -1554,7 +1577,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne */ @Override - public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException { + public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) + throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } @@ -2361,7 +2385,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne KubernetesControlNodeInstallAttemptWait, KubernetesControlNodeInstallReattempts, KubernetesWorkerNodeInstallAttemptWait, - KubernetesWorkerNodeInstallReattempts + KubernetesWorkerNodeInstallReattempts, + KubernetesEtcdNodeStartPort }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index e6b35f29da8..23be9edb58a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.kubernetes.cluster; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; @@ -129,6 +132,12 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Number of times the offline installation of K8S will be re-attempted", true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesEtcdNodeStartPort = new ConfigKey("Advanced", Integer.class, + "cloud.kubernetes.etcd.node.start.port", + "50000", + "Start port for Port forwarding rules for etcd nodes", + true, + KubernetesServiceEnabled.key()); KubernetesCluster findById(final Long id); @@ -136,7 +145,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException; - boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException; + boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, boolean onCreate) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException; boolean stopKubernetesCluster(StopKubernetesClusterCmd cmd) throws CloudRuntimeException; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 095ec66aba1..b916e22bde4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -133,6 +133,8 @@ public class KubernetesClusterActionWorker { public static final int CLUSTER_API_PORT = 6443; public static final int DEFAULT_SSH_PORT = 22; public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; + public static final int ETCD_NODE_CLIENT_REQUEST_PORT = 2379; + public static final int ETCD_NODE_PEER_COMM_PORT = 2380; public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = DEFAULT_SSH_PORT; public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; @@ -358,14 +360,16 @@ public class KubernetesClusterActionWorker { return new File(keyFile); } - protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode, - boolean isExternalNode, boolean markForManualUpgrade) { + protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, + boolean isControlNode, boolean isExternalNode, + boolean isEtcdNode, boolean markForManualUpgrade) { return Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId, isControlNode); newClusterVmMap.setExternalNode(isExternalNode); newClusterVmMap.setManualUpgrade(markForManualUpgrade); + newClusterVmMap.setEtcdNode(isEtcdNode); kubernetesClusterVmMapDao.persist(newClusterVmMap); return newClusterVmMap; } @@ -436,7 +440,7 @@ public class KubernetesClusterActionWorker { return publicIp; } - protected IpAddress acquireVpcTierKubernetesPublicIp(Network network) throws + protected IpAddress acquireVpcTierKubernetesPublicIp(Network network, boolean forEtcd) throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException { IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null); if (ip == null) { @@ -444,7 +448,19 @@ public class KubernetesClusterActionWorker { } ip = vpcService.associateIPToVpc(ip.getId(), network.getVpcId()); ip = ipAddressManager.associateIPToGuestNetwork(ip.getId(), network.getId(), false); - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID, ip.getUuid(), false); + if (!forEtcd) { + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID, ip.getUuid(), false); + } + return ip; + } + + protected IpAddress acquirePublicIpForIsolatedNetwork(Network network) throws + InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException { + IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null); + if (ip == null) { + return null; + } + ip = networkService.associateIPToNetwork(ip.getId(), network.getId()); return ip; } @@ -476,7 +492,7 @@ public class KubernetesClusterActionWorker { return new Pair<>(address.getAddress().addr(), port); } if (acquireNewPublicIpForVpcTierIfNeeded) { - address = acquireVpcTierKubernetesPublicIp(network); + address = acquireVpcTierKubernetesPublicIp(network, false); if (address != null) { return new Pair<>(address.getAddress().addr(), port); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java index 8abfc66e23e..df63c5a1bb0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java @@ -264,7 +264,7 @@ public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker { kubernetesClusterDao.update(clusterId, kubernetesClusterVO); kubernetesCluster = kubernetesClusterVO; - finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true, manualUpgrade)); + finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true, false, manualUpgrade)); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 37d280510d1..94e8fedce02 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -335,7 +335,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu List nodes = new ArrayList<>(); for (int i = offset + 1; i <= nodeCount; i++) { UserVm vm = createKubernetesNode(publicIpAddress, domainId, accountId); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } @@ -778,6 +778,21 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu return prefix; } + protected String getEtcdNodeNameForCluster() { + String prefix = kubernetesCluster.getName(); + if (!NetUtils.verifyDomainNameLabel(prefix, true)) { + prefix = prefix.replaceAll("[^a-zA-Z0-9-]", ""); + if (prefix.isEmpty()) { + prefix = kubernetesCluster.getUuid(); + } + } + prefix = "etcd-" + prefix; + if (prefix.length() > 40) { + prefix = prefix.substring(0, 40); + } + return prefix; + } + protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, final Long size, final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize, diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index cfe6657f1e8..d5a580e05d0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -31,6 +31,8 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.utils.Ternary; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.framework.ca.Certificate; @@ -78,6 +80,7 @@ import com.cloud.vm.VmDetailConstants; import org.apache.logging.log4j.Level; import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD; public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { @@ -133,7 +136,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } private String getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, - final String hostName, final boolean haSupported, + final List etcdIps, final String hostName, final boolean haSupported, final boolean ejectIso) throws IOException { String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; @@ -145,12 +148,20 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String ejectIsoKey = "{{ k8s.eject.iso }}"; final String installWaitTime = "{{ k8s.install.wait.time }}"; final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + final String externalEtcdNodes = "{{ etcd.unstacked_etcd }}"; + final String etcdEndpointList = "{{ etcd.etcd_endpoint_list }}"; + final String k8sServerIp = "{{ k8s_control.server_ip }}"; + final String k8sApiPort = "{{ k8s.api_server_port }}"; + final String certSans = "{{ k8s_control.server_ips }}"; + final String k8sCertificate = "{{ k8s_control.certificate_key }}"; final List addresses = new ArrayList<>(); addresses.add(controlNodeIp); if (!serverIp.equals(controlNodeIp)) { addresses.add(serverIp); } + + boolean externalEtcd = !etcdIps.isEmpty(); final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), addresses, 3650, null); @@ -159,6 +170,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value(); final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value(); + String endpointList = getEtcdEndpointList(etcdIps); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); @@ -174,6 +187,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif k8sControlNodeConfig = k8sControlNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts)); k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(externalEtcdNodes, String.valueOf(externalEtcd)); String initArgs = ""; if (haSupported) { initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", @@ -185,12 +199,18 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(etcdEndpointList, endpointList); + k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sServerIp, serverIp); + k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sApiPort, String.valueOf(CLUSTER_API_PORT)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; } - private UserVm createKubernetesControlNode(final Network network, String serverIp, Long domainId, Long accountId) throws ManagementServerException, + private UserVm createKubernetesControlNode(final Network network, String serverIp, List etcdIps, Long domainId, Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { UserVm controlVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); @@ -217,7 +237,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif boolean haSupported = isKubernetesVersionSupportsHA(); String k8sControlNodeConfig = null; try { - k8sControlNodeConfig = getKubernetesControlNodeConfig(controlNodeIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sControlNodeConfig = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e); } @@ -281,6 +301,78 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return k8sControlNodeConfig; } + private String getInitialEtcdClusterDetails(List ipAddresses, List hostnames) { + String initialCluster = "%s=http://%s:2380"; + StringBuilder clusterInfo = new StringBuilder(); + for (int i = 0; i < ipAddresses.size(); i++) { + clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i))); + if (i < ipAddresses.size()-1) { + clusterInfo.append(","); + } + } + return clusterInfo.toString(); + } + + /** + * + * @param ipAddresses list of etcd node guest IPs + * @return a formatted list of etcd endpoints adhering to YAML syntax + */ + private String getEtcdEndpointList(List ipAddresses) { + StringBuilder endpoints = new StringBuilder(); + for (int i = 0; i < ipAddresses.size(); i++) { + endpoints.append(String.format("- http://%s:2379", ipAddresses.get(i).getIp4Address())); + if (i < ipAddresses.size()-1) { + endpoints.append("\n "); + } + } + return endpoints.toString(); + } + + + private List getEtcdNodeHostnames() { + List hostnames = new ArrayList<>(); + for (int etcdNodeIndex = 0; etcdNodeIndex <= kubernetesCluster.getEtcdNodeCount(); etcdNodeIndex++) { + String suffix = Long.toHexString(System.currentTimeMillis()); + hostnames.add(String.format("%s-%s-%s", getEtcdNodeNameForCluster(), etcdNodeIndex, suffix)); + } + return hostnames; + } + + private String getEtcdNodeConfig(final List ipAddresses, final List hostnames, final int etcdNodeIndex, + final boolean ejectIso) throws IOException { + String k8sEtcdNodeConfig = readResourceFile("/conf/etcd-node.yml"); + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; + final String installWaitTime = "{{ k8s.install.wait.time }}"; + final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + final String etcdNodeName = "{{ etcd.node_name }}"; + final String etcdNodeIp = "{{ etcd.node_ip }}"; + final String etcdInitialClusterNodes = "{{ etcd.initial_cluster_nodes }}"; + + final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value(); + final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value(); + String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (StringUtils.isNotEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + String initialClusterDetails = getInitialEtcdClusterDetails(ipAddresses, hostnames); + + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(installWaitTime, String.valueOf(waitTime)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(sshPubKey, pubKey); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdNodeName, hostnames.get(etcdNodeIndex)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdNodeIp, ipAddresses.get(etcdNodeIndex)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdInitialClusterNodes, initialClusterDetails); + + return k8sEtcdNodeConfig; + } + private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance, final Long domainId, final Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { @@ -336,12 +428,56 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return additionalControlVm; } - private UserVm provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress, + private UserVm createEtcdNode(List requestedIps, List etcdNodeHostnames, int etcdNodeIndex, Long domainId, Long accountId) throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException { + UserVm etcdNode = null; + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(ETCD, kubernetesCluster); + List networkIds = Collections.singletonList(kubernetesCluster.getNetworkId()); + Network.IpAddresses addrs = new Network.IpAddresses(null, null); + List guestIps = requestedIps.stream().map(Network.IpAddresses::getIp4Address).collect(Collectors.toList()); + String k8sControlNodeConfig = null; + try { + k8sControlNodeConfig = getEtcdNodeConfig(guestIps, etcdNodeHostnames, etcdNodeIndex, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + } catch (IOException e) { + logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e); + } + + String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + List keypairs = new ArrayList(); + if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) { + keypairs.add(kubernetesCluster.getKeyPair()); + } + Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); + String hostName = etcdNodeHostnames.get(etcdNodeIndex); + Map customParameterMap = new HashMap(); + if (zone.isSecurityGroupEnabled()) { + List securityGroupIds = new ArrayList<>(); + securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); + etcdNode = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, securityGroupIds, owner, + hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, + Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, + null, true, null, null); + } else { + etcdNode = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, owner, + hostName, hostName, null, null, null, + Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, + Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); + } + + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Created control VM ID : %s, %s in the Kubernetes cluster : %s", etcdNode.getUuid(), hostName, kubernetesCluster.getName())); + } + return etcdNode; + } + + private UserVm provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress, final List etcdIps, final Long domainId, final Long accountId) throws ManagementServerException, InsufficientCapacityException, ResourceUnavailableException { UserVm k8sControlVM = null; - k8sControlVM = createKubernetesControlNode(network, publicIpAddress, domainId, accountId); - addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false); + k8sControlVM = createKubernetesControlNode(network, publicIpAddress, etcdIps, domainId, accountId); + addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(k8sControlVM); } @@ -356,14 +492,15 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return k8sControlVM; } - private List provisionKubernetesClusterAdditionalControlVms(final String publicIpAddress, final Long domainId, final Long accountId) throws + private List provisionKubernetesClusterAdditionalControlVms(final String publicIpAddress, final Long domainId, + final Long accountId) throws InsufficientCapacityException, ManagementServerException, ResourceUnavailableException { List additionalControlVms = new ArrayList<>(); if (kubernetesCluster.getControlNodeCount() > 1) { for (int i = 1; i < kubernetesCluster.getControlNodeCount(); i++) { UserVm vm = null; vm = createKubernetesAdditionalControlNode(publicIpAddress, i, domainId, accountId); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } @@ -381,6 +518,55 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return additionalControlVms; } + private Ternary, List, List> provisionEtcdCluster(final Network network, final Long domainId, final Long accountId) + throws InsufficientCapacityException, ResourceUnavailableException, ManagementServerException { + List etcdNodeVms = new ArrayList<>(); + List etcdNodeIps = getEtcdNodePublicIpAddresses(network, kubernetesCluster.getEtcdNodeCount()); + List etcdNodeGuestIps = getEtcdNodeGuestIps(network, kubernetesCluster.getEtcdNodeCount()); + List etcdHostnames = getEtcdNodeHostnames(); + for (int i = 0; i < kubernetesCluster.getEtcdNodeCount(); i++) { + IpAddress ip = etcdNodeIps.get(i); + if (Objects.isNull(ip)) { + String errMsg = String.format("No public IP found for the network: %s, to create Etcd node for " + + "Kubernetes cluster: %s", network, kubernetesCluster.getName()); + LOGGER.error(errMsg); + logAndThrow(Level.ERROR, errMsg); + } + UserVm vm = createEtcdNode(etcdNodeGuestIps, etcdHostnames, i, domainId, accountId); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, true, true); + startKubernetesVM(vm, domainId, accountId); + vm = userVmDao.findById(vm.getId()); + if (vm == null) { + throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName())); + } + etcdNodeVms.add(vm); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned additional control VM : %s in to the Kubernetes cluster : %s", vm.getDisplayName(), kubernetesCluster.getName())); + } + } + return new Ternary<>(etcdNodeVms, etcdNodeGuestIps, etcdNodeIps); + } + + private List getEtcdNodePublicIpAddresses(final Network network, final long etcdNodeCount) throws InsufficientAddressCapacityException, ResourceUnavailableException, ResourceAllocationException { + List ipAddresses = new ArrayList<>(); + for (int i = 1; i <= etcdNodeCount; i++) { + if (network.getVpcId() == null) { + ipAddresses.add(acquirePublicIpForIsolatedNetwork(network)); + } else { + ipAddresses.add(acquireVpcTierKubernetesPublicIp(network, true)); + } + } + return ipAddresses; + } + + private List getEtcdNodeGuestIps(final Network network, final long etcdNodeCount) { + List guestIps = new ArrayList<>(); + for (int i = 1; i <= etcdNodeCount; i++) { + guestIps.add(new Network.IpAddresses(ipAddressManager.acquireGuestIpAddress(network, null), null)); + } + return guestIps; + } + private Network startKubernetesClusterNetwork(final DeployDestination destination) throws ManagementServerException { final ReservationContext context = new ReservationContextImpl(null, null, null, owner); Network network = networkDao.findById(kubernetesCluster.getNetworkId()); @@ -428,6 +614,29 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, true); } + protected void setupKubernetesEtcdNetworkRules(List etcdNodeIps, List etcdVms, Network network) throws ManagementServerException, ResourceUnavailableException { + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); + } + } + List etcdVmIds = etcdVms.stream().map(UserVm::getId).collect(Collectors.toList()); + Integer startPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + for (int i = 0; i < etcdVmIds.size(); i++) { + IpAddress publicIp = etcdNodeIps.get(i); + try { + provisionFirewallRules(publicIp, owner, startPort, startPort); + provisionFirewallRules(publicIp, owner, ETCD_NODE_CLIENT_REQUEST_PORT, ETCD_NODE_PEER_COMM_PORT); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | + NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), startPort, DEFAULT_SSH_PORT); + provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), ETCD_NODE_CLIENT_REQUEST_PORT, ETCD_NODE_CLIENT_REQUEST_PORT); + provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), ETCD_NODE_PEER_COMM_PORT, ETCD_NODE_PEER_COMM_PORT); + } + } + private void startKubernetesClusterVMs(Long domainId, Long accountId) { List clusterVms = getKubernetesClusterVMs(); for (final UserVm vm : clusterVms) { @@ -490,7 +699,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); } - public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId) { + public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { init(); if (logger.isInfoEnabled()) { logger.info(String.format("Starting Kubernetes cluster : %s", kubernetesCluster.getName())); @@ -526,10 +735,20 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif launchPermissionDao.persist(launchPermission); } + List etcdVms = new ArrayList<>(); + List etcdGuestNodeIps = new ArrayList<>(); + List etcdPublicNodeIps = new ArrayList<>(); + if (kubernetesCluster.getEtcdNodeCount() > 0) { + Ternary, List, List> etcdNodesAndIps = provisionEtcdCluster(network, domainId, accountId); + etcdVms = etcdNodesAndIps.first(); + etcdGuestNodeIps = etcdNodesAndIps.second(); + etcdPublicNodeIps = etcdNodesAndIps.third(); + } + List clusterVMs = new ArrayList<>(); UserVm k8sControlVM = null; try { - k8sControlVM = provisionKubernetesClusterControlVm(network, publicIpAddress, domainId, accountId); + k8sControlVM = provisionKubernetesClusterControlVm(network, publicIpAddress, etcdGuestNodeIps, domainId, accountId); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the control VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } @@ -561,6 +780,12 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } catch (ManagementServerException e) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s, unable to setup network rules", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } + try { + setupKubernetesEtcdNetworkRules(etcdPublicNodeIps, etcdVms, network); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s, unable to setup network rules for etcd nodes", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + attachIsoKubernetesVMs(etcdVms); attachIsoKubernetesVMs(clusterVMs); if (!KubernetesClusterUtil.isKubernetesClusterControlVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) { String msg = String.format("Failed to setup Kubernetes cluster : %s in usable state as unable to access control node VMs of the cluster", kubernetesCluster.getName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 77f785d2747..3ed8b3eb68a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -217,8 +217,8 @@ public class KubernetesClusterUtil { user, sshKeyFile, null, "sudo /opt/bin/kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", 10000, 10000, 20000); - if (result.first()) { - return Integer.parseInt(result.second().trim().replace("\"", "")); + if (Boolean.TRUE.equals(result.first())) { + return Integer.parseInt(result.second().trim().replace("\"", "")) + kubernetesCluster.getEtcdNodeCount().intValue(); } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster : %s. Output: %s", kubernetesCluster.getName(), result.second())); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 56e130fd25a..f3e557579f2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -21,7 +21,10 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesClusterHelper; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -328,7 +331,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getEntityId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (CloudRuntimeException e) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index 9c83c3c83e0..f615ca4440f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -18,6 +18,9 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.Account; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; @@ -116,7 +119,8 @@ public class StartKubernetesClusterCmd extends BaseAsyncCmd { final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (CloudRuntimeException ex) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | + InsufficientCapacityException ex) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 48f5bab471b..63c7bb5b136 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.response; import java.util.Date; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponseWithAnnotations; @@ -86,10 +87,6 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "the number of the etcd nodes on the Kubernetes cluster") private Long etcdNodes; - @SerializedName(ApiConstants.EXTERNAL_NODES) - @Param(description = "the number of the externally added worker nodes to the Kubernetes cluster") - private Long externalNodes; - @SerializedName(ApiConstants.TEMPLATE_ID) @Param(description = "the ID of the template of the Kubernetes cluster") private String templateId; @@ -179,6 +176,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "Public IP Address ID of the cluster") private String ipAddressId; + @SerializedName(ApiConstants.ETCD_IPS) + @Param(description = "Public IP Addresses of the etcd nodes") + private Map etcdIps; + @SerializedName(ApiConstants.AUTOSCALING_ENABLED) @Param(description = "Whether autoscaling is enabled for the cluster") private boolean isAutoscalingEnabled; @@ -447,14 +448,6 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple this.etcdNodes = etcdNodes; } - public Long getExternalNodes() { - return externalNodes; - } - - public void setExternalNodes(Long externalNodes) { - this.externalNodes = externalNodes; - } - public void setVirtualMachines(List virtualMachines) { this.virtualMachines = virtualMachines; } @@ -471,6 +464,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple this.ipAddressId = ipAddressId; } + public void setEtcdIps(Map etcdIps) { + this.etcdIps = etcdIps; + } + public void setAutoscalingEnabled(boolean isAutoscalingEnabled) { this.isAutoscalingEnabled = isAutoscalingEnabled; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml new file mode 100644 index 00000000000..e5ec6ab70a9 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml @@ -0,0 +1,134 @@ +#cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +users: + - name: cloud + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + {{ k8s.ssh.pub.key }} + +write_files: + - path: /opt/bin/setup-etcd-node + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + ISO_MOUNT_DIR=/mnt/etcddisk + BINARIES_DIR=${ISO_MOUNT_DIR}/ + ATTEMPT_ONLINE_INSTALL=false + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }} + MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }} + if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + fi + if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then + MAX_OFFLINE_INSTALL_ATTEMPTS=100 + fi + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} + crucial_cmd_attempts=1 + iso_drive_path="" + while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] + done + + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/bin/ + tar -zxf ${BINARIES_DIR}/etcd/etcd-linux-amd64.tar.gz -C /opt/bin/ + mv /opt/bin/etcd*/etcd* /opt/bin/ + sudo rm -rf /opt/bin/etcd-* + fi + + - path: /etc/systemd/system/etcd.service + permissions: '0755' + owner: root:root + content: | + [Unit] + Description=etcd + + [Service] + Type=exec + ExecStart=/opt/bin/etcd \ + --name {{ etcd.node_name }} \ + --initial-advertise-peer-urls http://{{ etcd.node_ip }}:2380 \ + --listen-peer-urls http://{{ etcd.node_ip }}:2380 \ + --advertise-client-urls http://{{ etcd.node_ip }}:2379 \ + --listen-client-urls http://{{ etcd.node_ip }}:2379,http://127.0.0.1:2379 \ + --initial-cluster-token etcd-cluster-1 \ + --initial-cluster {{ etcd.initial_cluster_nodes }} \ + --initial-cluster-state new + Restart=on-failure + RestartSec=5 + + [Install] + WantedBy=multi-user.target + +runcmd: + - chown -R cloud:cloud /home/cloud/.ssh + - /opt/bin/setup-etcd-node + - systemctl daemon-reload + - systemctl enable --now etcd \ No newline at end of file diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 3154fb20251..26a09ee26ae 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -225,6 +225,33 @@ write_files: done fi + - path: /etc/kubernetes/kubeadm-config.yaml + permissions: '0644' + owner: root:root + content: | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + apiServer: + certSANs: + {{ k8s_control.server_ips }} + controlPlaneEndpoint: {{ k8s_control.server_ip }}:{{ k8s.api_server_port }} + etcd: + external: + endpoints: + {{ etcd.etcd_endpoint_list }} + --- + apiVersion: kubeadm.k8s.io/v1beta3 + kind: InitConfiguration + bootstrapTokens: + - token: "{{ k8s_control_node.cluster.token }}" + nodeRegistration: + criSocket: /run/containerd/containerd.sock + localAPIEndpoint: + advertiseAddress: {{ k8s_control.server_ip }} + bindPort: {{ k8s.api_server_port }} + certificateKey: {{ k8s_control.certificate_key }} + + - path: /opt/bin/deploy-kube-system permissions: '0700' owner: root:root @@ -240,6 +267,7 @@ write_files: export PATH=$PATH:/opt/bin fi + EXTERNAL_ETCD_NODES={{ etcd.unstacked_etcd }} MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 crucial_cmd_attempts=1 while true; do @@ -249,7 +277,11 @@ write_files: fi retval=0 set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock + if [[ ${EXTERNAL_ETCD_NODES} == true ]]; then + kubeadm init --config /etc/kubernetes/kubeadm-config.yaml --upload-certs + else + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock + fi retval=$? set -e if [ $retval -eq 0 ]; then diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index d5fb014f220..265c0ce5ce2 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -19,8 +19,8 @@ set -e if [ $# -lt 6 ]; then - echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG BUILD_NAME" - echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml setup-v1.11.4" + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG ETCD_VER BUILD_NAME" + echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml 3.5.1 setup-v1.11.4" exit 1 fi @@ -31,7 +31,7 @@ start_dir="$PWD" iso_dir="/tmp/iso" working_dir="${iso_dir}/" mkdir -p "${working_dir}" -build_name="${7}.iso" +build_name="${8}.iso" [ -z "${build_name}" ] && build_name="setup-${RELEASE}.iso" CNI_VERSION="v${3}" @@ -148,6 +148,12 @@ chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" echo "Updating imagePullPolicy to IfNotPresent in yaml files..." sed -i "s/imagePullPolicy:.*/imagePullPolicy: IfNotPresent/g" ${working_dir}/*.yaml +# Install etcd dependencies +etcd_dir="${working_dir}/etcd" +mkdir -p "${etcd_dir}" +ETCD_VER=v${7} +wget -q --show-progress "https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz" -O ${etcd_dir}/etcd-linux-amd64.tar.gz + mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}" rm -rf "${iso_dir}" diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 65b44209b38..803565313ef 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -2443,6 +2443,7 @@ "label.quotagb": "Quota in GB", "label.encryption": "Encryption", "label.etcdnodes": "Number of etcd nodes", +"label.etcd.ips": "etcd Node(s) IP address(es)", "label.versioning": "Versioning", "label.objectlocking": "Object Lock", "label.bucket.policy": "Bucket Policy", diff --git a/ui/src/components/view/InfoCard.vue b/ui/src/components/view/InfoCard.vue index 0bd20cd4409..5d34fad586c 100644 --- a/ui/src/components/view/InfoCard.vue +++ b/ui/src/components/view/InfoCard.vue @@ -188,6 +188,24 @@ +
+
{{ $t('label.etcd.ips') }}
+
+
+
+ + +   + +
+
+
+
{{ $t('label.cpu') }}
diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index 4f096634340..15bcb8ba2cc 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -114,7 +114,8 @@