Fix CKS HA clusters

plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java

@@ -161,6 +161,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         final String certSans = "{{ k8s_control.server_ips }}";
         final String k8sCertificate = "{{ k8s_control.certificate_key }}";
         final String externalCniPlugin = "{{ k8s.external.cni.plugin }}";
+        final String isHaCluster = "{{ k8s.ha.cluster }}";
+        final String publicIP = "{{ k8s.public.ip }}";
 
         final List<String> addresses = new ArrayList<>();
         addresses.add(controlNodeIp);
@@ -202,7 +204,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
                     CLUSTER_API_PORT,
                     KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
         }
-        initArgs += String.format("--apiserver-cert-extra-sans=%s", controlNodeIp);
+        initArgs += String.format("--apiserver-cert-extra-sans=%s", String.join(",", addresses));
         initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion());
         k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs);
         k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
@@ -212,6 +214,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp));
         k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
         k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress);
 
         k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
 
@@ -309,6 +313,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         final String ejectIsoKey = "{{ k8s.eject.iso }}";
         final String installWaitTime = "{{ k8s.install.wait.time }}";
         final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
+        final String isHaCluster = "{{ k8s.ha.cluster }}";
+        final String publicIP = "{{ k8s.public.ip }}";
 
         final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
         final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
@@ -328,6 +334,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
         k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
         k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1));
+        k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress);
         k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
 
         return k8sControlNodeConfig;
@@ -426,7 +434,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
         String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix);
         String k8sControlNodeConfig = null;
         try {
-            k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
+            k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(publicIpAddress, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
         } catch (IOException e) {
             logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e);
         }

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml

@@ -225,6 +225,9 @@ write_files:
         exit 0
       fi
 
+      HA_CLUSTER={{ k8s.ha.cluster }}
+      CLUSTER_PUBLIC_IP={{ k8s.public.ip }}
+
       if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
         echo "setup-kube-system is running!"
         exit 1
@@ -242,6 +245,10 @@ write_files:
       cp -i /etc/kubernetes/admin.conf /root/.kube/config
       chown $(id -u):$(id -g) /root/.kube/config
 
+      if [[ "$HA_CLUSTER" == "true" ]]; then
+        sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config
+      fi
+
       sudo touch /home/cloud/success
       echo "true" > /home/cloud/success
 

plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml

@@ -276,6 +276,9 @@ write_files:
       exit 0
       fi
       
+      HA_CLUSTER={{ k8s.ha.cluster }}
+      CLUSTER_PUBLIC_IP={{ k8s.public.ip }}
+
       if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
         export PATH=$PATH:/opt/bin
       fi
@@ -320,6 +323,10 @@ write_files:
       chown $(id -u):$(id -g) /root/.kube/config
       echo export PATH=\$PATH:/opt/bin >> /root/.bashrc
       
+      if [[ "$HA_CLUSTER" == "true" ]]; then
+        sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config
+      fi
+
       if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then
         ### Network, dashboard configs available offline ###
         echo "Offline configs are available!"