diff --git a/api/src/main/java/com/cloud/network/vpc/Vpc.java b/api/src/main/java/com/cloud/network/vpc/Vpc.java index e9a831c9d83..b94089d2d43 100644 --- a/api/src/main/java/com/cloud/network/vpc/Vpc.java +++ b/api/src/main/java/com/cloud/network/vpc/Vpc.java @@ -105,4 +105,6 @@ public interface Vpc extends ControlledEntity, Identity, InternalIdentity { String getIp6Dns1(); String getIp6Dns2(); + + boolean useRouterIpAsResolver(); } diff --git a/api/src/main/java/com/cloud/network/vpc/VpcService.java b/api/src/main/java/com/cloud/network/vpc/VpcService.java index af2a9847a62..9b7a83c29c1 100644 --- a/api/src/main/java/com/cloud/network/vpc/VpcService.java +++ b/api/src/main/java/com/cloud/network/vpc/VpcService.java @@ -48,17 +48,17 @@ public interface VpcService { * @param vpcName * @param displayText * @param cidr - * @param networkDomain TODO + * @param networkDomain TODO * @param ip4Dns1 * @param ip4Dns2 - * @param displayVpc TODO + * @param displayVpc TODO + * @param useVrIpResolver * @return * @throws ResourceAllocationException TODO */ Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize, - Long asNumber, List bgpPeerIds) - throws ResourceAllocationException; + Long asNumber, List bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException; /** * Persists VPC record in the database diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index d7038f33c4b..302a08d692c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -549,6 +549,7 @@ public class ApiConstants { public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; public static final String USER_SECRET_KEY = "usersecretkey"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; + public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver"; public static final String UPDATE_IN_SEQUENCE = "updateinsequence"; public static final String VALUE = "value"; public static final String VIRTUAL_MACHINE_ID = "virtualmachineid"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java index 2f62d0d7210..08b7b9911b1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.vpc; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.cloudstack.acl.RoleType; @@ -125,6 +126,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, since = "4.20.0", description="the AS Number of the VPC tiers") private Long asNumber; + @Parameter(name=ApiConstants.USE_VIRTUAL_ROUTER_IP_RESOLVER, type=CommandType.BOOLEAN, + description="(optional) for NSX based VPCs: when set to true, use the VR IP as nameserver, otherwise use DNS1 and DNS2") + private Boolean useVrIpResolver; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -205,6 +210,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { return asNumber; } + public boolean getUseVrIpResolver() { + return BooleanUtils.toBoolean(useVrIpResolver); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/debian/rules b/debian/rules index e7ff6759d44..d178afa6730 100755 --- a/debian/rules +++ b/debian/rules @@ -70,6 +70,7 @@ override_dh_auto_install: mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/lib mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/setup mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm + mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf mkdir $(DESTDIR)/var/log/$(PACKAGE)/management mkdir $(DESTDIR)/var/cache/$(PACKAGE)/management mkdir $(DESTDIR)/var/log/$(PACKAGE)/ipallocator @@ -83,6 +84,7 @@ override_dh_auto_install: cp client/target/cloud-client-ui-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/cloudstack-$(VERSION).jar cp client/target/lib/*jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/ cp -r engine/schema/dist/systemvm-templates/* $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/ + cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/ rm -rf $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/md5sum.txt # Bundle cmk in cloudstack-management @@ -95,6 +97,12 @@ override_dh_auto_install: chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE) install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses + + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/etcd-node.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node-add.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-node.yml + # Remove configuration in /ur/share/cloudstack-management/webapps/client/WEB-INF # This should all be in /etc/cloudstack/management ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java index e8ccc2ebcf1..e942eadb8ff 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java @@ -105,6 +105,9 @@ public class VpcVO implements Vpc { @Column(name = "ip6Dns2") String ip6Dns2; + @Column(name = "use_router_ip_resolver") + boolean useRouterIpResolver = false; + @Transient boolean rollingRestart = false; @@ -309,4 +312,13 @@ public class VpcVO implements Vpc { public String getIp6Dns2() { return ip6Dns2; } + + @Override + public boolean useRouterIpAsResolver() { + return useRouterIpResolver; + } + + public void setUseRouterIpResolver(boolean useRouterIpResolver) { + this.useRouterIpResolver = useRouterIpResolver; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index ec20cb3fdd2..fe746c9884a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -68,6 +68,9 @@ ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_templa -- Add for_cks column to the user_data table to represent CNI Configuration stored as userdata CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_data','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the userdata represent CNI configuration meant for CKS use only"'); + +-- Add use VR IP as resolver option on VPC +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc','use_router_ip_resolver', 'tinyint(1) DEFAULT 0 COMMENT "use router ip as resolver instead of dns options"'); ----------------------------------------------------------- -- END - CKS Enhancements ----------------------------------------------------------- diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 22fede6fb85..2c6898cac7c 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -248,6 +248,7 @@ cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d @@ -273,7 +274,7 @@ wget https://github.com/apache/cloudstack-cloudmonkey/releases/download/$CMK_REL chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup - +cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/ @@ -308,6 +309,11 @@ touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml + # SystemVM template mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm @@ -608,6 +614,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %attr(0755,root,root) %{_bindir}/%{name}-sysvmadm %attr(0755,root,root) %{_bindir}/%{name}-setup-encryption %attr(0755,root,root) %{_bindir}/cmk +%{_datadir}/%{name}-management/cks/conf/*.yml %{_datadir}/%{name}-management/setup/*.sql %{_datadir}/%{name}-management/setup/*.sh %{_datadir}/%{name}-management/setup/server-setup.xml diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index bc5af08a9c1..864c68c96e7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -21,6 +21,9 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -143,6 +146,7 @@ public class KubernetesClusterActionWorker { public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes"; + public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks"; protected Logger logger = LogManager.getLogger(getClass()); @@ -264,6 +268,11 @@ public class KubernetesClusterActionWorker { return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), com.cloud.utils.StringUtils.getPreferredCharset()); } + protected String readK8sConfigFile(String resource) throws IOException { + Path path = Paths.get(String.format("%s%s", CKS_CONFIG_PATH, resource)); + return Files.readString(path); + } + protected String getControlNodeLoginUser() { List vmMapVOList = getKubernetesClusterVMMaps(); if (!vmMapVOList.isEmpty()) { @@ -316,7 +325,7 @@ public class KubernetesClusterActionWorker { } protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, - final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { logMessage(logLevel, message, e); stateTransitTo(kubernetesCluster.getId(), event); detachIsoKubernetesVMs(clusterVMs); @@ -670,14 +679,14 @@ public class KubernetesClusterActionWorker { try { String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'", - scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); + scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); Account account = accountDao.findById(kubernetesCluster.getAccountId()); if (account != null && account.getType() == Account.Type.PROJECT) { String projectId = projectService.findByProjectAccountId(account.getId()).getUuid(); command = String.format("%s -p '%s'", command, projectId); } Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); return result.first(); } catch (Exception e) { String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName()); @@ -696,7 +705,7 @@ public class KubernetesClusterActionWorker { writer.close(); } catch (IOException e) { logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to fetch script %s", - kubernetesCluster.getName(), filename), e); + kubernetesCluster.getName(), filename), e); } return file; } @@ -719,11 +728,11 @@ public class KubernetesClusterActionWorker { sshKeyFile = getManagementServerSshPublicKeyFile(); } SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000); + "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000); // Ensure destination dir scriptPath exists and copy file to destination String cmdStr = String.format("sudo mkdir -p %s ; sudo mv ~/%s %s/%s", scriptPath, file.getName(), scriptPath, destination); SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - cmdStr, 10000, 10000, 10 * 60 * 1000); + cmdStr, 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { throw new CloudRuntimeException(e); } @@ -771,7 +780,7 @@ public class KubernetesClusterActionWorker { // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it if (manager.isDirectAccess(network)) { logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network", - kubernetesCluster.getName()), null); + kubernetesCluster.getName()), null); return true; } File pkFile = getManagementServerSshPublicKeyFile(); @@ -782,7 +791,7 @@ public class KubernetesClusterActionWorker { try { String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename); Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); // Maybe the file isn't present. Try and copy it if (!result.first()) { @@ -792,12 +801,12 @@ public class KubernetesClusterActionWorker { if (!createCloudStackSecret(keys)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", - kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } // If at first you don't succeed ... result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); } @@ -871,7 +880,7 @@ public class KubernetesClusterActionWorker { } public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso, final boolean mountCksIsoOnVR) throws IOException { - String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); + String k8sNodeConfig = readK8sConfigFile("/conf/k8s-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String joinIpKey = "{{ k8s_control_node.join_ip }}"; final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java index c76609686a6..07d062e23a1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java @@ -41,6 +41,7 @@ import javax.inject.Inject; import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -95,7 +96,7 @@ public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker continue; } try { - removeNodeVmFromCluster(nodeId, vm.getDisplayName(), publicIp.getAddress().addr()); + removeNodeVmFromCluster(nodeId, vm.getDisplayName().toLowerCase(Locale.ROOT), publicIp.getAddress().addr()); result &= removeNodePortForwardingRules(nodeId, network, vm); if (System.currentTimeMillis() > removeNodeTimeoutTime) { logger.error(String.format("Removal of node %s from Kubernetes cluster %s timed out", vm.getName(), kubernetesCluster.getName())); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 5b0214f2095..69d493e85fe 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -142,9 +142,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } private Pair getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, - final List etcdIps, final String hostName, final boolean haSupported, - final boolean ejectIso, final boolean externalCni) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); + final List etcdIps, final String hostName, final boolean haSupported, + final boolean ejectIso, final boolean externalCni) throws IOException { + String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml"); final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; final String caCert = "{{ k8s_control_node.ca.crt }}"; @@ -161,6 +161,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String certSans = "{{ k8s_control.server_ips }}"; final String k8sCertificate = "{{ k8s_control.certificate_key }}"; final String externalCniPlugin = "{{ k8s.external.cni.plugin }}"; + final String isHaCluster = "{{ k8s.ha.cluster }}"; + final String publicIP = "{{ k8s.public.ip }}"; final List addresses = new ArrayList<>(); addresses.add(controlNodeIp); @@ -170,7 +172,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif boolean externalEtcd = !etcdIps.isEmpty(); final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", - "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), + "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), addresses, 3650, null); final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); @@ -202,7 +204,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif CLUSTER_API_PORT, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); } - initArgs += String.format("--apiserver-cert-extra-sans=%s", controlNodeIp); + initArgs += String.format("--apiserver-cert-extra-sans=%s", String.join(",", addresses)); initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); @@ -212,6 +214,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp)); k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); @@ -301,7 +305,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml"); + String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node-add.yml"); final String joinIpKey = "{{ k8s_control_node.join_ip }}"; final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; final String sshPubKey = "{{ k8s.ssh.pub.key }}"; @@ -309,6 +313,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String ejectIsoKey = "{{ k8s.eject.iso }}"; final String installWaitTime = "{{ k8s.install.wait.time }}"; final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + final String isHaCluster = "{{ k8s.ha.cluster }}"; + final String publicIP = "{{ k8s.public.ip }}"; final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value(); final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value(); @@ -328,6 +334,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress); k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); return k8sControlNodeConfig; @@ -336,13 +344,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif private String getInitialEtcdClusterDetails(List ipAddresses, List hostnames) { String initialCluster = "%s=http://%s:%s"; StringBuilder clusterInfo = new StringBuilder(); - for (int i = 0; i < ipAddresses.size(); i++) { - clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT)); - if (i < ipAddresses.size()-1) { - clusterInfo.append(","); - } + for (int i = 0; i < ipAddresses.size(); i++) { + clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT)); + if (i < ipAddresses.size()-1) { + clusterInfo.append(","); } - return clusterInfo.toString(); + } + return clusterInfo.toString(); } /** @@ -373,7 +381,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif private String getEtcdNodeConfig(final List ipAddresses, final List hostnames, final int etcdNodeIndex, final boolean ejectIso) throws IOException { - String k8sEtcdNodeConfig = readResourceFile("/conf/etcd-node.yml"); + String k8sEtcdNodeConfig = readK8sConfigFile("/conf/etcd-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String ejectIsoKey = "{{ k8s.eject.iso }}"; final String installWaitTime = "{{ k8s.install.wait.time }}"; @@ -426,7 +434,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sControlNodeConfig = null; try { - k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(publicIpAddress, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e); } @@ -576,7 +584,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } private List getEtcdNodeGuestIps(final Network network, final long etcdNodeCount) { - List guestIps = new ArrayList<>(); + List guestIps = new ArrayList<>(); for (int i = 1; i <= etcdNodeCount; i++) { guestIps.add(new Network.IpAddresses(ipAddressManager.acquireGuestIpAddress(network, null), null)); } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 38f217f403c..e1d021dd078 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -225,6 +225,9 @@ write_files: exit 0 fi + HA_CLUSTER={{ k8s.ha.cluster }} + CLUSTER_PUBLIC_IP={{ k8s.public.ip }} + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then echo "setup-kube-system is running!" exit 1 @@ -242,6 +245,10 @@ write_files: cp -i /etc/kubernetes/admin.conf /root/.kube/config chown $(id -u):$(id -g) /root/.kube/config + if [[ "$HA_CLUSTER" == "true" ]]; then + sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config + fi + sudo touch /home/cloud/success echo "true" > /home/cloud/success diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index dc066e10d06..d76dbe3f1f9 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -275,6 +275,9 @@ write_files: echo "Already provisioned!" exit 0 fi + + HA_CLUSTER={{ k8s.ha.cluster }} + CLUSTER_PUBLIC_IP={{ k8s.public.ip }} if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then export PATH=$PATH:/opt/bin @@ -319,6 +322,10 @@ write_files: cp -i /etc/kubernetes/admin.conf /root/.kube/config chown $(id -u):$(id -g) /root/.kube/config echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + + if [[ "$HA_CLUSTER" == "true" ]]; then + sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config + fi if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then ### Network, dashboard configs available offline ### diff --git a/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java b/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java index 91e4fddb69c..450f08c46e3 100644 --- a/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java @@ -301,7 +301,7 @@ public class NetworkMigrationManagerImpl implements NetworkMigrationManager { copyOfVpc = _vpcService.createVpc(vpc.getZoneId(), vpcOfferingId, vpc.getAccountId(), vpc.getName(), vpc.getDisplayText(), vpc.getCidr(), vpc.getNetworkDomain(), vpc.getIp4Dns1(), vpc.getIp4Dns2(), - vpc.getIp6Dns1(), vpc.getIp6Dns2(), vpc.isDisplay(), vpc.getPublicMtu(), null, null, null); + vpc.getIp6Dns1(), vpc.getIp6Dns2(), vpc.isDisplay(), vpc.getPublicMtu(), null, null, null, vpc.useRouterIpAsResolver()); copyOfVpcId = copyOfVpc.getId(); //on resume of migration the uuid will be swapped already. So the copy will have the value of the original vpcid. diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index e171b68399b..27f04234c33 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -2072,6 +2072,7 @@ Configurable, StateListener bgpPeerIds) throws ResourceAllocationException { + final Integer cidrSize, final Long asNumber, final List bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException { final Account caller = CallContext.current().getCallingAccount(); final Account owner = _accountMgr.getAccount(vpcOwnerId); @@ -1247,6 +1247,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis vpcOff.isRedundantRouter(), ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2); vpc.setPublicMtu(publicMtu); vpc.setDisplay(Boolean.TRUE.equals(displayVpc)); + vpc.setUseRouterIpResolver(Boolean.TRUE.equals(useVrIpResolver)); if (vpc.getCidr() == null && cidrSize != null) { // Allocate a CIDR for VPC @@ -1305,7 +1306,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis List bgpPeerIds = (cmd instanceof CreateVPCCmdByAdmin) ? ((CreateVPCCmdByAdmin)cmd).getBgpPeerIds() : null; Vpc vpc = createVpc(cmd.getZoneId(), cmd.getVpcOffering(), cmd.getEntityOwnerId(), cmd.getVpcName(), cmd.getDisplayText(), cmd.getCidr(), cmd.getNetworkDomain(), cmd.getIp4Dns1(), cmd.getIp4Dns2(), cmd.getIp6Dns1(), - cmd.getIp6Dns2(), cmd.isDisplay(), cmd.getPublicMtu(), cmd.getCidrSize(), cmd.getAsNumber(), bgpPeerIds); + cmd.getIp6Dns2(), cmd.isDisplay(), cmd.getPublicMtu(), cmd.getCidrSize(), cmd.getAsNumber(), bgpPeerIds, cmd.getUseVrIpResolver()); String sourceNatIP = cmd.getSourceNatIP(); boolean forNsx = isVpcForNsx(vpc); diff --git a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java index ee56a092dd1..8d513619805 100644 --- a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java @@ -492,7 +492,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], null, null, null, true, 1500, null, null, null); + ip4Dns[0], null, null, null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -504,7 +504,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], ip4Dns[1], ip6Dns[0], null, true, 1500, null, null, null); + ip4Dns[0], ip4Dns[1], ip6Dns[0], null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -519,7 +519,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null); + ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -536,7 +536,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null); + ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -559,7 +559,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, null, vpcDomain, - ip4Dns[0], ip4Dns[1], null, null, true, 1500, 24, null, bgpPeerIds); + ip4Dns[0], ip4Dns[1], null, null, true, 1500, 24, null, bgpPeerIds, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } diff --git a/systemvm/debian/opt/cloud/bin/cs/CsConfig.py b/systemvm/debian/opt/cloud/bin/cs/CsConfig.py index a17f6ac4aa5..1ad7e34db57 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsConfig.py @@ -114,6 +114,9 @@ class CsConfig(object): def expose_dns(self): return self.cmdline().idata().get('exposedns', 'false') == 'true' + def use_router_ip_as_resolver(self): + return self.cl.get_use_router_ip_as_resolver() + def get_dns(self): conf = self.cmdline().idata() dns = [] @@ -123,9 +126,10 @@ class CsConfig(object): else: dns.append(self.address().get_guest_ip()) - for name in ('dns1', 'dns2'): - if name in conf: - dns.append(conf[name]) + if not 'userouteripresolver' in conf: + for name in ('dns1', 'dns2'): + if name in conf: + dns.append(conf[name]) return dns def get_format(self): diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py b/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py index abbf23b4cdf..4c6b61240b8 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py @@ -176,6 +176,11 @@ class CsCmdLine(CsDataBag): return self.idata()['useextdns'] return False + def get_use_router_ip_as_resolver(self): + if "userouteripresolver" in self.idata(): + return self.idata()['userouteripresolver'] + return False + def get_advert_int(self): if 'advert_int' in self.idata(): return self.idata()['advert_int'] diff --git a/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py index fe8737208c4..65200fb8796 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py @@ -38,6 +38,9 @@ class CsGuestNetwork: if not self.guest: return self.config.get_dns() + if self.config.use_router_ip_as_resolver(): + return [self.data['router_guest_ip']] + dns = [] if 'router_guest_gateway' in self.data and not self.config.use_extdns() and ('is_vr_guest_gateway' not in self.data or not self.data['is_vr_guest_gateway']): dns.append(self.data['router_guest_gateway']) diff --git a/tools/appliance/cks/ubuntu/22.04/cks-ubuntu-2204.json b/tools/appliance/cks/ubuntu/22.04/cks-ubuntu-2204.json index c7ee09f0354..edaa11f96ce 100644 --- a/tools/appliance/cks/ubuntu/22.04/cks-ubuntu-2204.json +++ b/tools/appliance/cks/ubuntu/22.04/cks-ubuntu-2204.json @@ -46,6 +46,8 @@ "scripts/apt_upgrade.sh", "scripts/configure_networking.sh", "scripts/configure-cloud-init.sh", + "scripts/setup-interfaces.sh", + "scripts/add-interface-rule.sh", "scripts/cleanup.sh" ], "type": "shell" diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/add-interface-rule.sh b/tools/appliance/cks/ubuntu/22.04/scripts/add-interface-rule.sh new file mode 100644 index 00000000000..a21d978012e --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/scripts/add-interface-rule.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# File and rule definition +RULE_FILE="/etc/udev/rules.d/90-new-interface.rules" +RULE='ACTION=="add|change|remove", SUBSYSTEM=="net", DRIVERS=="?*", RUN+="/bin/systemctl --no-block start update-netplan.service"' + +# Ensure the file exists, or create it +if [[ ! -f $RULE_FILE ]]; then + touch "$RULE_FILE" + echo "Created $RULE_FILE." +fi + +# Check if the rule already exists to prevent duplication +if grep -Fxq "$RULE" "$RULE_FILE"; then + echo "Rule already exists in $RULE_FILE." +else + # Add the rule to the file + echo "$RULE" | tee -a "$RULE_FILE" > /dev/null + echo "Rule added to $RULE_FILE." +fi + +# Reload udev rules and apply the changes +udevadm control --reload-rules +udevadm trigger +echo "Udev rules reloaded and triggered." \ No newline at end of file diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/configure-cloud-init.sh b/tools/appliance/cks/ubuntu/22.04/scripts/configure-cloud-init.sh index 80b661f30f5..4e4979c936f 100644 --- a/tools/appliance/cks/ubuntu/22.04/scripts/configure-cloud-init.sh +++ b/tools/appliance/cks/ubuntu/22.04/scripts/configure-cloud-init.sh @@ -22,6 +22,8 @@ function install_packages() { apt-get install -y python3-json-pointer python3-jsonschema cloud-init resolvconf sudo mkdir -p /etc/apt/keyrings + echo "Creating /opt/bin directory" + sudo mkdir -p /opt/bin curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg echo \ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/setup-interfaces.sh b/tools/appliance/cks/ubuntu/22.04/scripts/setup-interfaces.sh new file mode 100644 index 00000000000..61983d75dd8 --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/scripts/setup-interfaces.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Create the script in the /opt/bin directory +SCRIPT_PATH="/usr/local/bin/update-netplan.sh" + +cat <<'EOF' > $SCRIPT_PATH +#!/bin/bash + +echo "New interface detected: $INTERFACE" >> /var/log/interface-events.log +CONFIG_FILE="/etc/netplan/config.yaml" + +# Generate a new netplan configuration +echo "network:" > $CONFIG_FILE +echo " ethernets:" >> $CONFIG_FILE + +# Loop through all available interfaces +for iface in $(ls /sys/class/net | grep -vE '^lo$'); do +cat <> $CONFIG_FILE + $iface: + dhcp4: true +EOL +done + +chmod 600 $CONFIG_FILE + +netplan apply +EOF + +tee /etc/systemd/system/update-netplan.service <kubectl and kubeconfig file to access cluster", "label.use.local.timezone": "Use local timezone", +"label.use.router.ip.resolver": "Use Virtual Router IP as resolver", "label.used": "Used", "label.usehttps": "Use HTTPS", "label.usenewdiskoffering": "Replace disk offering?", diff --git a/ui/src/views/auth/Login.vue b/ui/src/views/auth/Login.vue index 47e2f34c57e..8610900f271 100644 --- a/ui/src/views/auth/Login.vue +++ b/ui/src/views/auth/Login.vue @@ -97,6 +97,18 @@ + + + + +