Further improvements to CKS (#118)

* Multiple nics support on Ubuntu template

* Multiple nics support on Ubuntu template

* supports allocating IP to the nic when VM is added to another network - no delay

* Add option to select DNS or VR IP as resolver on VPC creation

* Add API param and UI to select option

* Add column on vpc and pass the value on the databags for CsDhcp.py to fix accordingly

* Externalize the CKS Configuration, so that end users can tweak the configuration before deploying the cluster

* Add new directory to c8 packaging for CKS config

* Remove k8s configuration from resources and make it configurable

* Revert "Remove k8s configuration from resources and make it configurable"

This reverts commit d5997033ebe4ba559e6478a64578b894f8e7d3db.

* copy conf to mgmt server and consume them from there

* Remove node from cluster

* Add missing /opt/bin directory requrired by external nodes

* Login to a specific Project view

* add indents

* Fix CKS HA clusters

* Fix build

---------

Co-authored-by: Nicolas Vazquez <nicovazquez90@gmail.com>
This commit is contained in:
Pearl Dsilva 2025-05-28 06:03:20 +05:30 committed by GitHub
parent f0130d2ebd
commit 138ed60076
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 278 additions and 48 deletions

View File

@ -105,4 +105,6 @@ public interface Vpc extends ControlledEntity, Identity, InternalIdentity {
String getIp6Dns1();
String getIp6Dns2();
boolean useRouterIpAsResolver();
}

View File

@ -48,17 +48,17 @@ public interface VpcService {
* @param vpcName
* @param displayText
* @param cidr
* @param networkDomain TODO
* @param networkDomain TODO
* @param ip4Dns1
* @param ip4Dns2
* @param displayVpc TODO
* @param displayVpc TODO
* @param useVrIpResolver
* @return
* @throws ResourceAllocationException TODO
*/
Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain,
String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize,
Long asNumber, List<Long> bgpPeerIds)
throws ResourceAllocationException;
Long asNumber, List<Long> bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException;
/**
* Persists VPC record in the database

View File

@ -549,6 +549,7 @@ public class ApiConstants {
public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist";
public static final String USER_SECRET_KEY = "usersecretkey";
public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork";
public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver";
public static final String UPDATE_IN_SEQUENCE = "updateinsequence";
public static final String VALUE = "value";
public static final String VIRTUAL_MACHINE_ID = "virtualmachineid";

View File

@ -16,6 +16,7 @@
// under the License.
package org.apache.cloudstack.api.command.user.vpc;
import org.apache.commons.lang3.BooleanUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.cloudstack.acl.RoleType;
@ -125,6 +126,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd {
@Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, since = "4.20.0", description="the AS Number of the VPC tiers")
private Long asNumber;
@Parameter(name=ApiConstants.USE_VIRTUAL_ROUTER_IP_RESOLVER, type=CommandType.BOOLEAN,
description="(optional) for NSX based VPCs: when set to true, use the VR IP as nameserver, otherwise use DNS1 and DNS2")
private Boolean useVrIpResolver;
// ///////////////////////////////////////////////////
// ///////////////// Accessors ///////////////////////
// ///////////////////////////////////////////////////
@ -205,6 +210,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd {
return asNumber;
}
public boolean getUseVrIpResolver() {
return BooleanUtils.toBoolean(useVrIpResolver);
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

8
debian/rules vendored
View File

@ -70,6 +70,7 @@ override_dh_auto_install:
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/lib
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/setup
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm
mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf
mkdir $(DESTDIR)/var/log/$(PACKAGE)/management
mkdir $(DESTDIR)/var/cache/$(PACKAGE)/management
mkdir $(DESTDIR)/var/log/$(PACKAGE)/ipallocator
@ -83,6 +84,7 @@ override_dh_auto_install:
cp client/target/cloud-client-ui-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/cloudstack-$(VERSION).jar
cp client/target/lib/*jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/
cp -r engine/schema/dist/systemvm-templates/* $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/
cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/
rm -rf $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/md5sum.txt
# Bundle cmk in cloudstack-management
@ -95,6 +97,12 @@ override_dh_auto_install:
chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE)
install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/etcd-node.yml
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node.yml
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node-add.yml
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-node.yml
# Remove configuration in /ur/share/cloudstack-management/webapps/client/WEB-INF
# This should all be in /etc/cloudstack/management
ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf

View File

@ -105,6 +105,9 @@ public class VpcVO implements Vpc {
@Column(name = "ip6Dns2")
String ip6Dns2;
@Column(name = "use_router_ip_resolver")
boolean useRouterIpResolver = false;
@Transient
boolean rollingRestart = false;
@ -309,4 +312,13 @@ public class VpcVO implements Vpc {
public String getIp6Dns2() {
return ip6Dns2;
}
@Override
public boolean useRouterIpAsResolver() {
return useRouterIpResolver;
}
public void setUseRouterIpResolver(boolean useRouterIpResolver) {
this.useRouterIpResolver = useRouterIpResolver;
}
}

View File

@ -68,6 +68,9 @@ ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_templa
-- Add for_cks column to the user_data table to represent CNI Configuration stored as userdata
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_data','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the userdata represent CNI configuration meant for CKS use only"');
-- Add use VR IP as resolver option on VPC
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc','use_router_ip_resolver', 'tinyint(1) DEFAULT 0 COMMENT "use router ip as resolver instead of dns options"');
-----------------------------------------------------------
-- END - CKS Enhancements
-----------------------------------------------------------

View File

@ -248,6 +248,7 @@ cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf
mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management
mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management
mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d
@ -273,7 +274,7 @@ wget https://github.com/apache/cloudstack-cloudmonkey/releases/download/$CMK_REL
chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk
cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup
cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf
cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/
cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp
cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/
@ -308,6 +309,11 @@ touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
#install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml
install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml
# SystemVM template
mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
@ -608,6 +614,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
%attr(0755,root,root) %{_bindir}/%{name}-sysvmadm
%attr(0755,root,root) %{_bindir}/%{name}-setup-encryption
%attr(0755,root,root) %{_bindir}/cmk
%{_datadir}/%{name}-management/cks/conf/*.yml
%{_datadir}/%{name}-management/setup/*.sql
%{_datadir}/%{name}-management/setup/*.sh
%{_datadir}/%{name}-management/setup/server-setup.xml

View File

@ -21,6 +21,9 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -143,6 +146,7 @@ public class KubernetesClusterActionWorker {
public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup";
public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes";
public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks";
protected Logger logger = LogManager.getLogger(getClass());
@ -264,6 +268,11 @@ public class KubernetesClusterActionWorker {
return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), com.cloud.utils.StringUtils.getPreferredCharset());
}
protected String readK8sConfigFile(String resource) throws IOException {
Path path = Paths.get(String.format("%s%s", CKS_CONFIG_PATH, resource));
return Files.readString(path);
}
protected String getControlNodeLoginUser() {
List<KubernetesClusterVmMapVO> vmMapVOList = getKubernetesClusterVMMaps();
if (!vmMapVOList.isEmpty()) {
@ -316,7 +325,7 @@ public class KubernetesClusterActionWorker {
}
protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster,
final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
final List<UserVm> clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException {
logMessage(logLevel, message, e);
stateTransitTo(kubernetesCluster.getId(), event);
detachIsoKubernetesVMs(clusterVMs);
@ -670,14 +679,14 @@ public class KubernetesClusterActionWorker {
try {
String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'",
scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
Account account = accountDao.findById(kubernetesCluster.getAccountId());
if (account != null && account.getType() == Account.Type.PROJECT) {
String projectId = projectService.findByProjectAccountId(account.getId()).getUuid();
command = String.format("%s -p '%s'", command, projectId);
}
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 60000);
pkFile, null, command, 10000, 10000, 60000);
return result.first();
} catch (Exception e) {
String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName());
@ -696,7 +705,7 @@ public class KubernetesClusterActionWorker {
writer.close();
} catch (IOException e) {
logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to fetch script %s",
kubernetesCluster.getName(), filename), e);
kubernetesCluster.getName(), filename), e);
}
return file;
}
@ -719,11 +728,11 @@ public class KubernetesClusterActionWorker {
sshKeyFile = getManagementServerSshPublicKeyFile();
}
SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
"~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000);
"~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000);
// Ensure destination dir scriptPath exists and copy file to destination
String cmdStr = String.format("sudo mkdir -p %s ; sudo mv ~/%s %s/%s", scriptPath, file.getName(), scriptPath, destination);
SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
cmdStr, 10000, 10000, 10 * 60 * 1000);
cmdStr, 10000, 10000, 10 * 60 * 1000);
} catch (Exception e) {
throw new CloudRuntimeException(e);
}
@ -771,7 +780,7 @@ public class KubernetesClusterActionWorker {
// Since the provider creates IP addresses, don't deploy it unless the underlying network supports it
if (manager.isDirectAccess(network)) {
logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network",
kubernetesCluster.getName()), null);
kubernetesCluster.getName()), null);
return true;
}
File pkFile = getManagementServerSshPublicKeyFile();
@ -782,7 +791,7 @@ public class KubernetesClusterActionWorker {
try {
String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename);
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 60000);
pkFile, null, command, 10000, 10000, 60000);
// Maybe the file isn't present. Try and copy it
if (!result.first()) {
@ -792,12 +801,12 @@ public class KubernetesClusterActionWorker {
if (!createCloudStackSecret(keys)) {
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s",
kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
}
// If at first you don't succeed ...
result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 60000);
pkFile, null, command, 10000, 10000, 60000);
if (!result.first()) {
throw new CloudRuntimeException(result.second());
}
@ -871,7 +880,7 @@ public class KubernetesClusterActionWorker {
}
public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso, final boolean mountCksIsoOnVR) throws IOException {
String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
String k8sNodeConfig = readK8sConfigFile("/conf/k8s-node.yml");
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";

View File

@ -41,6 +41,7 @@ import javax.inject.Inject;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
@ -95,7 +96,7 @@ public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker
continue;
}
try {
removeNodeVmFromCluster(nodeId, vm.getDisplayName(), publicIp.getAddress().addr());
removeNodeVmFromCluster(nodeId, vm.getDisplayName().toLowerCase(Locale.ROOT), publicIp.getAddress().addr());
result &= removeNodePortForwardingRules(nodeId, network, vm);
if (System.currentTimeMillis() > removeNodeTimeoutTime) {
logger.error(String.format("Removal of node %s from Kubernetes cluster %s timed out", vm.getName(), kubernetesCluster.getName()));

View File

@ -142,9 +142,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
}
private Pair<String, String> getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp,
final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported,
final boolean ejectIso, final boolean externalCni) throws IOException {
String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml");
final List<Network.IpAddresses> etcdIps, final String hostName, final boolean haSupported,
final boolean ejectIso, final boolean externalCni) throws IOException {
String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml");
final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}";
final String apiServerKey = "{{ k8s_control_node.apiserver.key }}";
final String caCert = "{{ k8s_control_node.ca.crt }}";
@ -161,6 +161,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
final String certSans = "{{ k8s_control.server_ips }}";
final String k8sCertificate = "{{ k8s_control.certificate_key }}";
final String externalCniPlugin = "{{ k8s.external.cni.plugin }}";
final String isHaCluster = "{{ k8s.ha.cluster }}";
final String publicIP = "{{ k8s.public.ip }}";
final List<String> addresses = new ArrayList<>();
addresses.add(controlNodeIp);
@ -170,7 +172,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
boolean externalEtcd = !etcdIps.isEmpty();
final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes",
"kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"),
"kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"),
addresses, 3650, null);
final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate());
final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey());
@ -202,7 +204,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
CLUSTER_API_PORT,
KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
}
initArgs += String.format("--apiserver-cert-extra-sans=%s", controlNodeIp);
initArgs += String.format("--apiserver-cert-extra-sans=%s", String.join(",", addresses));
initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion());
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs);
k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
@ -212,6 +214,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp));
k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni));
k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1));
k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress);
k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
@ -301,7 +305,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
}
private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml");
String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node-add.yml");
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
@ -309,6 +313,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
final String ejectIsoKey = "{{ k8s.eject.iso }}";
final String installWaitTime = "{{ k8s.install.wait.time }}";
final String installReattemptsCount = "{{ k8s.install.reattempts.count }}";
final String isHaCluster = "{{ k8s.ha.cluster }}";
final String publicIP = "{{ k8s.public.ip }}";
final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value();
final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value();
@ -328,6 +334,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster));
k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
k8sControlNodeConfig = k8sControlNodeConfig.replace(isHaCluster, String.valueOf(kubernetesCluster.getControlNodeCount() > 1));
k8sControlNodeConfig = k8sControlNodeConfig.replace(publicIP, publicIpAddress);
k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig);
return k8sControlNodeConfig;
@ -336,13 +344,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
private String getInitialEtcdClusterDetails(List<String> ipAddresses, List<String> hostnames) {
String initialCluster = "%s=http://%s:%s";
StringBuilder clusterInfo = new StringBuilder();
for (int i = 0; i < ipAddresses.size(); i++) {
clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT));
if (i < ipAddresses.size()-1) {
clusterInfo.append(",");
}
for (int i = 0; i < ipAddresses.size(); i++) {
clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT));
if (i < ipAddresses.size()-1) {
clusterInfo.append(",");
}
return clusterInfo.toString();
}
return clusterInfo.toString();
}
/**
@ -373,7 +381,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
private String getEtcdNodeConfig(final List<String> ipAddresses, final List<String> hostnames, final int etcdNodeIndex,
final boolean ejectIso) throws IOException {
String k8sEtcdNodeConfig = readResourceFile("/conf/etcd-node.yml");
String k8sEtcdNodeConfig = readK8sConfigFile("/conf/etcd-node.yml");
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
final String ejectIsoKey = "{{ k8s.eject.iso }}";
final String installWaitTime = "{{ k8s.install.wait.time }}";
@ -426,7 +434,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix);
String k8sControlNodeConfig = null;
try {
k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
k8sControlNodeConfig = getKubernetesAdditionalControlNodeConfig(publicIpAddress, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
} catch (IOException e) {
logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e);
}
@ -576,7 +584,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
}
private List<Network.IpAddresses> getEtcdNodeGuestIps(final Network network, final long etcdNodeCount) {
List<Network.IpAddresses> guestIps = new ArrayList<>();
List<Network.IpAddresses> guestIps = new ArrayList<>();
for (int i = 1; i <= etcdNodeCount; i++) {
guestIps.add(new Network.IpAddresses(ipAddressManager.acquireGuestIpAddress(network, null), null));
}

View File

@ -225,6 +225,9 @@ write_files:
exit 0
fi
HA_CLUSTER={{ k8s.ha.cluster }}
CLUSTER_PUBLIC_IP={{ k8s.public.ip }}
if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
echo "setup-kube-system is running!"
exit 1
@ -242,6 +245,10 @@ write_files:
cp -i /etc/kubernetes/admin.conf /root/.kube/config
chown $(id -u):$(id -g) /root/.kube/config
if [[ "$HA_CLUSTER" == "true" ]]; then
sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config
fi
sudo touch /home/cloud/success
echo "true" > /home/cloud/success

View File

@ -275,6 +275,9 @@ write_files:
echo "Already provisioned!"
exit 0
fi
HA_CLUSTER={{ k8s.ha.cluster }}
CLUSTER_PUBLIC_IP={{ k8s.public.ip }}
if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
export PATH=$PATH:/opt/bin
@ -319,6 +322,10 @@ write_files:
cp -i /etc/kubernetes/admin.conf /root/.kube/config
chown $(id -u):$(id -g) /root/.kube/config
echo export PATH=\$PATH:/opt/bin >> /root/.bashrc
if [[ "$HA_CLUSTER" == "true" ]]; then
sed -i -E "s|(server:\\s*).*|\\1https://${CLUSTER_PUBLIC_IP}:6443|" /root/.kube/config
fi
if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then
### Network, dashboard configs available offline ###

View File

@ -301,7 +301,7 @@ public class NetworkMigrationManagerImpl implements NetworkMigrationManager {
copyOfVpc = _vpcService.createVpc(vpc.getZoneId(), vpcOfferingId, vpc.getAccountId(), vpc.getName(),
vpc.getDisplayText(), vpc.getCidr(), vpc.getNetworkDomain(), vpc.getIp4Dns1(), vpc.getIp4Dns2(),
vpc.getIp6Dns1(), vpc.getIp6Dns2(), vpc.isDisplay(), vpc.getPublicMtu(), null, null, null);
vpc.getIp6Dns1(), vpc.getIp6Dns2(), vpc.isDisplay(), vpc.getPublicMtu(), null, null, null, vpc.useRouterIpAsResolver());
copyOfVpcId = copyOfVpc.getId();
//on resume of migration the uuid will be swapped already. So the copy will have the value of the original vpcid.

View File

@ -2072,6 +2072,7 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
* service, we need to override the DHCP response to return DNS server
* rather than virtual router itself.
*/
boolean useRouterIpResolver = getUseRouterIpAsResolver(router);
if (dnsProvided || dhcpProvided) {
if (defaultDns1 != null) {
buf.append(" dns1=").append(defaultDns1);
@ -2093,6 +2094,9 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
if (useExtDns) {
buf.append(" useextdns=true");
}
if (useRouterIpResolver) {
buf.append(" userouteripresolver=true");
}
}
if (Boolean.TRUE.equals(ExposeDnsAndBootpServer.valueIn(dc.getId()))) {
@ -2132,6 +2136,18 @@ Configurable, StateListener<VirtualMachine.State, VirtualMachine.Event, VirtualM
return true;
}
private boolean getUseRouterIpAsResolver(DomainRouterVO router) {
if (router == null || router.getVpcId() == null) {
return false;
}
Vpc vpc = _vpcDao.findById(router.getVpcId());
if (vpc == null) {
logger.warn(String.format("Cannot find VPC with ID %s from router %s", router.getVpcId(), router.getName()));
return false;
}
return vpc.useRouterIpAsResolver();
}
/**
* @param routerLogrotateFrequency The string to be checked if matches with any acceptable values.
* Checks if the value in the global configuration is an acceptable value to be informed to the Virtual Router.

View File

@ -1145,7 +1145,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
@ActionEvent(eventType = EventTypes.EVENT_VPC_CREATE, eventDescription = "creating vpc", create = true)
public Vpc createVpc(final long zoneId, final long vpcOffId, final long vpcOwnerId, final String vpcName, final String displayText, final String cidr, String networkDomain,
final String ip4Dns1, final String ip4Dns2, final String ip6Dns1, final String ip6Dns2, final Boolean displayVpc, Integer publicMtu,
final Integer cidrSize, final Long asNumber, final List<Long> bgpPeerIds) throws ResourceAllocationException {
final Integer cidrSize, final Long asNumber, final List<Long> bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException {
final Account caller = CallContext.current().getCallingAccount();
final Account owner = _accountMgr.getAccount(vpcOwnerId);
@ -1247,6 +1247,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
vpcOff.isRedundantRouter(), ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2);
vpc.setPublicMtu(publicMtu);
vpc.setDisplay(Boolean.TRUE.equals(displayVpc));
vpc.setUseRouterIpResolver(Boolean.TRUE.equals(useVrIpResolver));
if (vpc.getCidr() == null && cidrSize != null) {
// Allocate a CIDR for VPC
@ -1305,7 +1306,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis
List<Long> bgpPeerIds = (cmd instanceof CreateVPCCmdByAdmin) ? ((CreateVPCCmdByAdmin)cmd).getBgpPeerIds() : null;
Vpc vpc = createVpc(cmd.getZoneId(), cmd.getVpcOffering(), cmd.getEntityOwnerId(), cmd.getVpcName(), cmd.getDisplayText(),
cmd.getCidr(), cmd.getNetworkDomain(), cmd.getIp4Dns1(), cmd.getIp4Dns2(), cmd.getIp6Dns1(),
cmd.getIp6Dns2(), cmd.isDisplay(), cmd.getPublicMtu(), cmd.getCidrSize(), cmd.getAsNumber(), bgpPeerIds);
cmd.getIp6Dns2(), cmd.isDisplay(), cmd.getPublicMtu(), cmd.getCidrSize(), cmd.getAsNumber(), bgpPeerIds, cmd.getUseVrIpResolver());
String sourceNatIP = cmd.getSourceNatIP();
boolean forNsx = isVpcForNsx(vpc);

View File

@ -492,7 +492,7 @@ public class VpcManagerImplTest {
try {
doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc);
manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain,
ip4Dns[0], null, null, null, true, 1500, null, null, null);
ip4Dns[0], null, null, null, true, 1500, null, null, null, false);
} catch (ResourceAllocationException e) {
Assert.fail(String.format("failure with exception: %s", e.getMessage()));
}
@ -504,7 +504,7 @@ public class VpcManagerImplTest {
try {
doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc);
manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain,
ip4Dns[0], ip4Dns[1], ip6Dns[0], null, true, 1500, null, null, null);
ip4Dns[0], ip4Dns[1], ip6Dns[0], null, true, 1500, null, null, null, false);
} catch (ResourceAllocationException e) {
Assert.fail(String.format("failure with exception: %s", e.getMessage()));
}
@ -519,7 +519,7 @@ public class VpcManagerImplTest {
try {
doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc);
manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain,
ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null);
ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null, false);
} catch (ResourceAllocationException e) {
Assert.fail(String.format("failure with exception: %s", e.getMessage()));
}
@ -536,7 +536,7 @@ public class VpcManagerImplTest {
try {
doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc);
manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain,
ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null);
ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null, false);
} catch (ResourceAllocationException e) {
Assert.fail(String.format("failure with exception: %s", e.getMessage()));
}
@ -559,7 +559,7 @@ public class VpcManagerImplTest {
try {
doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc);
manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, null, vpcDomain,
ip4Dns[0], ip4Dns[1], null, null, true, 1500, 24, null, bgpPeerIds);
ip4Dns[0], ip4Dns[1], null, null, true, 1500, 24, null, bgpPeerIds, false);
} catch (ResourceAllocationException e) {
Assert.fail(String.format("failure with exception: %s", e.getMessage()));
}

View File

@ -114,6 +114,9 @@ class CsConfig(object):
def expose_dns(self):
return self.cmdline().idata().get('exposedns', 'false') == 'true'
def use_router_ip_as_resolver(self):
return self.cl.get_use_router_ip_as_resolver()
def get_dns(self):
conf = self.cmdline().idata()
dns = []
@ -123,9 +126,10 @@ class CsConfig(object):
else:
dns.append(self.address().get_guest_ip())
for name in ('dns1', 'dns2'):
if name in conf:
dns.append(conf[name])
if not 'userouteripresolver' in conf:
for name in ('dns1', 'dns2'):
if name in conf:
dns.append(conf[name])
return dns
def get_format(self):

View File

@ -176,6 +176,11 @@ class CsCmdLine(CsDataBag):
return self.idata()['useextdns']
return False
def get_use_router_ip_as_resolver(self):
if "userouteripresolver" in self.idata():
return self.idata()['userouteripresolver']
return False
def get_advert_int(self):
if 'advert_int' in self.idata():
return self.idata()['advert_int']

View File

@ -38,6 +38,9 @@ class CsGuestNetwork:
if not self.guest:
return self.config.get_dns()
if self.config.use_router_ip_as_resolver():
return [self.data['router_guest_ip']]
dns = []
if 'router_guest_gateway' in self.data and not self.config.use_extdns() and ('is_vr_guest_gateway' not in self.data or not self.data['is_vr_guest_gateway']):
dns.append(self.data['router_guest_gateway'])

View File

@ -46,6 +46,8 @@
"scripts/apt_upgrade.sh",
"scripts/configure_networking.sh",
"scripts/configure-cloud-init.sh",
"scripts/setup-interfaces.sh",
"scripts/add-interface-rule.sh",
"scripts/cleanup.sh"
],
"type": "shell"

View File

@ -0,0 +1,25 @@
#!/bin/bash
# File and rule definition
RULE_FILE="/etc/udev/rules.d/90-new-interface.rules"
RULE='ACTION=="add|change|remove", SUBSYSTEM=="net", DRIVERS=="?*", RUN+="/bin/systemctl --no-block start update-netplan.service"'
# Ensure the file exists, or create it
if [[ ! -f $RULE_FILE ]]; then
touch "$RULE_FILE"
echo "Created $RULE_FILE."
fi
# Check if the rule already exists to prevent duplication
if grep -Fxq "$RULE" "$RULE_FILE"; then
echo "Rule already exists in $RULE_FILE."
else
# Add the rule to the file
echo "$RULE" | tee -a "$RULE_FILE" > /dev/null
echo "Rule added to $RULE_FILE."
fi
# Reload udev rules and apply the changes
udevadm control --reload-rules
udevadm trigger
echo "Udev rules reloaded and triggered."

View File

@ -22,6 +22,8 @@ function install_packages() {
apt-get install -y python3-json-pointer python3-jsonschema cloud-init resolvconf
sudo mkdir -p /etc/apt/keyrings
echo "Creating /opt/bin directory"
sudo mkdir -p /opt/bin
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \

View File

@ -0,0 +1,47 @@
#!/bin/bash
# Create the script in the /opt/bin directory
SCRIPT_PATH="/usr/local/bin/update-netplan.sh"
cat <<'EOF' > $SCRIPT_PATH
#!/bin/bash
echo "New interface detected: $INTERFACE" >> /var/log/interface-events.log
CONFIG_FILE="/etc/netplan/config.yaml"
# Generate a new netplan configuration
echo "network:" > $CONFIG_FILE
echo " ethernets:" >> $CONFIG_FILE
# Loop through all available interfaces
for iface in $(ls /sys/class/net | grep -vE '^lo$'); do
cat <<EOL >> $CONFIG_FILE
$iface:
dhcp4: true
EOL
done
chmod 600 $CONFIG_FILE
netplan apply
EOF
tee /etc/systemd/system/update-netplan.service <<EOF
[Unit]
Description=Update netplan configuration on boot
After=network.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/update-netplan.sh
[Install]
WantedBy=multi-user.target
EOF
chmod 600 /etc/netplan/config.yaml
chmod 777 $SCRIPT_PATH
systemctl daemon-reload || true
systemctl enable update-netplan.service || true
systemctl start update-netplan.service || true

View File

@ -2488,6 +2488,7 @@
"label.usagetypedescription": "Usage description",
"label.use.kubectl.access.cluster": "<code><b>kubectl</b></code> and <code><b>kubeconfig</b></code> file to access cluster",
"label.use.local.timezone": "Use local timezone",
"label.use.router.ip.resolver": "Use Virtual Router IP as resolver",
"label.used": "Used",
"label.usehttps": "Use HTTPS",
"label.usenewdiskoffering": "Replace disk offering?",

View File

@ -97,6 +97,18 @@
</template>
</a-input>
</a-form-item>
<a-form-item ref="project" name="project">
<a-input
size="large"
type="text"
:placeholder="$t('label.project')"
v-model:value="form.project"
>
<template #prefix>
<block-outlined />
</template>
</a-input>
</a-form-item>
</a-tab-pane>
<a-tab-pane key="saml" :disabled="idps.length === 0">
<template #tab>
@ -230,7 +242,8 @@ export default {
loginType: 0
},
server: '',
forgotPasswordEnabled: false
forgotPasswordEnabled: false,
project: null
}
},
created () {
@ -255,7 +268,8 @@ export default {
this.form = reactive({
server: (this.server.apiHost || '') + this.server.apiBase,
username: this.$route.query?.username || '',
domain: this.$route.query?.domain || ''
domain: this.$route.query?.domain || '',
project: null
})
this.rules = reactive({})
this.setRules()
@ -447,7 +461,7 @@ export default {
})
})
},
loginSuccess (res) {
async loginSuccess (res) {
this.$notification.destroy()
this.$store.commit('SET_COUNT_NOTIFY', 0)
if (store.getters.twoFaEnabled === true && store.getters.twoFaProvider !== '' && store.getters.twoFaProvider !== undefined) {
@ -456,9 +470,33 @@ export default {
this.$router.push({ path: '/setup2FA' }).catch(() => {})
} else {
this.$store.commit('SET_LOGIN_FLAG', true)
const values = toRaw(this.form)
if (values.project) {
await this.getProject(values.project)
this.$store.dispatch('ProjectView', this.project.id)
this.$store.dispatch('SetProject', this.project)
this.$store.dispatch('ToggleTheme', this.project.id === undefined ? 'light' : 'dark')
}
this.$router.push({ path: '/dashboard' }).catch(() => {})
}
},
getProject (projectName) {
return new Promise((resolve, reject) => {
api('listProjects', {
response: 'json',
domainId: this.selectedDomain,
details: 'min'
}).then((response) => {
const projects = response.listprojectsresponse.project
this.project = projects.filter(project => project.name === projectName)?.[0] || null
resolve(this.project)
}).catch((error) => {
this.$notifyError(error)
}).finally(() => {
this.loading = false
})
})
},
requestFailed (err) {
if (err && err.response && err.response.data && err.response.data.loginresponse) {
const error = err.response.data.loginresponse.errorcode + ': ' + err.response.data.loginresponse.errortext

View File

@ -142,7 +142,15 @@
<div style="color: red" v-if="errorPublicMtu" v-html="errorPublicMtu"></div>
</a-form-item>
</div>
<a-row :gutter="12" v-if="selectedVpcOfferingSupportsDns">
<div v-if="isNsxNetwork">
<a-form-item name="userouteripresolver" ref="userouteripresolver">
<template #label>
<tooltip-label :title="$t('label.use.router.ip.resolver')" :tooltip="apiParams.userouteripresolver.description"/>
</template>
<a-switch v-model:checked="useRouterIpResolver" />
</a-form-item>
</div>
<a-row :gutter="12" v-if="selectedVpcOfferingSupportsDns && !useRouterIpResolver">
<a-col :md="12" :lg="12">
<a-form-item v-if="'dns1' in apiParams" name="dns1" ref="dns1">
<template #label>
@ -240,7 +248,8 @@ export default {
isNsxNetwork: false,
asNumberLoading: false,
asNumbersZone: [],
selectedAsNumber: 0
selectedAsNumber: 0,
useRouterIpResolver: false
}
},
beforeCreate () {
@ -459,6 +468,9 @@ export default {
if ('asnumber' in values && this.isASNumberRequired()) {
params.asnumber = values.asnumber
}
if (this.useRouterIpResolver) {
params.userouteripresolver = true
}
this.loading = true
const title = this.$t('label.add.vpc')
const description = this.$t('message.success.add.vpc')