etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CKS Enhancements - Phase 2: Add and Remove external nodes to and from a kubernetes cluster 

---------

Co-authored-by: nvazquez <nicovazquez90@gmail.com>
This commit is contained in:
Pearl Dsilva 2024-04-10 09:47:56 -04:00 committed by nvazquez
parent 4982b1b059
commit 20c40298e6
No known key found for this signature in database
GPG Key ID: 656E1BCC8CB54F84
49 changed files with 1918 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java → api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java
View File

@ -44,6 +44,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
AutoscaleRequested,
ScaleUpRequested,
ScaleDownRequested,
AddNodeRequested,
RemoveNodeRequested,
UpgradeRequested,
OperationSucceeded,
OperationFailed,
@ -59,6 +61,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
Stopped("All resources for the Kubernetes cluster are destroyed, Kubernetes cluster may still have ephemeral resource like persistent volumes provisioned"),
Scaling("Transient state in which resources are either getting scaled up/down"),
Upgrading("Transient state in which cluster is getting upgraded"),
Importing("Transient state in which additional nodes are added as worker nodes to a cluster"),
RemovingNodes("Transient state in which additional nodes are removed from a cluster"),
Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."),
Recovering("State in which Kubernetes cluster is recovering from alert state"),
Destroyed("End state of Kubernetes cluster in which all resources are destroyed, cluster will not be usable further"),
@ -96,6 +100,17 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running);
s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert);
s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing);
s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing);
s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running);
s_fsm.addTransition(State.Importing, Event.OperationFailed, State.Running);
s_fsm.addTransition(State.Alert, Event.OperationSucceeded, State.Running);
s_fsm.addTransition(State.Running, Event.RemoveNodeRequested, State.RemovingNodes);
s_fsm.addTransition(State.Alert, Event.RemoveNodeRequested, State.RemovingNodes);
s_fsm.addTransition(State.RemovingNodes, Event.OperationSucceeded, State.Running);
s_fsm.addTransition(State.RemovingNodes, Event.OperationFailed, State.Running);
s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering);
s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running);
s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert);

2
api/src/main/java/com/cloud/network/NetworkService.java
View File

@ -263,4 +263,6 @@ public interface NetworkService {
InternalLoadBalancerElementService getInternalLoadBalancerElementByNetworkServiceProviderId(long networkProviderId);
InternalLoadBalancerElementService getInternalLoadBalancerElementById(long providerId);
List<InternalLoadBalancerElementService> getInternalLoadBalancerElements();
boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException;
}

19
api/src/main/java/com/cloud/template/TemplateApiService.java
View File

@ -58,10 +58,23 @@ public interface TemplateApiService {
VirtualMachineTemplate prepareTemplate(long templateId, long zoneId, Long storageId);
/**
* Detach ISO from VM
* @param vmId id of the VM
* @param isoId id of the ISO (when passed). If it is not passed, it will get it from user_vm table
* @param extraParams forced, isVirtualRouter
* @return true when operation succeeds, false if not
*/
boolean detachIso(long vmId, Long isoId, Boolean... extraParams);
boolean detachIso(long vmId, boolean forced);
boolean attachIso(long isoId, long vmId, boolean forced);
/**
* Attach ISO to a VM
* @param isoId id of the ISO to attach
* @param vmId id of the VM to attach the ISO to
* @param extraParams: forced, isVirtualRouter
* @return true when operation succeeds, false if not
*/
boolean attachIso(long isoId, long vmId, Boolean... extraParams);
/**
* Deletes a template

3
api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java
View File

@ -81,7 +81,8 @@ public enum ApiCommandResourceType {
ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class),
ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class),
Bucket(org.apache.cloudstack.storage.object.Bucket.class),
QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class);
QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class),
KubernetesCluster(com.cloud.kubernetes.cluster.KubernetesCluster.class);
private final Class<?> clazz;

3
api/src/main/java/org/apache/cloudstack/api/ApiConstants.java
View File

@ -305,6 +305,7 @@ public class ApiConstants {
public static final String MIGRATIONS = "migrations";
public static final String MEMORY = "memory";
public static final String MODE = "mode";
public static final String MOUNT_CKS_ISO_ON_VR = "mountcksisoonvr";
public static final String NSX_MODE = "nsxmode";
public static final String NSX_ENABLED = "isnsxenabled";
public static final String NAME = "name";
@ -1051,6 +1052,8 @@ public class ApiConstants {
public static final String NODE_IDS = "nodeids";
public static final String CONTROL_NODES = "controlnodes";
public static final String ETCD_NODES = "etcdnodes";
public static final String EXTERNAL_NODES = "externalnodes";
public static final String IS_EXTERNAL_NODE = "isexternalnode";
public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion";
public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid";
public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize";

2
api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java
View File

@ -104,7 +104,7 @@ public class DetachIsoCmd extends BaseAsyncCmd implements UserCmd {
@Override
public void execute() {
boolean result = _templateService.detachIso(virtualMachineId, isForced());
boolean result = _templateService.detachIso(virtualMachineId, null, isForced());
if (result) {
UserVm userVm = _entityMgr.findById(UserVm.class, virtualMachineId);
UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", userVm).get(0);

9
api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java
View File

@ -46,6 +46,11 @@ public class UpdateTemplateCmd extends BaseUpdateTemplateOrIsoCmd implements Use
@Parameter(name = ApiConstants.TEMPLATE_TAG, type = CommandType.STRING, description = "the tag for this template.", since = "4.20.0")
private String templateTag;
@Parameter(name = ApiConstants.FOR_CKS, type = CommandType.BOOLEAN,
description = "indicates that the template can be used for deployment of CKS clusters",
since = "4.20.0")
private Boolean forCks;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -63,6 +68,10 @@ public class UpdateTemplateCmd extends BaseUpdateTemplateOrIsoCmd implements Use
return templateTag;
}
public boolean getForCks() {
return Boolean.TRUE.equals(forCks);
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////

24
api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java Normal file
View File

@ -0,0 +1,24 @@
package org.apache.cloudstack.api.response;
import com.cloud.network.router.VirtualRouter;
import com.cloud.serializer.Param;
import com.cloud.uservm.UserVm;
import com.cloud.vm.VirtualMachine;
import com.google.gson.annotations.SerializedName;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.EntityReference;
@EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class})
public class KubernetesUserVmResponse extends UserVmResponse {
@SerializedName(ApiConstants.IS_EXTERNAL_NODE)
@Param(description = "If the VM is an externally added node")
private boolean isExternalNode;
public boolean isExternalNode() {
return isExternalNode;
}
public void setExternalNode(boolean externalNode) {
isExternalNode = externalNode;
}
}

34
core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java Normal file
View File

@ -0,0 +1,34 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package com.cloud.agent.api;
import com.cloud.agent.api.routing.NetworkElementCommand;
public class HandleCksIsoCommand extends NetworkElementCommand {
private boolean mountCksIso;
public HandleCksIsoCommand(boolean mountCksIso) {
this.mountCksIso = mountCksIso;
}
public boolean isMountCksIso() {
return mountCksIso;
}
}

3
core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java
View File

@ -81,4 +81,7 @@ public class VRScripts {
public static final String VR_UPDATE_INTERFACE_CONFIG = "update_interface_config.sh";
public static final String ROUTER_FILESYSTEM_WRITABLE_CHECK = "filesystem_writable_check.py";
// CKS ISO mount
public static final String CKS_ISO_MOUNT_SERVE = "cks_iso.sh";
}

12
core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java
View File

@ -34,6 +34,7 @@ import java.util.concurrent.locks.ReentrantLock;
import javax.naming.ConfigurationException;
import com.cloud.agent.api.HandleCksIsoCommand;
import com.cloud.agent.api.routing.UpdateNetworkCommand;
import com.cloud.agent.api.to.IpAddressTO;
import com.cloud.network.router.VirtualRouter;
@ -144,6 +145,10 @@ public class VirtualRoutingResource {
return execute((UpdateNetworkCommand) cmd);
}
if (cmd instanceof HandleCksIsoCommand) {
return execute((HandleCksIsoCommand) cmd);
}
if (_vrAggregateCommandsSet.containsKey(routerName)) {
_vrAggregateCommandsSet.get(routerName).add(cmd);
aggregated = true;
@ -171,6 +176,13 @@ public class VirtualRoutingResource {
}
}
protected Answer execute(final HandleCksIsoCommand cmd) {
String routerIp = getRouterSshControlIp(cmd);
s_logger.info("Attempting to mount CKS ISO on Virtual Router");
ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.CKS_ISO_MOUNT_SERVE, String.valueOf(cmd.isMountCksIso()));
return new Answer(cmd, result.isSuccess(), result.getDetails());
}
private Answer execute(final SetupKeyStoreCommand cmd) {
final String args = String.format("/usr/local/cloud/systemvm/conf/agent.properties " +
"/usr/local/cloud/systemvm/conf/%s " +

2
engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java
View File

@ -72,4 +72,6 @@ public interface FirewallRulesDao extends GenericDao<FirewallRuleVO, Long> {
void loadSourceCidrs(FirewallRuleVO rule);
void loadDestinationCidrs(FirewallRuleVO rule);
FirewallRuleVO findByNetworkIdAndPorts(long networkId, int startPort, int endPort);
}

17
engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java
View File

@ -48,6 +48,7 @@ public class FirewallRulesDaoImpl extends GenericDaoBase<FirewallRuleVO, Long> i
protected final SearchBuilder<FirewallRuleVO> NotRevokedSearch;
protected final SearchBuilder<FirewallRuleVO> ReleaseSearch;
protected SearchBuilder<FirewallRuleVO> VmSearch;
protected SearchBuilder<FirewallRuleVO> FirewallByPortsAndNetwork;
protected final SearchBuilder<FirewallRuleVO> SystemRuleSearch;
protected final GenericSearchBuilder<FirewallRuleVO, Long> RulesByIpCount;
@ -104,6 +105,12 @@ public class FirewallRulesDaoImpl extends GenericDaoBase<FirewallRuleVO, Long> i
RulesByIpCount.and("ipAddressId", RulesByIpCount.entity().getSourceIpAddressId(), Op.EQ);
RulesByIpCount.and("state", RulesByIpCount.entity().getState(), Op.EQ);
RulesByIpCount.done();
FirewallByPortsAndNetwork = createSearchBuilder();
FirewallByPortsAndNetwork.and("networkId", FirewallByPortsAndNetwork.entity().getNetworkId(), Op.EQ);
FirewallByPortsAndNetwork.and("sourcePortStart", FirewallByPortsAndNetwork.entity().getSourcePortStart(), Op.EQ);
FirewallByPortsAndNetwork.and("sourcePortEnd", FirewallByPortsAndNetwork.entity().getSourcePortEnd(), Op.EQ);
FirewallByPortsAndNetwork.done();
}
@Override
@ -386,4 +393,14 @@ public class FirewallRulesDaoImpl extends GenericDaoBase<FirewallRuleVO, Long> i
rule.setDestinationCidrsList(destCidrs);
}
@Override
public FirewallRuleVO findByNetworkIdAndPorts(long networkId, int startPort, int endPort) {
SearchCriteria<FirewallRuleVO> sc = FirewallByPortsAndNetwork.create();
sc.setParameters("networkId", networkId);
sc.setParameters("sourcePortStart", startPort);
sc.setParameters("sourcePortEnd", endPort);
return findOneBy(sc);
}
}

2
engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java
View File

@ -47,4 +47,6 @@ public interface PortForwardingRulesDao extends GenericDao<PortForwardingRuleVO,
PortForwardingRuleVO findByIdAndIp(long id, String secondaryIp);
List<PortForwardingRuleVO> listByNetworkAndDestIpAddr(String ip4Address, long networkId);
PortForwardingRuleVO findByNetworkAndPorts(long networkId, int startPort, int endPort);
}

11
engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java
View File

@ -54,6 +54,8 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase<PortForwardingRul
AllFieldsSearch.and("vmId", AllFieldsSearch.entity().getVirtualMachineId(), Op.EQ);
AllFieldsSearch.and("purpose", AllFieldsSearch.entity().getPurpose(), Op.EQ);
AllFieldsSearch.and("dstIp", AllFieldsSearch.entity().getDestinationIpAddress(), Op.EQ);
AllFieldsSearch.and("sourcePortStart", AllFieldsSearch.entity().getSourcePortStart(), Op.EQ);
AllFieldsSearch.and("sourcePortEnd", AllFieldsSearch.entity().getSourcePortEnd(), Op.EQ);
AllFieldsSearch.done();
ApplicationSearch = createSearchBuilder();
@ -170,4 +172,13 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase<PortForwardingRul
sc.setParameters("dstIp", secondaryIp);
return findOneBy(sc);
}
@Override
public PortForwardingRuleVO findByNetworkAndPorts(long networkId, int startPort, int endPort) {
SearchCriteria<PortForwardingRuleVO> sc = AllFieldsSearch.create();
sc.setParameters("networkId", networkId);
sc.setParameters("sourcePortStart", startPort);
sc.setParameters("sourcePortEnd", endPort);
return findOneBy(sc);
}
}

1
engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql
View File

@ -336,6 +336,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_templat
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_template_id', 'bigint unsigned COMMENT "template id to be used for Worker Node(s)"');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_template_id', 'bigint unsigned COMMENT "template id to be used for etcd Nodes"');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','etcd_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the VM is an etcd node"');
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','external_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node was imported into the Kubernetes cluster"');
ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_service_offering_id` FOREIGN KEY `fk_cluster__control_service_offering_id`(`control_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE;
ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_service_offering_id` FOREIGN KEY `fk_cluster__worker_service_offering_id`(`worker_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE;

2
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java
View File

@ -23,4 +23,6 @@ public class KubernetesClusterEventTypes {
public static final String EVENT_KUBERNETES_CLUSTER_STOP = "KUBERNETES.CLUSTER.STOP";
public static final String EVENT_KUBERNETES_CLUSTER_SCALE = "KUBERNETES.CLUSTER.SCALE";
public static final String EVENT_KUBERNETES_CLUSTER_UPGRADE = "KUBERNETES.CLUSTER.UPGRADE";
public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD";
public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE";
}

107
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View File

@ -23,10 +23,12 @@ import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClu
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static com.cloud.vm.UserVmManager.AllowUserExpungeRecoverVm;
import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
@ -45,23 +47,33 @@ import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker;
import com.cloud.network.dao.NsxProviderDao;
import com.cloud.network.element.NsxProviderVO;
import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker;
import com.cloud.template.TemplateApiService;
import com.cloud.uservm.UserVm;
import com.cloud.vm.NicVO;
import com.cloud.vm.UserVmService;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.dao.UserVmDetailsDao;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.SecurityChecker;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiConstants.VMDetails;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.ResponseObject.ResponseView;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd;
@ -69,6 +81,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesC
import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd;
import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.KubernetesUserVmResponse;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.api.response.RemoveVirtualMachinesFromKubernetesClusterResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
@ -77,6 +90,7 @@ import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
@ -234,6 +248,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
@Inject
protected UserDao userDao;
@Inject
protected UserVmDetailsDao userVmDetailsDao;
@Inject
protected VMInstanceDao vmInstanceDao;
@Inject
protected UserVmJoinDao userVmJoinDao;
@ -271,9 +287,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
public NetworkHelper networkHelper;
@Inject
private NsxProviderDao nsxProviderDao;
@Inject
private NicDao nicDao;
@Inject
private UserVmService userVmService;
@Inject
private TemplateApiService templateService;
private void logMessage(final Level logLevel, final String message, final Exception e) {
if (logLevel == Level.WARN) {
@ -662,7 +681,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
}
}
List<UserVmResponse> vmResponses = new ArrayList<UserVmResponse>();
List<KubernetesUserVmResponse> vmResponses = new ArrayList<>();
List<KubernetesClusterVmMapVO> vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
ResponseView respView = ResponseView.Restricted;
Account caller = CallContext.current().getCallingAccount();
@ -676,9 +695,17 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
if (userVM != null) {
UserVmResponse vmResponse = ApiDBUtils.newUserVmResponse(respView, responseName, userVM,
EnumSet.of(VMDetails.nics), caller);
vmResponses.add(vmResponse);
KubernetesUserVmResponse kubernetesUserVmResponse = new KubernetesUserVmResponse();
try {
BeanUtils.copyProperties(kubernetesUserVmResponse, vmResponse);
} catch (IllegalAccessException | InvocationTargetException e) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate zone metrics response");
}
kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode());
vmResponses.add(kubernetesUserVmResponse);
}
}
response.setExternalNodes(vmList.stream().filter(KubernetesClusterVmMapVO::isEtcdNode).count());
}
response.setHasAnnotation(annotationDao.hasAnnotations(kubernetesCluster.getUuid(),
AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), accountService.isRootAdmin(caller.getId())));
@ -776,7 +803,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
BaseCmd.getCommandNameByClass(ScaleKubernetesClusterCmd.class),
BaseCmd.getCommandNameByClass(StartKubernetesClusterCmd.class),
BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class),
BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class)
BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class),
BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class),
BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class)
).contains(cmdName);
case ExternalManaged:
return Arrays.asList(
@ -924,7 +953,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
} catch (InvalidParameterValueException e) {
String msg = String.format("Given service offering ID: %s for %s nodes is not suitable for the Kubernetes cluster version %s - %s",
serviceOffering, key, clusterKubernetesVersion, e.getMessage());
LOGGER.error(msg);
logger.error(msg);
throw new InvalidParameterValueException(msg);
}
}
@ -1824,6 +1853,71 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
return true;
}
@Override
public boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd) {
KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId());
long networkId = kubernetesCluster.getNetworkId();
NetworkVO networkVO = networkDao.findById(networkId);
List<Long> validNodeIds = validateNodes(cmd.getNodeIds(), networkId, networkVO.getName(), kubernetesCluster, false);
if (validNodeIds.isEmpty()) {
throw new CloudRuntimeException("No valid nodes found to be added to the Kubernetes cluster");
}
KubernetesClusterAddWorker addWorker = new KubernetesClusterAddWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
addWorker = ComponentContext.inject(addWorker);
return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr());
}
@Override
public boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesClusterCmd cmd) throws Exception {
KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId());
List<Long> validNodeIds = validateNodes(cmd.getNodeIds(), null, null, kubernetesCluster, true);
if (validNodeIds.isEmpty()) {
throw new CloudRuntimeException("No valid nodes found to be removed from the Kubernetes cluster");
}
KubernetesClusterRemoveWorker removeWorker = new KubernetesClusterRemoveWorker(kubernetesCluster, KubernetesClusterManagerImpl.this);
removeWorker = ComponentContext.inject(removeWorker);
return removeWorker.removeNodesFromCluster(validNodeIds);
}
private KubernetesClusterVO validateCluster(long clusterId) {
KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId);
if (kubernetesCluster == null) {
throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified");
}
return kubernetesCluster;
}
private List<Long> validateNodes(List<Long> nodeIds, Long networkId, String networkName, KubernetesCluster cluster, boolean removeNodes) {
List<Long> validNodeIds = new ArrayList<>(nodeIds);
for (Long id : nodeIds) {
VMInstanceVO node = vmInstanceDao.findById(id);
if (Objects.isNull(node)) {
logger.error(String.format("Failed to find node (physical or virtual machine) with ID: %s", id));
validNodeIds.remove(id);
} else if (!removeNodes) {
VMTemplateVO template = templateDao.findById(node.getTemplateId());
if (Objects.isNull(template)) {
logger.error((String.format("Failed to find template with ID: %s", id)));
validNodeIds.remove(id);
} else if (!template.isForCks()) {
logger.error(String.format("Node: %s is deployed with a template that is not marked to be used for CKS", node.getId()));
validNodeIds.remove(id);
}
NicVO nicVO = nicDao.findDefaultNicForVM(id);
if (networkId != nicVO.getNetworkId()) {
logger.error(String.format("Node: %s does not have its default NIC in the kubernetes cluster network: %s", node.getId(), networkName));
validNodeIds.remove(id);
}
List<KubernetesClusterVmMapVO> clusterVmMapVO = kubernetesClusterVmMapDao.listByClusterIdAndVmIdsIn(cluster.getId(), Collections.singletonList(id));
if (Objects.nonNull(clusterVmMapVO) && !clusterVmMapVO.isEmpty()) {
logger.warn(String.format("Node: %s is already part of the cluster %s", node.getId(), cluster.getName()));
validNodeIds.remove(id);
}
}
}
return validNodeIds;
}
@Override
public List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd) {
if (!KubernetesServiceEnabled.value()) {
@ -1898,6 +1992,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
cmdList.add(UpgradeKubernetesClusterCmd.class);
cmdList.add(AddVirtualMachinesToKubernetesClusterCmd.class);
cmdList.add(RemoveVirtualMachinesFromKubernetesClusterCmd.class);
cmdList.add(AddNodesToKubernetesClusterCmd.class);
cmdList.add(RemoveNodesFromKubernetesClusterCmd.class);
return cmdList;
}
@ -2190,6 +2286,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
KubernetesClusterScaleTimeout,
KubernetesClusterUpgradeTimeout,
KubernetesClusterUpgradeRetries,
KubernetesClusterAddNodeTimeout,
KubernetesClusterExperimentalFeaturesEnabled,
KubernetesMaxClusterSize
};

18
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
View File

@ -16,11 +16,13 @@
// under the License.
package com.cloud.kubernetes.cluster;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd;
import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd;
@ -78,6 +80,18 @@ public interface KubernetesClusterService extends PluggableService, Configurable
"The number of retries if fail to upgrade kubernetes cluster due to some reasons (e.g. drain node, etcdserver leader changed)",
true,
KubernetesServiceEnabled.key());
static final ConfigKey<Long> KubernetesClusterAddNodeTimeout = new ConfigKey<Long>("Advanced", Long.class,
"cloud.kubernetes.cluster.add.node.timeout",
"3600",
"Timeout interval (in seconds) in which an external node(VM / baremetal host) addition to a cluster should be completed",
true,
KubernetesServiceEnabled.key());
static final ConfigKey<Long> KubernetesClusterRemoveNodeTimeout = new ConfigKey<Long>("Advanced", Long.class,
"cloud.kubernetes.cluster.add.node.timeout",
"900",
"Timeout interval (in seconds) in which an external node(VM / baremetal host) removal from a cluster should be completed",
true,
KubernetesServiceEnabled.key());
static final ConfigKey<Boolean> KubernetesClusterExperimentalFeaturesEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
"cloud.kubernetes.cluster.experimental.features.enabled",
"false",
@ -118,5 +132,9 @@ public interface KubernetesClusterService extends PluggableService, Configurable
boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd);
boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd);
boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesClusterCmd cmd) throws Exception;
List<RemoveVirtualMachinesFromKubernetesClusterResponse> removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd);
}

11
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java
View File

@ -45,6 +45,9 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap {
@Column(name = "etcd_node")
boolean etcdNode;
@Column(name = "external_node")
boolean externalNode;
public KubernetesClusterVmMapVO() {
}
@ -94,4 +97,12 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap {
public void setEtcdNode(boolean etcdNode) {
this.etcdNode = etcdNode;
}
public boolean isExternalNode() {
return externalNode;
}
public void setExternalNode(boolean externalNode) {
this.externalNode = externalNode;
}
}

272
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
View File

@ -1,19 +1,4 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.kubernetes.cluster.actionworkers;
@ -21,13 +6,17 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import javax.inject.Inject;
@ -36,12 +25,32 @@ import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType;
import com.cloud.network.dao.NetworkVO;
import com.cloud.offering.ServiceOffering;
import com.cloud.exception.ManagementServerException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
import com.cloud.network.firewall.FirewallService;
import com.cloud.network.rules.FirewallRule;
import com.cloud.network.rules.PortForwardingRuleVO;
import com.cloud.network.rules.RulesService;
import com.cloud.network.rules.dao.PortForwardingRulesDao;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.db.TransactionCallbackWithException;
import com.cloud.utils.net.Ip;
import com.cloud.vm.Nic;
import com.cloud.vm.NicVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.dao.NicDao;
import com.cloud.vm.UserVmManager;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@ -151,6 +160,8 @@ public class KubernetesClusterActionWorker {
@Inject
protected UserVmService userVmService;
@Inject
protected UserVmManager userVmManager;
@Inject
protected VlanDao vlanDao;
@Inject
protected VirtualMachineManager itMgr;
@ -160,6 +171,14 @@ public class KubernetesClusterActionWorker {
public ProjectService projectService;
@Inject
public VpcService vpcService;
@Inject
public PortForwardingRulesDao portForwardingRulesDao;
@Inject
protected RulesService rulesService;
@Inject
protected FirewallService firewallService;
@Inject
private NicDao nicDao;
protected KubernetesClusterDao kubernetesClusterDao;
protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@ -180,6 +199,8 @@ public class KubernetesClusterActionWorker {
protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret";
protected final String deployProviderScriptFilename = "deploy-provider";
protected final String autoscaleScriptFilename = "autoscale-kube-cluster";
protected final String validateNodeScript = "validate-cks-node";
protected final String removeNodeFromClusterScript = "remove-node-from-cluster";
protected final String scriptPath = "/opt/bin/";
protected File deploySecretsScriptFile;
protected File deployProviderScriptFile;
@ -318,11 +339,12 @@ public class KubernetesClusterActionWorker {
return new File(keyFile);
}
protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode) {
protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode, boolean isExternalNode) {
return Transaction.execute(new TransactionCallback<KubernetesClusterVmMapVO>() {
@Override
public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) {
KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId, isControlNode);
newClusterVmMap.setExternalNode(isExternalNode);
kubernetesClusterVmMapDao.persist(newClusterVmMap);
return newClusterVmMap;
}
@ -377,6 +399,22 @@ public class KubernetesClusterActionWorker {
return address;
}
protected IpAddress getPublicIp(Network network) throws ManagementServerException {
if (network.getVpcId() != null) {
IpAddress publicIp = getVpcTierKubernetesPublicIp(network);
if (publicIp == null) {
throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName()));
}
return publicIp;
}
IpAddress publicIp = getNetworkSourceNatIp(network);
if (publicIp == null) {
throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s",
network.getName(), kubernetesCluster.getName()));
}
return publicIp;
}
protected IpAddress acquireVpcTierKubernetesPublicIp(Network network) throws
InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException {
IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null);
@ -503,7 +541,7 @@ public class KubernetesClusterActionWorker {
for (UserVm vm : clusterVMs) {
boolean result = false;
try {
result = templateService.detachIso(vm.getId(), true);
result = templateService.detachIso(vm.getId(), null, true);
} catch (CloudRuntimeException ex) {
logger.warn(String.format("Failed to detach binaries ISO from VM : %s in the Kubernetes cluster : %s ", vm.getDisplayName(), kubernetesCluster.getName()), ex);
}
@ -613,12 +651,15 @@ public class KubernetesClusterActionWorker {
copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile, autoscaleScriptFilename);
}
protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String desitnation) {
protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String destination) {
try {
if (Objects.isNull(sshKeyFile)) {
sshKeyFile = getManagementServerSshPublicKeyFile();
}
SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
"~/", file.getAbsolutePath(), "0755");
String cmdStr = String.format("sudo mv ~/%s %s/%s", file.getName(), scriptPath, desitnation);
SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
"~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000);
String cmdStr = String.format("sudo mv ~/%s %s/%s", file.getName(), scriptPath, destination);
SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null,
cmdStr, 10000, 10000, 10 * 60 * 1000);
} catch (Exception e) {
throw new CloudRuntimeException(e);
@ -729,4 +770,191 @@ public class KubernetesClusterActionWorker {
}
return false;
}
protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network network, Account account,
final long vmId, final int sourcePort, final int destPort) throws NetworkRuleConflictException, ResourceUnavailableException {
final long publicIpId = publicIp.getId();
final long networkId = network.getId();
final long accountId = account.getId();
final long domainId = account.getDomainId();
Nic vmNic = networkModel.getNicInNetwork(vmId, networkId);
final Ip vmIp = new Ip(vmNic.getIPv4Address());
PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>) status -> {
PortForwardingRuleVO newRule =
new PortForwardingRuleVO(null, publicIpId,
sourcePort, sourcePort,
vmIp,
destPort, destPort,
"tcp", networkId, accountId, domainId, vmId);
newRule.setDisplay(true);
newRule.setState(FirewallRule.State.Add);
newRule = portForwardingRulesDao.persist(newRule);
return newRule;
});
rulesService.applyPortForwardingRules(publicIp.getId(), account);
if (logger.isInfoEnabled()) {
logger.info(String.format("Provisioned SSH port forwarding rule: %s from port %d to %d on %s to the VM IP : %s in Kubernetes cluster : %s", pfRule.getUuid(), sourcePort, destPort, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName()));
}
}
public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
final String ejectIsoKey = "{{ k8s.eject.iso }}";
final String routerIpKey = "{{ k8s.vr.iso.mounted.ip }}";
NicVO routerNicOnNetwork = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster);
String routerIp = routerNicOnNetwork.getIPv4Address();
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
// if (Objects.isNull(owner)) {
// owner = accountDao.findById(kubernetesCluster.getAccountId());
// }
String sshKeyPair = kubernetesCluster.getKeyPair();
if (StringUtils.isNotEmpty(sshKeyPair)) {
SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair);
if (sshkp != null) {
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
}
}
k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey);
k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp);
k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
k8sNodeConfig = k8sNodeConfig.replace(routerIpKey, routerIp);
k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig);
return k8sNodeConfig;
}
protected String updateKubeConfigWithRegistryDetails(String k8sConfig) {
/* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to
* use docker private registry */
String registryUsername = null;
String registryPassword = null;
String registryUrl = null;
List<KubernetesClusterDetailsVO> details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId());
for (KubernetesClusterDetailsVO detail : details) {
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) {
registryUsername = detail.getValue();
}
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) {
registryPassword = detail.getValue();
}
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) {
registryUrl = detail.getValue();
}
}
if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) {
// Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details
String runCmd = "- bash -x /opt/bin/setup-containerd";
String registryEp = registryUrl.split("://")[1];
k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd);
final String registryUrlKey = "{{registry.url}}";
final String registryUrlEpKey = "{{registry.url.endpoint}}";
final String registryAuthKey = "{{registry.token}}";
final String registryUname = "{{registry.username}}";
final String registryPsswd = "{{registry.password}}";
final String usernamePasswordKey = registryUsername + ":" + registryPassword;
String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl);
k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp);
k8sConfig = k8sConfig.replace(registryUname, registryUsername);
k8sConfig = k8sConfig.replace(registryPsswd, registryPassword);
k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth);
}
return k8sConfig;
}
public Map<Long, Integer> addFirewallRulesForNodes(IpAddress publicIp, int size) throws ManagementServerException {
Map<Long, Integer> vmIdPortMap = new HashMap<>();
try {
List<KubernetesClusterVmMapVO> clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
List<KubernetesClusterVmMapVO> externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList());
int endPort = (CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVmList.size() - externalNodes.size() - 1);
provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort);
if (logger.isInfoEnabled()) {
logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName()));
}
if (!externalNodes.isEmpty()) {
AtomicInteger additionalNodes = new AtomicInteger(1);
externalNodes.forEach(externalNode -> {
int port = endPort + additionalNodes.get();
try {
provisionFirewallRules(publicIp, owner, port, port);
vmIdPortMap.put(externalNode.getVmId(), port);
} catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
throw new CloudRuntimeException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
}
additionalNodes.addAndGet(1);
});
}
} catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
}
return vmIdPortMap;
}
protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException,
IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException {
List<String> sourceCidrList = new ArrayList<String>();
sourceCidrList.add("0.0.0.0/0");
CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd();
rule = ComponentContext.inject(rule);
Field addressField = rule.getClass().getDeclaredField("ipAddressId");
addressField.setAccessible(true);
addressField.set(rule, publicIp.getId());
Field protocolField = rule.getClass().getDeclaredField("protocol");
protocolField.setAccessible(true);
protocolField.set(rule, "TCP");
Field startPortField = rule.getClass().getDeclaredField("publicStartPort");
startPortField.setAccessible(true);
startPortField.set(rule, startPort);
Field endPortField = rule.getClass().getDeclaredField("publicEndPort");
endPortField.setAccessible(true);
endPortField.set(rule, endPort);
Field cidrField = rule.getClass().getDeclaredField("cidrlist");
cidrField.setAccessible(true);
cidrField.set(rule, sourceCidrList);
firewallService.createIngressFirewallRule(rule);
firewallService.applyIngressFwRules(publicIp.getId(), account);
}
protected NicVO getVirtualRouterNicOnKubernetesClusterNetwork(KubernetesCluster kubernetesCluster) {
long networkId = kubernetesCluster.getNetworkId();
NetworkVO kubernetesClusterNetwork = networkDao.findById(networkId);
if (kubernetesClusterNetwork == null) {
logAndThrow(Level.ERROR, String.format("Cannot find network %s set on Kubernetes Cluster %s", networkId, kubernetesCluster.getName()));
}
NicVO routerNicOnNetwork = nicDao.findByNetworkIdAndType(networkId, VirtualMachine.Type.DomainRouter);
if (routerNicOnNetwork == null) {
logAndThrow(Level.ERROR, String.format("Cannot find a Virtual Router on Kubernetes Cluster %s network %s", kubernetesCluster.getName(), kubernetesClusterNetwork.getName()));
}
return routerNicOnNetwork;
}
protected Map<Long, Integer> getVmPortMap() {
List<KubernetesClusterVmMapVO> clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
List<KubernetesClusterVmMapVO> externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList());
Map<Long, Integer> vmIdPortMap = new HashMap<>();
int defaultNodesCount = clusterVmList.size() - externalNodes.size();
AtomicInteger i = new AtomicInteger(0);
externalNodes.forEach(node -> {
vmIdPortMap.put(node.getVmId(), CLUSTER_NODES_DEFAULT_START_SSH_PORT + defaultNodesCount + i.get());
i.addAndGet(1);
});
return vmIdPortMap;
}
}

301
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java Normal file
View File

@ -0,0 +1,301 @@
package com.cloud.kubernetes.cluster.actionworkers;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventVO;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.ManagementServerException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesClusterService;
import com.cloud.kubernetes.cluster.KubernetesClusterVO;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
import com.cloud.network.IpAddress;
import com.cloud.network.Network;
import com.cloud.network.dao.FirewallRulesDao;
import com.cloud.network.rules.FirewallRuleVO;
import com.cloud.network.rules.PortForwardingRuleVO;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.user.Account;
import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.ssh.SshHelper;
import com.cloud.vm.UserVmVO;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.command.user.vm.RebootVMCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Level;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker {
@Inject
private FirewallRulesDao firewallRulesDao;
private long addNodeTimeoutTime;
List<Long> finalNodeIds = new ArrayList<>();
public KubernetesClusterAddWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) {
super(kubernetesCluster, clusterManager);
}
public boolean addNodesToCluster(List<Long> nodeIds, boolean mountCksIsoOnVr) throws CloudRuntimeException {
try {
init();
addNodeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterAddNodeTimeout.value() * 1000;
Long networkId = kubernetesCluster.getNetworkId();
Network network = networkDao.findById(networkId);
if (Objects.isNull(network)) {
throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId));
}
templateDao.findById(kubernetesCluster.getTemplateId());
IpAddress publicIp = null;
try {
publicIp = getPublicIp(network);
} catch (ManagementServerException e) {
throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName()));
}
attachCksIsoForNodesAdditionToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr);
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AddNodeRequested);
Ternary<Integer, Long, Long> nodesAddedAndMemory = importNodeToCluster(nodeIds, network, publicIp, mountCksIsoOnVr);
int nodesAdded = nodesAddedAndMemory.first();
updateKubernetesCluster(kubernetesCluster.getId(), nodesAddedAndMemory);
if (nodeIds.size() != nodesAdded) {
String msg = String.format("Not every node was added to the CKS cluster %s, nodes added: %s out of %s", kubernetesCluster.getUuid(), nodesAdded, nodeIds.size());
LOGGER.info(msg);
detachCksIsoFromNodesAddedToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr);
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
EventVO.LEVEL_ERROR, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD,
msg, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0);
return false;
}
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpSshPort.first(), publicIpSshPort.second(),
getControlNodeLoginUser(), sshKeyFile, addNodeTimeoutTime, 15000);
detachCksIsoFromNodesAddedToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr);
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
String description = String.format("Successfully added %s nodes to Kubernetes Cluster %s", nodesAdded, kubernetesCluster.getUuid());
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD,
description, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0);
return true;
} catch (Exception e) {
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
throw new CloudRuntimeException(e);
}
}
private void detachCksIsoFromNodesAddedToCluster(List<Long> nodeIds, long kubernetesClusterId, boolean mountCksIsoOnVr) {
if (mountCksIsoOnVr) {
detachIsoOnVirtualRouter(kubernetesClusterId);
} else {
LOGGER.info("Detaching CKS ISO from the nodes");
List<UserVm> vms = nodeIds.stream().map(nodeId -> userVmDao.findById(nodeId)).collect(Collectors.toList());
detachIsoKubernetesVMs(vms);
}
}
public void detachIsoOnVirtualRouter(Long kubernetesClusterId) {
KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId);
Long virtualRouterId = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster).getInstanceId();
long isoId = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getIsoId();
try {
networkService.handleCksIsoOnNetworkVirtualRouter(virtualRouterId, false);
} catch (ResourceUnavailableException e) {
String err = String.format("Error trying to handle ISO %s on virtual router %s", isoId, virtualRouterId);
LOGGER.error(err);
throw new CloudRuntimeException(err);
}
try {
templateService.detachIso(virtualRouterId, isoId, true, true);
} catch (CloudRuntimeException e) {
String err = String.format("Error trying to detach ISO %s from virtual router %s", isoId, virtualRouterId);
LOGGER.error(err, e);
}
}
public void attachCksIsoForNodesAdditionToCluster(List<Long> nodeIds, Long kubernetesClusterId, boolean mountCksIsoOnVr) {
if (mountCksIsoOnVr) {
attachAndServeIsoOnVirtualRouter(kubernetesClusterId);
} else {
LOGGER.info("Attaching CKS ISO to the nodes");
List<UserVm> vms = nodeIds.stream().map(nodeId -> userVmDao.findById(nodeId)).collect(Collectors.toList());
attachIsoKubernetesVMs(vms);
}
}
public void attachAndServeIsoOnVirtualRouter(Long kubernetesClusterId) {
KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId);
Long virtualRouterId = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster).getInstanceId();
long isoId = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getIsoId();
try {
templateService.attachIso(isoId, virtualRouterId, true, true);
} catch (CloudRuntimeException e) {
String err = String.format("Error trying to attach ISO %s to virtual router %s", isoId, virtualRouterId);
LOGGER.error(err);
throw new CloudRuntimeException(err);
}
try {
networkService.handleCksIsoOnNetworkVirtualRouter(virtualRouterId, true);
} catch (ResourceUnavailableException e) {
String err = String.format("Error trying to handle ISO %s on virtual router %s", isoId, virtualRouterId);
LOGGER.error(err);
throw new CloudRuntimeException(err);
}
}
private Ternary<Integer, Long, Long> importNodeToCluster(List<Long> nodeIds, Network network, IpAddress publicIp, boolean mountCksIsoOnVr) {
int nodeIndex = 0;
Long additionalMemory = 0L;
Long additionalCores = 0L;
for (Long nodeId : nodeIds) {
UserVmVO vm = userVmDao.findById(nodeId);
String k8sControlNodeConfig = null;
try {
k8sControlNodeConfig = getKubernetesNodeConfig(publicIp.getAddress().addr(), Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()));
} catch (IOException e) {
logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e);
}
if (Objects.isNull(k8sControlNodeConfig)) {
logAndThrow(Level.ERROR, "Error generating worker node configuration");
}
String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
Pair<Boolean, Integer> result = validateAndSetupNode(network, publicIp, owner, nodeId, nodeIndex, base64UserData);
if (Boolean.TRUE.equals(result.first())) {
ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId());
additionalMemory += offeringVO.getRamSize();
additionalCores += offeringVO.getCpu();
String msg = String.format("VM %s added as a node on the Kubernetes Cluster %s", vm.getUuid(), kubernetesCluster.getUuid());
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD,
msg, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), 0);
}
if (Boolean.FALSE.equals(result.first())) {
LOGGER.error(String.format("Failed to add node %s [%s] to Kubernetes cluster : %s", vm.getName(), vm.getUuid(), kubernetesCluster.getName()));
}
if (System.currentTimeMillis() > addNodeTimeoutTime) {
LOGGER.error(String.format("Failed to add node %s to Kubernetes cluster : %s", nodeId, kubernetesCluster.getName()));
}
nodeIndex = result.second();
}
return new Ternary<>(nodeIndex, additionalMemory, additionalCores);
}
private Pair<Boolean, Integer> validateAndSetupNode(Network network, IpAddress publicIp, Account account,
Long nodeId, int nodeIndex, String base64UserData) {
int startSshPortNumber = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount();
int sshStartPort = startSshPortNumber + nodeIndex;
try {
if (Objects.isNull(network.getVpcId())) {
provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort);
}
provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT);
boolean isCompatible = validateNodeCompatibility(publicIp, nodeId, sshStartPort);
if (!isCompatible) {
revertNetworkRules(network, nodeId, sshStartPort);
return new Pair<>(false, nodeIndex);
}
userVmManager.updateVirtualMachine(nodeId, null, null, null, null,
null, base64UserData, null, null, null,
BaseCmd.HTTPMethod.POST, null, null, null, null, null);
RebootVMCmd rebootVMCmd = new RebootVMCmd();
Field idField = rebootVMCmd.getClass().getDeclaredField("id");
idField.setAccessible(true);
idField.set(rebootVMCmd, nodeId);
userVmService.rebootVirtualMachine(rebootVMCmd);
finalNodeIds.add(nodeId);
} catch (ResourceUnavailableException | NetworkRuleConflictException | NoSuchFieldException |
InsufficientCapacityException | IllegalAccessException e) {
LOGGER.error(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()));
// remove added Firewall and PF rules
revertNetworkRules(network, nodeId, sshStartPort);
return new Pair<>( false, nodeIndex);
} catch (Exception e) {
throw new CloudRuntimeException(e);
}
return new Pair<>(true, ++nodeIndex);
}
private void updateKubernetesCluster(long clusterId, Ternary<Integer, Long, Long> additionalNodesDetails) {
int additionalNodeCount = additionalNodesDetails.first();
KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId);
kubernetesClusterVO.setNodeCount(kubernetesClusterVO.getNodeCount() + additionalNodeCount);
kubernetesClusterVO.setMemory(kubernetesClusterVO.getMemory() + additionalNodesDetails.second());
kubernetesClusterVO.setCores(kubernetesClusterVO.getCores() + additionalNodesDetails.third());
kubernetesClusterDao.update(clusterId, kubernetesClusterVO);
kubernetesCluster = kubernetesClusterVO;
finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true));
}
private boolean validateNodeCompatibility(IpAddress publicIp, long nodeId, int nodeSshPort) throws CloudRuntimeException {
File pkFile = getManagementServerSshPublicKeyFile();
try {
File validateNodeScriptFile = retrieveScriptFile(validateNodeScript);
Thread.sleep(15*1000);
copyScriptFile(publicIp.getAddress().addr(), nodeSshPort, validateNodeScriptFile, validateNodeScript);
String command = String.format("%s%s", scriptPath, validateNodeScript);
Pair<Boolean, String> result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 10 * 60 * 1000);
if (Boolean.FALSE.equals(result.first())) {
LOGGER.error(String.format("Node with ID: %s cannot be added as a worker node as it does not have " +
"the following dependencies: %s ", nodeId, result.second()));
return false;
}
} catch (Exception e) {
LOGGER.error(String.format("Failed to validate node with ID: %s", nodeId), e);
return false;
}
UserVmVO userVm = userVmDao.findById(nodeId);
cleanupCloudInitSemFolder(userVm, publicIp, pkFile, nodeSshPort);
return true;
}
private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File pkFile, int nodeSshPort) {
try {
String command = String.format("sudo rm -rf /var/lib/cloud/instances/%s/sem/*", userVm.getUuid());
Pair<Boolean, String> result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 10 * 60 * 1000);
if (Boolean.FALSE.equals(result.first())) {
LOGGER.error(String.format("Failed to cleanup previous applied userdata on node: %s; This may hamper to addition of the node to the cluster ", userVm.getName()));
}
} catch (Exception e) {
LOGGER.error(String.format("Failed to cleanup previous applied userdata on node: %s; This may hamper to addition of the node to the cluster ", userVm.getName()), e);
}
}
private void revertNetworkRules(Network network, long vmId, int port) {
FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), port, port);
if (Objects.isNull(network.getVpcId())) {
firewallService.revokeIngressFirewallRule(ruleVO.getId(), true);
}
List<PortForwardingRuleVO> pfRules = portForwardingRulesDao.listByVm(vmId);
for (PortForwardingRuleVO pfRule : pfRules) {
rulesService.revokePortForwardingRule(pfRule.getId(), true);
}
}
}

2
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java
View File

@ -151,7 +151,7 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod
if (firewallRule == null) {
logMessage(Level.WARN, "Firewall rule for API access can't be removed", null);
}
firewallRule = removeSshFirewallRule(publicIp);
firewallRule = removeSshFirewallRule(publicIp, network.getId());
if (firewallRule == null) {
logMessage(Level.WARN, "Firewall rule for SSH access can't be removed", null);
}

166
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java Normal file
View File

@ -0,0 +1,166 @@
package com.cloud.kubernetes.cluster.actionworkers;
import com.cloud.event.ActionEventUtils;
import com.cloud.event.EventVO;
import com.cloud.exception.ManagementServerException;
import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesClusterService;
import com.cloud.kubernetes.cluster.KubernetesClusterVO;
import com.cloud.network.IpAddress;
import com.cloud.network.Network;
import com.cloud.network.dao.FirewallRulesDao;
import com.cloud.network.rules.FirewallRuleVO;
import com.cloud.network.rules.PortForwardingRuleVO;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.ssh.SshHelper;
import com.cloud.vm.UserVmVO;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.context.CallContext;
import javax.inject.Inject;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker {
@Inject
private FirewallRulesDao firewallRulesDao;
private long removeNodeTimeoutTime;
public KubernetesClusterRemoveWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) {
super(kubernetesCluster, clusterManager);
}
public boolean removeNodesFromCluster(List<Long> nodeIds) {
init();
removeNodeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterAddNodeTimeout.value() * 1000;
Long networkId = kubernetesCluster.getNetworkId();
Network network = networkDao.findById(networkId);
if (Objects.isNull(network)) {
throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId));
}
IpAddress publicIp = null;
try {
publicIp = getPublicIp(network);
} catch (ManagementServerException e) {
throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName()));
}
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RemoveNodeRequested);
boolean result = removeNodesFromCluster(nodeIds, network, publicIp);
if (!result) {
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
} else {
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
}
String description = String.format("Successfully removed %s nodes from the Kubernetes Cluster %s", nodeIds.size(), kubernetesCluster.getUuid());
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE,
description, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0);
return result;
}
private boolean removeNodesFromCluster(List<Long> nodeIds, Network network, IpAddress publicIp) {
boolean result = true;
List<Long> removedNodeIds = new ArrayList<>();
long removedMemory = 0L;
long removedCores = 0L;
for (Long nodeId : nodeIds) {
UserVmVO vm = userVmDao.findById(nodeId);
if (vm == null) {
LOGGER.debug(String.format("Couldn't find a VM with ID %s, skipping removal from Kubernetes cluster", nodeId));
continue;
}
try {
removeNodeVmFromCluster(nodeId, vm.getDisplayName(), publicIp.getAddress().addr());
result &= removeNodePortForwardingRules(nodeId, network, vm);
if (System.currentTimeMillis() > removeNodeTimeoutTime) {
LOGGER.error(String.format("Removal of node %s from Kubernetes cluster %s timed out", vm.getName(), kubernetesCluster.getName()));
result = false;
continue;
}
ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId());
removedNodeIds.add(nodeId);
removedMemory += offeringVO.getRamSize();
removedCores += offeringVO.getCpu();
String description = String.format("Successfully removed the node %s from Kubernetes cluster %s", vm.getUuid(), kubernetesCluster.getUuid());
LOGGER.info(description);
ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(),
EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE,
description, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), 0);
} catch (Exception e) {
String err = String.format("Error trying to remove node %s from Kubernetes Cluster %s: %s", vm.getUuid(), kubernetesCluster.getUuid(), e.getMessage());
LOGGER.error(err, e);
result = false;
}
}
updateKubernetesCluster(kubernetesCluster.getId(), removedNodeIds, removedMemory, removedCores);
return result;
}
protected boolean removeNodePortForwardingRules(Long nodeId, Network network, UserVmVO vm) {
List<PortForwardingRuleVO> pfRules = portForwardingRulesDao.listByVm(nodeId);
boolean result = true;
for (PortForwardingRuleVO pfRule : pfRules) {
try {
result &= rulesService.revokePortForwardingRule(pfRule.getId(), true);
if (Objects.isNull(network.getVpcId())) {
FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd());
result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true);
}
} catch (Exception e) {
String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage());
LOGGER.error(err, e);
}
}
return result;
}
private void removeNodeVmFromCluster(Long nodeId, String nodeName, String publicIp) throws Exception {
File removeNodeScriptFile = retrieveScriptFile(removeNodeFromClusterScript);
copyScriptFile(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, removeNodeScriptFile, removeNodeFromClusterScript);
File pkFile = getManagementServerSshPublicKeyFile();
String command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "remove");
Pair<Boolean, String> result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 10 * 60 * 1000);
if (Boolean.FALSE.equals(result.first())) {
LOGGER.error(String.format("Node: %s failed to be gracefully drained as a worker node from cluster %s ", nodeName, kubernetesCluster.getName()));
}
List<PortForwardingRuleVO> nodePfRules = portForwardingRulesDao.listByVm(nodeId);
Optional<PortForwardingRuleVO> nodeSshPort = nodePfRules.stream().filter(rule -> rule.getDestinationPortStart() == DEFAULT_SSH_PORT
&& rule.getVirtualMachineId() == nodeId && rule.getSourcePortStart() >= CLUSTER_NODES_DEFAULT_START_SSH_PORT).findFirst();
if (nodeSshPort.isPresent()) {
copyScriptFile(publicIp, nodeSshPort.get().getSourcePortStart(), removeNodeScriptFile, removeNodeFromClusterScript);
command = String.format("sudo %s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "worker", "remove");
result = SshHelper.sshExecute(publicIp, nodeSshPort.get().getSourcePortStart(), getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 10 * 60 * 1000);
if (Boolean.FALSE.equals(result.first())) {
LOGGER.error(String.format("Failed to reset node: %s from cluster %s ", nodeName, kubernetesCluster.getName()));
}
command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "delete");
result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(),
pkFile, null, command, 10000, 10000, 10 * 60 * 1000);
if (Boolean.FALSE.equals(result.first())) {
LOGGER.error(String.format("Node: %s failed to be gracefully delete node from cluster %s ", nodeName, kubernetesCluster.getName()));
}
}
}
private void updateKubernetesCluster(long clusterId, List<Long> nodesRemoved, long deallocatedRam, long deallocatedCores) {
KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId);
kubernetesClusterVO.setNodeCount(kubernetesClusterVO.getNodeCount() - nodesRemoved.size());
kubernetesClusterVO.setMemory(kubernetesClusterVO.getMemory() - deallocatedRam);
kubernetesClusterVO.setCores(kubernetesClusterVO.getCores() - deallocatedCores);
kubernetesClusterDao.update(clusterId, kubernetesClusterVO);
nodesRemoved.forEach(id -> kubernetesClusterVmMapDao.removeByClusterIdAndVmIdsIn(clusterId, nodesRemoved));
}
}

133
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java
View File

@ -21,6 +21,7 @@ import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClu
import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.ETCD;
import static com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType.WORKER;
import static com.cloud.utils.NumbersUtil.toHumanReadableSize;
import static com.cloud.utils.db.Transaction.execute;
import java.io.File;
import java.io.IOException;
@ -29,6 +30,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@ -36,9 +38,12 @@ import javax.inject.Inject;
import com.cloud.kubernetes.cluster.KubernetesClusterHelper.KubernetesClusterNodeType;
import com.cloud.network.rules.FirewallManager;
import com.cloud.network.rules.RulesService;
import com.cloud.network.rules.dao.PortForwardingRulesDao;
import com.cloud.offering.NetworkOffering;
import com.cloud.offerings.dao.NetworkOfferingDao;
import org.apache.cloudstack.api.ApiConstants;
import com.cloud.utils.db.TransactionCallbackWithException;
import com.cloud.utils.net.Ip;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
@ -69,23 +74,18 @@ import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.kubernetes.cluster.KubernetesCluster;
import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO;
import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl;
import com.cloud.kubernetes.cluster.KubernetesClusterVO;
import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil;
import com.cloud.network.IpAddress;
import com.cloud.network.Network;
import com.cloud.network.dao.FirewallRulesDao;
import com.cloud.network.dao.LoadBalancerDao;
import com.cloud.network.dao.LoadBalancerVO;
import com.cloud.network.firewall.FirewallService;
import com.cloud.network.lb.LoadBalancingRulesService;
import com.cloud.network.rules.FirewallRule;
import com.cloud.network.rules.FirewallRuleVO;
import com.cloud.network.rules.LoadBalancer;
import com.cloud.network.rules.PortForwardingRuleVO;
import com.cloud.network.rules.RulesService;
import com.cloud.network.rules.dao.PortForwardingRulesDao;
import com.cloud.network.vpc.NetworkACL;
import com.cloud.network.vpc.NetworkACLItem;
import com.cloud.network.vpc.NetworkACLItemDao;
@ -99,16 +99,12 @@ import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.LaunchPermissionDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.user.Account;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.uservm.UserVm;
import com.cloud.utils.Pair;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionCallbackWithException;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.net.Ip;
import com.cloud.utils.net.NetUtils;
import com.cloud.utils.ssh.SshHelper;
import com.cloud.vm.Nic;
@ -131,8 +127,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
@Inject
protected FirewallRulesDao firewallRulesDao;
@Inject
protected FirewallService firewallService;
@Inject
protected NetworkACLService networkACLService;
@Inject
protected NetworkACLItemDao networkACLItemDao;
@ -172,72 +166,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
kubernetesClusterNodeNamePrefix = getKubernetesClusterNodeNamePrefix();
}
private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException {
String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml");
final String sshPubKey = "{{ k8s.ssh.pub.key }}";
final String joinIpKey = "{{ k8s_control_node.join_ip }}";
final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}";
final String ejectIsoKey = "{{ k8s.eject.iso }}";
String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\"";
String sshKeyPair = kubernetesCluster.getKeyPair();
if (StringUtils.isNotEmpty(sshKeyPair)) {
SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair);
if (sshkp != null) {
pubKey += "\n - \"" + sshkp.getPublicKey() + "\"";
}
}
k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey);
k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp);
k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster));
k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso));
k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig);
return k8sNodeConfig;
}
protected String updateKubeConfigWithRegistryDetails(String k8sConfig) {
/* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to
* use docker private registry */
String registryUsername = null;
String registryPassword = null;
String registryUrl = null;
List<KubernetesClusterDetailsVO> details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId());
for (KubernetesClusterDetailsVO detail : details) {
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) {
registryUsername = detail.getValue();
}
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) {
registryPassword = detail.getValue();
}
if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) {
registryUrl = detail.getValue();
}
}
if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) {
// Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details
String runCmd = "- bash -x /opt/bin/setup-containerd";
String registryEp = registryUrl.split("://")[1];
k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd);
final String registryUrlKey = "{{registry.url}}";
final String registryUrlEpKey = "{{registry.url.endpoint}}";
final String registryAuthKey = "{{registry.token}}";
final String registryUname = "{{registry.username}}";
final String registryPsswd = "{{registry.password}}";
final String usernamePasswordKey = registryUsername + ":" + registryPassword;
String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset()));
k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl);
k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp);
k8sConfig = k8sConfig.replace(registryUname, registryUsername);
k8sConfig = k8sConfig.replace(registryPsswd, registryPassword);
k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth);
}
return k8sConfig;
}
protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException {
final int cpu_requested = offering.getCpu() * offering.getSpeed();
final long ram_requested = offering.getRamSize() * 1024L * 1024L;
@ -357,7 +285,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
List<UserVm> nodes = new ArrayList<>();
for (int i = offset + 1; i <= nodeCount; i++) {
UserVm vm = createKubernetesNode(publicIpAddress);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}
@ -469,7 +397,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
final long domainId = account.getDomainId();
Nic vmNic = networkModel.getNicInNetwork(vmId, networkId);
final Ip vmIp = new Ip(vmNic.getIPv4Address());
PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>) status -> {
PortForwardingRuleVO pfRule = execute((TransactionCallbackWithException<PortForwardingRuleVO, NetworkRuleConflictException>) status -> {
PortForwardingRuleVO newRule =
new PortForwardingRuleVO(null, publicIpId,
sourcePort, sourcePort,
@ -501,11 +429,18 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
* @throws NetworkRuleConflictException
*/
protected void provisionSshPortForwardingRules(IpAddress publicIp, Network network, Account account,
List<Long> clusterVMIds) throws ResourceUnavailableException,
List<Long> clusterVMIds, Map<Long, Integer> vmIdPortMap) throws ResourceUnavailableException,
NetworkRuleConflictException {
if (!CollectionUtils.isEmpty(clusterVMIds)) {
for (int i = 0; i < clusterVMIds.size(); ++i) {
provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), CLUSTER_NODES_DEFAULT_START_SSH_PORT + i, DEFAULT_SSH_PORT);
int defaultNodesCount = clusterVMIds.size() - vmIdPortMap.size();
int sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT;
for (int i = 0; i < defaultNodesCount; ++i) {
sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + i;
provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT);
}
for (int i = defaultNodesCount; i < clusterVMIds.size(); ++i) {
sourcePort += 1;
provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT);
}
}
}
@ -524,14 +459,14 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
return rule;
}
protected FirewallRule removeSshFirewallRule(final IpAddress publicIp) {
protected FirewallRule removeSshFirewallRule(final IpAddress publicIp, final long networkId) {
FirewallRule rule = null;
List<FirewallRuleVO> firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall);
for (FirewallRuleVO firewallRule : firewallRules) {
if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) {
PortForwardingRuleVO pfRule = portForwardingRulesDao.findByNetworkAndPorts(networkId, firewallRule.getSourcePortStart(), firewallRule.getSourcePortEnd());
if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT || (Objects.nonNull(pfRule) && pfRule.getDestinationPortStart() == DEFAULT_SSH_PORT) ) {
rule = firewallRule;
firewallService.revokeIngressFwRule(firewallRule.getId(), true);
break;
}
}
return rule;
@ -546,7 +481,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
for (PortForwardingRuleVO pfRule : pfRules) {
if (pfRule.getVirtualMachineId() == vmId) {
portForwardingRulesDao.remove(pfRule.getId());
LOGGER.trace("Marking PF rule " + pfRule + " with Revoke state");
logger.trace("Marking PF rule " + pfRule + " with Revoke state");
pfRule.setState(FirewallRule.State.Revoke);
revokedRules.add(pfRule);
break;
@ -643,19 +578,11 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap, false);
}
protected void createFirewallRules(IpAddress publicIp, List<Long> clusterVMIds, boolean apiRule) throws ManagementServerException {
protected Map<Long, Integer> createFirewallRules(IpAddress publicIp, List<Long> clusterVMIds, boolean apiRule) throws ManagementServerException {
// Firewall rule for SSH access on each node VM
try {
int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1;
provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort);
if (logger.isInfoEnabled()) {
logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName()));
}
} catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
}
Map<Long, Integer> vmIdPortMap = addFirewallRulesForNodes(publicIp, clusterVMIds.size());
if (!apiRule) {
return;
return vmIdPortMap;
}
// Firewall rule for API access for control node VMs
try {
@ -667,6 +594,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
} catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) {
throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
}
return vmIdPortMap;
}
/**
@ -681,11 +609,11 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
* @throws ManagementServerException
*/
protected void setupKubernetesClusterIsolatedNetworkRules(IpAddress publicIp, Network network, List<Long> clusterVMIds, boolean apiRule) throws ManagementServerException {
createFirewallRules(publicIp, clusterVMIds, apiRule);
Map<Long, Integer> vmIdPortMap = createFirewallRules(publicIp, clusterVMIds, apiRule);
// Port forwarding rule for SSH access on each node VM
try {
provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds);
provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap);
} catch (ResourceUnavailableException | NetworkRuleConflictException e) {
throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
}
@ -775,7 +703,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
// Add port forwarding rule for SSH access on each node VM
try {
provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds);
Map<Long, Integer> vmIdPortMap = getVmPortMap();
provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap);
} catch (ResourceUnavailableException | NetworkRuleConflictException e) {
throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
}
@ -802,7 +731,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu
final KubernetesClusterNodeType nodeType,
final boolean updateNodeOffering,
final boolean updateClusterOffering) {
return Transaction.execute(new TransactionCallback<KubernetesClusterVO>() {
return execute(new TransactionCallback<KubernetesClusterVO>() {
@Override
public KubernetesClusterVO doInTransaction(TransactionStatus status) {
KubernetesClusterVO updatedCluster = kubernetesClusterDao.findById(kubernetesCluster.getId());

14
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java
View File

@ -134,7 +134,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
}
// Remove existing SSH firewall rules
FirewallRule firewallRule = removeSshFirewallRule(publicIp);
FirewallRule firewallRule = removeSshFirewallRule(publicIp, network.getId());
if (firewallRule == null) {
throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned");
}
@ -159,7 +159,8 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
}
// Add port forwarding rule for SSH access on each node VM
try {
provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds);
Map<Long, Integer> vmIdPortMap = getVmPortMap();
provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap);
} catch (ResourceUnavailableException | NetworkRuleConflictException e) {
throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e);
}
@ -416,10 +417,11 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
}
List<KubernetesClusterVmMapVO> vmList;
if (this.nodeIds != null) {
vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds);
vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList());
} else {
vmList = getKubernetesClusterVMMaps();
vmList = vmList.subList((int) (kubernetesCluster.getControlNodeCount() + clusterSize), vmList.size());
vmList = vmList.stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList());
vmList = vmList.subList((int) (kubernetesCluster.getControlNodeCount() + clusterSize - 1), vmList.size());
}
Collections.reverse(vmList);
removeNodesFromCluster(vmList);
@ -441,7 +443,9 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif
logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to provision node VM in the cluster", kubernetesCluster.getName()), e);
}
try {
List<Long> clusterVMIds = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList());
List<Long> externalNodeIds = getKubernetesClusterVMMaps().stream().filter(KubernetesClusterVmMapVO::isExternalNode).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList());
List<Long> clusterVMIds = getKubernetesClusterVMMaps().stream().filter(vm -> !vm.isExternalNode()).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList());
clusterVMIds.addAll(externalNodeIds);
scaleKubernetesClusterNetworkRules(clusterVMIds);
} catch (ManagementServerException e) {
logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to update network rules", kubernetesCluster.getName()), e);

4
plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
View File

@ -317,7 +317,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
ManagementServerException, InsufficientCapacityException, ResourceUnavailableException {
UserVm k8sControlVM = null;
k8sControlVM = createKubernetesControlNode(network, publicIpAddress);
addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true);
addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(k8sControlVM);
}
@ -339,7 +339,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif
for (int i = 1; i < kubernetesCluster.getControlNodeCount(); i++) {
UserVm vm = null;
vm = createKubernetesAdditionalControlNode(publicIpAddress, i);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true);
addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false);
if (kubernetesCluster.getNodeRootDiskSize() > 0) {
resizeNodeVolume(vm);
}

132
plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java Normal file
View File

@ -0,0 +1,132 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.user.kubernetes.cluster;
import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes;
import com.cloud.kubernetes.cluster.KubernetesClusterService;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.util.List;
@APICommand(name = "addNodesToKubernetesCluster",
description = "Add nodes as workers to an existing CKS cluster. ",
responseObject = KubernetesClusterResponse.class,
since = "4.20.0",
authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
public class AddNodesToKubernetesClusterCmd extends BaseAsyncCmd {
@Inject
public KubernetesClusterService kubernetesClusterService;
public static final Logger LOGGER = Logger.getLogger(AddNodesToKubernetesClusterCmd.class.getName());
@Parameter(name = ApiConstants.NODE_IDS,
type = CommandType.LIST,
collectionType = CommandType.UUID,
entityType= UserVmResponse.class,
description = "comma separated list of (external) node (physical or virtual machines) IDs that need to be" +
"added as worker nodes to an existing managed Kubernetes cluster (CKS)",
required = true,
since = "4.20.0")
private List<Long> nodeIds;
@Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true,
entityType = KubernetesClusterResponse.class,
description = "the ID of the Kubernetes cluster", since = "4.20.0")
private Long clusterId;
@Parameter(name = ApiConstants.MOUNT_CKS_ISO_ON_VR, type = CommandType.BOOLEAN,
description = "(optional) Vmware only, uses the CKS cluster network VR to mount the CKS ISO",
since = "4.20.0")
private Boolean mountCksIsoOnVr;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public List<Long> getNodeIds() {
return nodeIds;
}
public Long getClusterId() {
return clusterId;
}
public boolean isMountCksIsoOnVr() {
return mountCksIsoOnVr != null && mountCksIsoOnVr;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getEventType() {
return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD;
}
@Override
public String getEventDescription() {
return String.format("Adding %s nodes to the Kubernetes cluster with ID: %s", nodeIds.size(), clusterId);
}
@Override
public void execute() {
try {
if (!kubernetesClusterService.addNodesToKubernetesCluster(this)) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to add node(s) Kubernetes cluster ID: %d", getClusterId()));
}
final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getClusterId());
response.setResponseName(getCommandName());
setResponseObject(response);
} catch (Exception e) {
throw new CloudRuntimeException(String.format("Failed to add nodes to cluster due to: %s", e.getLocalizedMessage()), e);
}
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccount().getId();
}
@Override
public ApiCommandResourceType getApiResourceType() {
return ApiCommandResourceType.KubernetesCluster;
}
@Override
public Long getApiResourceId() {
return getClusterId();
}
}

109
plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java Normal file
View File

@ -0,0 +1,109 @@
package org.apache.cloudstack.api.command.user.kubernetes.cluster;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes;
import com.cloud.kubernetes.cluster.KubernetesClusterService;
import org.apache.cloudstack.acl.RoleType;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
import org.apache.cloudstack.context.CallContext;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.util.List;
@APICommand(name = "removeNodesFromKubernetesCluster",
description = "Removes external nodes from a CKS cluster. ",
responseObject = KubernetesClusterResponse.class,
since = "4.20.0",
authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
public class RemoveNodesFromKubernetesClusterCmd extends BaseAsyncCmd {
@Inject
public KubernetesClusterService kubernetesClusterService;
protected static final Logger LOGGER = Logger.getLogger(RemoveNodesFromKubernetesClusterCmd.class);
@Parameter(name = ApiConstants.NODE_IDS,
type = CommandType.LIST,
collectionType = CommandType.UUID,
entityType= UserVmResponse.class,
description = "comma separated list of node (physical or virtual machines) IDs that need to be" +
"removed from the Kubernetes cluster (CKS)",
required = true,
since = "4.20.0")
private List<Long> nodeIds;
@Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true,
entityType = KubernetesClusterResponse.class,
description = "the ID of the Kubernetes cluster", since = "4.20.0")
private Long clusterId;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public List<Long> getNodeIds() {
return nodeIds;
}
public Long getClusterId() {
return clusterId;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getEventType() {
return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE;
}
@Override
public String getEventDescription() {
return String.format("Removing %s nodes from the Kubernetes Cluster with ID: %s", nodeIds.size(), clusterId);
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
try {
if (!kubernetesClusterService.removeNodesFromKubernetesCluster(this)) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to remove node(s) from Kubernetes cluster ID: %d", getClusterId()));
}
final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getClusterId());
response.setResponseName(getCommandName());
setResponseObject(response);
} catch (Exception e) {
String err = String.format("Failed to remove node(s) from Kubernetes cluster ID: %d due to: %s", getClusterId(), e.getMessage());
LOGGER.error(err, e);
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, err);
}
}
@Override
public long getEntityOwnerId() {
return CallContext.current().getCallingAccount().getId();
}
@Override
public ApiCommandResourceType getApiResourceType() {
return ApiCommandResourceType.KubernetesCluster;
}
@Override
public Long getApiResourceId() {
return getClusterId();
}
}

18
plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java
View File

@ -86,6 +86,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
@Param(description = "the number of the etcd nodes on the Kubernetes cluster")
private Long etcdNodes;
@SerializedName(ApiConstants.EXTERNAL_NODES)
@Param(description = "the number of the externally added worker nodes to the Kubernetes cluster")
private Long externalNodes;
@SerializedName(ApiConstants.TEMPLATE_ID)
@Param(description = "the ID of the template of the Kubernetes cluster")
private String templateId;
@ -165,7 +169,7 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
@SerializedName(ApiConstants.VIRTUAL_MACHINES)
@Param(description = "the list of virtualmachine associated with this Kubernetes cluster")
private List<UserVmResponse> virtualMachines;
private List<KubernetesUserVmResponse> virtualMachines;
@SerializedName(ApiConstants.IP_ADDRESS)
@Param(description = "Public IP Address of the cluster")
@ -443,11 +447,19 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple
this.etcdNodes = etcdNodes;
}
public void setVirtualMachines(List<UserVmResponse> virtualMachines) {
public Long getExternalNodes() {
return externalNodes;
}
public void setExternalNodes(Long externalNodes) {
this.externalNodes = externalNodes;
}
public void setVirtualMachines(List<KubernetesUserVmResponse> virtualMachines) {
this.virtualMachines = virtualMachines;
}
public List<UserVmResponse> getVirtualMachines() {
public List<KubernetesUserVmResponse> getVirtualMachines() {
return virtualMachines;
}

13
plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
View File

@ -76,6 +76,19 @@ write_files:
fi
fi
done <<< "$output"
else
### Download from VR ###
ROUTER_IP="{{ k8s.vr.iso.mounted.ip }}"
echo "Downloading CKS binaries from the VR $ROUTER_IP"
if [ ! -d "${ISO_MOUNT_DIR}" ]; then
mkdir "${ISO_MOUNT_DIR}"
fi
### Download from ROUTER_IP/cks-iso into ISO_MOUNT_DIR
AUX_DOWNLOAD_DIR=/aux-dwnld
mkdir -p $AUX_DOWNLOAD_DIR
wget -r -R "index.html*" $ROUTER_IP/cks-iso -P $AUX_DOWNLOAD_DIR
mv $AUX_DOWNLOAD_DIR/$ROUTER_IP/cks-iso/* $ISO_MOUNT_DIR
rm -rf $AUX_DOWNLOAD_DIR
fi
if [ -d "$BINARIES_DIR" ]; then
break

27
plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash
export PATH=$PATH:/opt/bin
node_name=$1
node_type=$2
operation=$3
if [ $operation == "remove" ]; then
if [ $node_type == "control" ]; then
# get the specific node
kubectl get nodes $node_name >/dev/null 2>&1
if [[ $(echo $?) -eq 1 ]]; then
echo "No node with name $node_name present in the cluster, exiting..."
exit 0
else
# Drain the node
kubectl drain $node_name --delete-local-data --force --ignore-daemonsets
fi
else
kubeadm reset -f
fi
else
sudo mkdir -p /home/cloud/.kube
sudo cp /root/.kube/config /home/cloud/.kube/
sudo chown -R cloud:cloud /home/cloud/.kube
kubectl delete node $node_name
fi

45
plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node Normal file
View File

@ -0,0 +1,45 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
OS=`awk -F= '/^NAME/{print $2}' /etc/os-release`
REQUIRED_PACKAGES=(cloud-init cloud-guest-utils conntrack apt-transport-https ca-certificates curl gnupg gnupg-agent \
software-properties-common gnupg lsb-release python3-json-pointer python3-jsonschema cloud-init containerd.io)
declare -a MISSING_PACKAGES
if [[ $OS == *"Ubuntu"* || $OS == *"Debian"* ]]; then
for package in ${REQUIRED_PACKAGES[@]}; do
dpkg -s $package >/dev/null 2>&1
if [ $? -eq 1 ]; then
MISSING_PACKAGES+="$package"
fi
done
else
for package in ${REQUIRED_PACKAGES[@]}; do
rpm -qa | grep $package >/dev/null 2>&1
if [ $? -eq 1 ]; then
MISSING_PACKAGES[${#MISSING_PACKAGES[@]}]=$package
fi
done
fi
echo ${#MISSING_PACKAGES[@]}
if (( ${#MISSING_PACKAGES[@]} )); then
echo "Following packages are missing in the node template: ${MISSING_PACKAGES[@]}"
exit 1
else
echo 0
fi

21
server/src/main/java/com/cloud/network/NetworkServiceImpl.java
View File

@ -6144,6 +6144,27 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C
return new ArrayList<>(this.internalLoadBalancerElementServiceMap.values());
}
@Override
public boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException {
DomainRouterVO router = routerDao.findById(virtualRouterId);
if (router == null) {
String err = String.format("Cannot find VR with ID %s", virtualRouterId);
s_logger.error(err);
throw new CloudRuntimeException(err);
}
Commands commands = new Commands(Command.OnError.Stop);
commandSetupHelper.createHandleCksIsoCommand(router, mount, commands);
if (!networkHelper.sendCommandsToRouter(router, commands)) {
throw new CloudRuntimeException(String.format("Unable to send commands to virtual router: %s", router.getHostId()));
}
Answer answer = commands.getAnswer("handleCksIso");
if (answer == null || !answer.getResult()) {
s_logger.error(String.format("Could not handle the CKS ISO properly: %s", answer.getDetails()));
return false;
}
return true;
}
/**
* Retrieves the active quarantine for the given public IP address. It can find by the ID of the quarantine or the address of the public IP.
* @throws CloudRuntimeException if it does not find an active quarantine for the given public IP.

8
server/src/main/java/com/cloud/network/router/CommandSetupHelper.java
View File

@ -28,6 +28,7 @@ import java.util.Set;
import javax.inject.Inject;
import com.cloud.agent.api.HandleCksIsoCommand;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
@ -1404,4 +1405,11 @@ public class CommandSetupHelper {
cmds.addCommand("updateNetwork", cmd);
}
}
public void createHandleCksIsoCommand(final VirtualRouter router, final boolean mount, Commands cmds) {
HandleCksIsoCommand command = new HandleCksIsoCommand(mount);
command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId()));
command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName());
cmds.addCommand("handleCksIso", command);
}
}

61
server/src/main/java/com/cloud/template/TemplateManagerImpl.java
View File

@ -34,6 +34,7 @@ import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.vm.VirtualMachine;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.BaseCmd;
@ -1141,35 +1142,33 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Override
@ActionEvent(eventType = EventTypes.EVENT_ISO_DETACH, eventDescription = "detaching ISO", async = true)
public boolean detachIso(long vmId, boolean forced) {
public boolean detachIso(long vmId, Long isoParamId, Boolean... extraParams) {
Account caller = CallContext.current().getCallingAccount();
Long userId = CallContext.current().getCallingUserId();
// Verify input parameters
UserVmVO vmInstanceCheck = _userVmDao.findById(vmId);
if (vmInstanceCheck == null) {
throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId);
}
boolean forced = extraParams != null && extraParams.length > 0 ? extraParams[0] : false;
boolean isVirtualRouter = extraParams != null && extraParams.length > 1 ? extraParams[1] : false;
UserVm userVM = _userVmDao.findById(vmId);
if (userVM == null) {
// Verify input parameters
VirtualMachine virtualMachine = !isVirtualRouter ? _userVmDao.findById(vmId) : _vmInstanceDao.findById(vmId);
if (virtualMachine == null || (isVirtualRouter && virtualMachine.getType() != VirtualMachine.Type.DomainRouter)) {
throw new InvalidParameterValueException("Please specify a valid VM.");
}
_accountMgr.checkAccess(caller, null, true, userVM);
_accountMgr.checkAccess(caller, null, true, virtualMachine);
Long isoId = userVM.getIsoId();
Long isoId = !isVirtualRouter ? ((UserVm) virtualMachine).getIsoId() : isoParamId;
if (isoId == null) {
throw new InvalidParameterValueException("The specified VM has no ISO attached to it.");
}
CallContext.current().setEventDetails("Vm Id: " + userVM.getUuid() + " ISO Id: " + isoId);
CallContext.current().setEventDetails("Vm Id: " + virtualMachine.getUuid() + " ISO Id: " + isoId);
State vmState = userVM.getState();
State vmState = virtualMachine.getState();
if (vmState != State.Running && vmState != State.Stopped) {
throw new InvalidParameterValueException("Please specify a VM that is either Stopped or Running.");
}
boolean result = attachISOToVM(vmId, userId, isoId, false, forced); // attach=false
boolean result = attachISOToVM(vmId, userId, isoId, false, forced, isVirtualRouter); // attach=false
// => detach
if (result) {
return result;
@ -1180,14 +1179,26 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Override
@ActionEvent(eventType = EventTypes.EVENT_ISO_ATTACH, eventDescription = "attaching ISO", async = true)
public boolean attachIso(long isoId, long vmId, boolean forced) {
public boolean attachIso(long isoId, long vmId, Boolean... extraParams) {
Account caller = CallContext.current().getCallingAccount();
Long userId = CallContext.current().getCallingUserId();
boolean forced = extraParams != null && extraParams.length > 0 ? extraParams[0] : false;
boolean isVirtualRouter = extraParams != null && extraParams.length > 1 ? extraParams[1] : false;
// Verify input parameters
UserVmVO vm = _userVmDao.findById(vmId);
VirtualMachine vm = _userVmDao.findById(vmId);
if (vm == null) {
throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId);
if (isVirtualRouter) {
vm = _vmInstanceDao.findById(vmId);
if (vm == null) {
throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId);
} else if (vm.getType() != VirtualMachine.Type.DomainRouter) {
throw new InvalidParameterValueException("Unable to find a virtual router with id " + vmId);
}
} else {
throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId);
}
}
VMTemplateVO iso = _tmpltDao.findById(isoId);
@ -1223,7 +1234,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
if (VMWARE_TOOLS_ISO.equals(iso.getUniqueName()) && vm.getHypervisorType() != Hypervisor.HypervisorType.VMware) {
throw new InvalidParameterValueException("Cannot attach VMware tools drivers to incompatible hypervisor " + vm.getHypervisorType());
}
boolean result = attachISOToVM(vmId, userId, isoId, true, forced);
boolean result = attachISOToVM(vmId, userId, isoId, true, forced, isVirtualRouter);
if (result) {
return result;
} else {
@ -1262,10 +1273,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
}
private boolean attachISOToVM(long vmId, long isoId, boolean attach, boolean forced) {
UserVmVO vm = _userVmDao.findById(vmId);
private boolean attachISOToVM(long vmId, long isoId, boolean attach, boolean forced, boolean isVirtualRouter) {
VirtualMachine vm = !isVirtualRouter ? _userVmDao.findById(vmId) : _vmInstanceDao.findById(vmId);
if (vm == null) {
if (vm == null || (isVirtualRouter && vm.getType() != VirtualMachine.Type.DomainRouter)) {
return false;
} else if (vm.getState() != State.Running) {
return true;
@ -1304,16 +1315,16 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
return (a != null && a.getResult());
}
private boolean attachISOToVM(long vmId, long userId, long isoId, boolean attach, boolean forced) {
private boolean attachISOToVM(long vmId, long userId, long isoId, boolean attach, boolean forced, boolean isVirtualRouter) {
UserVmVO vm = _userVmDao.findById(vmId);
VMTemplateVO iso = _tmpltDao.findById(isoId);
boolean success = attachISOToVM(vmId, isoId, attach, forced);
if (success && attach) {
boolean success = attachISOToVM(vmId, isoId, attach, forced, isVirtualRouter);
if (success && attach && !isVirtualRouter) {
vm.setIsoId(iso.getId());
_userVmDao.update(vmId, vm);
}
if (success && !attach) {
if (success && !attach && !isVirtualRouter) {
vm.setIsoId(null);
_userVmDao.update(vmId, vm);
}
@ -2113,6 +2124,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
Map details = cmd.getDetails();
Account account = CallContext.current().getCallingAccount();
boolean cleanupDetails = cmd.isCleanupDetails();
boolean forCks = cmd instanceof UpdateTemplateCmd && ((UpdateTemplateCmd) cmd).getForCks();
// verify that template exists
VMTemplateVO template = _tmpltDao.findById(id);
@ -2260,6 +2272,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
template.setDetails(details);
_tmpltDao.saveDetails(template);
}
template.setForCks(forCks);
_tmpltDao.update(id, template);

5
server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java
View File

@ -1108,4 +1108,9 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches
public List<InternalLoadBalancerElementService> getInternalLoadBalancerElements() {
return null;
}
@Override
public boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) {
return false;
}
}

34
systemvm/debian/opt/cloud/bin/cks_iso.sh Normal file
View File

@ -0,0 +1,34 @@
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
BASE_DIR=/var/www/html
CKS_ISO_DIR=$BASE_DIR/cks-iso
if [ "$1" == "true" ]
then
mkdir -p $CKS_ISO_DIR
echo "Options +Indexes" > $BASE_DIR/.htaccess
echo "Mounting CKS ISO into $CKS_ISO_DIR"
mount /dev/cdrom $CKS_ISO_DIR
else
echo "Unmounting CKS ISO from $CKS_ISO_DIR"
umount $CKS_ISO_DIR
echo "Options -Indexes" > $BASE_DIR/.htaccess
rm -rf $CKS_ISO_DIR
fi
echo "Restarting apache2 service"
service apache2 restart

3
systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py
View File

@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
from merge import DataBag
from . import CsHelper
class CsGuestNetwork:
@ -40,7 +39,7 @@ class CsGuestNetwork:
return self.config.get_dns()
dns = []
if 'router_guest_gateway' in self.data and not self.config.use_extdns() and 'is_vr_guest_gateway' not in self.data:
if 'router_guest_gateway' in self.data and not self.config.use_extdns() and ('is_vr_guest_gateway' not in self.data or not self.data['is_vr_guest_gateway']):
dns.append(self.data['router_guest_gateway'])
if 'dns' in self.data:

27
tools/appliance/cks/ubuntu/22.04/scripts/setup_template.sh Normal file
View File

@ -0,0 +1,27 @@
#!/bin/bash
function create_user() {
username=$1
password=$2
# Create the user with the specified username
sudo useradd -m -s /bin/bash $username
# Set the user's password
echo "$username:$password" | sudo chpasswd
echo "User '$username' has been created with the password '$password'"
}
sudo mkdir -p /opt/bin
create_user cloud password
echo $SSHKEY
if [[ ! -z "$SSHKEY" ]]; then
mkdir -p /home/cloud/.ssh/
mkdir .ssh
echo $SSHKEY > ~/.ssh/authorized_keys
else
echo "Please place Management server public key in the variables"
exit 1
fi

18
ui/public/locales/en.json
View File

@ -47,6 +47,8 @@
"label.acquiring.ip": "Acquiring IP",
"label.associated.resource": "Associated resource",
"label.action": "Action",
"label.action.add.nodes.to.kubernetes.cluster": "Add nodes to Kubernetes cluster",
"label.action.remove.nodes.from.kubernetes.cluster": "Remove nodes from Kubernetes cluster",
"label.action.attach.disk": "Attach disk",
"label.action.attach.iso": "Attach ISO",
"label.action.bulk.delete.egress.firewall.rules": "Bulk delete egress firewall rules",
@ -245,6 +247,7 @@
"label.add.list.name": "ACL List name",
"label.add.logical.router": "Add Logical Router to this Network",
"label.add.more": "Add more",
"label.add.nodes": "Add Nodes to Kubernetes Cluster",
"label.add.netscaler.device": "Add Netscaler device",
"label.add.network": "Add Network",
"label.add.network.acl": "Add Network ACL",
@ -922,7 +925,7 @@
"label.fix.errors": "Fix errors",
"label.fixed": "Fixed offering",
"label.for": "for",
"label.for.cks": "For CKS",
"label.forcks": "For CKS",
"label.forbidden": "Forbidden",
"label.forced": "Force",
"label.force.stop": "Force stop",
@ -1119,6 +1122,7 @@
"label.ipv6.subnets": "IPv6 Subnets",
"label.ip.addresses": "IP Addresses",
"label.iqn": "Target IQN",
"label.is.base64.encoded": "Base64 encoded",
"label.is.in.progress": "is in progress",
"label.is.shared": "Is shared",
"label.is2faenabled": "Is 2FA enabled",
@ -1176,11 +1180,13 @@
"label.kubernetes": "Kubernetes",
"label.kubernetes.access.details": "The kubernetes nodes can be accessed via ssh using: <br> <code><b> ssh -i [ssh_key] -p [port_number] cloud@[public_ip_address] </b></code> <br><br> where, <br> <code><b>ssh_key:</b></code> points to the ssh private key file corresponding to the key that was associated while creating the Kubernetes cluster. If no ssh key was provided during Kubernetes cluster creation, use the ssh private key of the management server. <br> <code><b>port_number:</b></code> can be obtained from the Port Forwarding Tab (Public Port column)",
"label.kubernetes.cluster": "Kubernetes cluster",
"label.kubernetes.cluster.add.nodes.to.cluster": "Add nodes to Kubernetes cluster",
"label.kubernetes.cluster.create": "Create Kubernetes cluster",
"label.kubernetes.cluster.delete": "Delete Kubernetes cluster",
"label.kubernetes.cluster.scale": "Scale Kubernetes cluster",
"label.kubernetes.cluster.start": "Start Kubernetes cluster",
"label.kubernetes.cluster.stop": "Stop Kubernetes cluster",
"label.kubernetes.cluster.remove.nodes.from.cluster": "Remove nodes from Kubernetes cluster",
"label.kubernetes.cluster.upgrade": "Upgrade Kubernetes cluster",
"label.kubernetes.dashboard": "Kubernetes dashboard UI",
"label.kubernetes.dashboard.create.token": "Create token for Kubernetes dashboard",
@ -1374,6 +1380,7 @@
"label.monitor.url": "URL Path",
"label.monthly": "Monthly",
"label.more.access.dashboard.ui": "More about accessing dashboard UI",
"label.mount.cks.iso.on.vr": "Mount and serve CKS ISO on the CKS cluster's network Virtual Router",
"label.move.down.row": "Move down one row",
"label.move.to.bottom": "Move to bottom",
"label.move.to.top": "Move to top",
@ -1741,6 +1748,7 @@
"label.remove.logical.router": "Remove logical router",
"label.remove.network.offering": "Remove Network offering",
"label.remove.network.route.table": "Remove Tungsten Fabric Network routing table",
"label.remove.nodes": "Remove nodes from Kubernetes cluster",
"label.remove.pf": "Remove port forwarding rule",
"label.remove.policy": "Remove policy",
"label.remove.project.account": "Remove Account from project",
@ -2353,6 +2361,8 @@
"label.vnmc": "VNMC",
"label.volgroup": "Volume group",
"label.volume": "Volume",
"label.vms.empty": "No VMs available to be added to the Kubernetes cluster",
"label.vms.remove.empty": "No external VMs present in the Kubernetes cluster to be removed",
"label.volume.empty": "No data volumes attached to this Instance",
"label.volume.volumefileupload.description": "Click or drag file to this area to upload.",
"label.volume.encryption.support": "Volume Encryption Supported",
@ -2590,6 +2600,8 @@
"message.adding.host": "Adding host",
"message.adding.netscaler.device": "Adding Netscaler device",
"message.adding.netscaler.provider": "Adding Netscaler provider",
"message.adding.nodes.to.cluster": "Adding nodes to Kubernetes cluster",
"message.removing.nodes.from.cluster": "Removing nodes from Kubernetes cluster",
"message.advanced.security.group": "Choose this if you wish to use security groups to provide guest Instance isolation.",
"message.allowed": "Allowed",
"message.alert.show.all.stats.data": "This may return a lot of data depending on VM statistics and retention settings",
@ -3008,10 +3020,12 @@
"message.ip.v6.prefix.delete": "IPv6 prefix deleted",
"message.iso.desc": "Disc image containing data or bootable media for OS.",
"message.kubeconfig.cluster.not.available": "Kubernetes cluster kubeconfig not available currently.",
"message.kubernetes.cluster.add.nodes": "Please confirm that you want to add the following nodes to the cluster",
"message.kubernetes.cluster.delete": "Please confirm that you want to destroy the cluster.",
"message.kubernetes.cluster.scale": "Please select desired cluster configuration.",
"message.kubernetes.cluster.start": "Please confirm that you want to start the cluster.",
"message.kubernetes.cluster.stop": "Please confirm that you want to stop the cluster.",
"message.kubernetes.cluster.remove.nodes": "Please confirm that you want to remove the following nodes from the cluster",
"message.kubernetes.cluster.upgrade": "Please select new Kubernetes version.",
"message.kubernetes.version.delete": "Please confirm that you want to delete this Kubernetes version.",
"message.l2.network.unsupported.for.nsx": "L2 networks aren't supported for NSX enabled zones",
@ -3189,6 +3203,8 @@
"message.success.add.network.acl": "Successfully added Network ACL list",
"message.success.add.network.static.route": "Successfully added Network Static Route",
"message.success.add.network.permissions": "Successfully added Network permissions",
"message.success.add.nodes.to.cluster": "Successfully added nodes to Kubernetes cluster",
"message.success.remove.nodes.from.cluster": "Successfully removed nodes from Kubernetes cluster",
"message.success.add.physical.network": "Successfully added Physical Network",
"message.success.add.object.storage": "Successfully added Object Storage",
"message.success.add.policy.rule": "Successfully added Policy rule",

20
ui/src/config/section/compute.js
View File

@ -601,6 +601,26 @@ export default {
popup: true,
component: shallowRef(defineAsyncComponent(() => import('@/views/compute/UpgradeKubernetesCluster.vue')))
},
{
api: 'addNodesToKubernetesCluster',
icon: 'plus-outlined',
label: 'label.kubernetes.cluster.add.nodes.to.cluster',
message: 'message.kubernetes.cluster.add.nodes',
dataView: true,
show: (record) => { return ['Running', 'Alert'].includes(record.state) && record.clustertype === 'CloudManaged' },
popup: true,
component: shallowRef(defineAsyncComponent(() => import('@/views/compute/KubernetesAddNodes.vue')))
},
{
api: 'removeNodesFromKubernetesCluster',
icon: 'minus-outlined',
label: 'label.kubernetes.cluster.remove.nodes.from.cluster',
message: 'message.kubernetes.cluster.remove.nodes',
dataView: true,
show: (record) => { return ['Running', 'Alert'].includes(record.state) && record.clustertype === 'CloudManaged' && (record.virtualmachines.filter(vm => vm.isexternalnode) || []).length > 0 },
popup: true,
component: shallowRef(defineAsyncComponent(() => import('@/views/compute/KubernetesRemoveNodes.vue')))
},
{
api: 'deleteKubernetesCluster',
icon: 'delete-outlined',

2
ui/src/config/section/image.js
View File

@ -59,7 +59,7 @@ export default {
details: () => {
var fields = ['name', 'id', 'displaytext', 'checksum', 'hypervisor', 'format', 'ostypename', 'size', 'physicalsize', 'isready', 'passwordenabled',
'crossZones', 'templatetype', 'directdownload', 'deployasis', 'ispublic', 'isfeatured', 'isextractable', 'isdynamicallyscalable', 'crosszones', 'type',
'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy']
'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'forcks']
if (['Admin'].includes(store.getters.userInfo.roletype)) {
fields.push('templatetag', 'templatetype', 'url')
}

18
ui/src/utils/plugins.js
View File

@ -390,6 +390,8 @@ export const resourceTypePlugin = {
return 'publicip'
case 'NetworkAcl':
return 'acllist'
case 'KubernetesCluster':
return 'kubernetes'
case 'SystemVm':
case 'PhysicalNetwork':
case 'Backup':
@ -427,6 +429,9 @@ export const resourceTypePlugin = {
var routePath = this.$getRouteFromResourceType(resourceType)
if (!routePath) return ''
var route = this.$router.resolve('/' + routePath)
if (routePath === 'kubernetes') {
return route?.meta?.icon[0]
}
return route?.meta?.icon || ''
}
}
@ -488,6 +493,15 @@ export const fileSizeUtilPlugin = {
}
}
function isBase64 (str) {
try {
const decoded = new TextDecoder().decode(Uint8Array.from(atob(str), c => c.charCodeAt(0)))
return btoa(decoded) === str
} catch (err) {
return false
}
}
export const genericUtilPlugin = {
install (app) {
app.config.globalProperties.$isValidUuid = function (uuid) {
@ -496,8 +510,8 @@ export const genericUtilPlugin = {
}
app.config.globalProperties.$toBase64AndURIEncoded = function (text) {
const base64regex = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/
if (base64regex.test(text)) {
console.log(isBase64(text))
if (isBase64(text)) {
return text
}
return encodeURIComponent(btoa(unescape(encodeURIComponent(text))))

177
ui/src/views/compute/KubernetesAddNodes.vue Normal file
View File

@ -0,0 +1,177 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
<template>
<a-spin :spinning="loading">
<a-form
:model="form"
:ref="formRef"
:rules="rules"
@finish="handleSubmit"
layout="vertical">
<div v-if="vms.length > 0">
<a-form-item name="nodeids" ref="nodeids">
<template #label>
<tooltip-label :title="$t('label.add.nodes')" :tooltip="apiParams.nodeids.description"/>
</template>
<a-select
v-model:value="form.nodeids"
:placeholder="$t('label.add.nodes')"
mode="multiple"
:loading="loading"
showSearch
optionFilterProp="label"
:filterOption="(input, option) => {
return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
}" >
<a-select-option v-for="vm in vms" :key="vm.id" :label="vm.name">
{{ vm.name }}
</a-select-option>
</a-select>
</a-form-item>
<a-form-item name="mountcksiso" ref="mountcksiso">
<a-checkbox v-model:checked="form.mountcksiso">
{{ $t('label.mount.cks.iso.on.vr') }}
</a-checkbox>
</a-form-item>
</div>
<p v-else v-html="$t('label.vms.empty')" />
<div :span="24" class="action-button">
<a-button @click="closeAction">{{ $t('label.cancel') }}</a-button>
<a-button :loading="loading" ref="submit" type="primary" @click="handleSubmit">{{ $t('label.ok') }}</a-button>
</div>
</a-form>
</a-spin>
</template>
<script>
import { ref, reactive, toRaw } from 'vue'
import { api } from '@/api'
import TooltipLabel from '@/components/widgets/TooltipLabel'
export default {
name: 'AddNodesToKubernetesCluster',
components: {
TooltipLabel
},
props: {
resource: {
type: Object,
required: true
}
},
data () {
return {
vms: [],
loading: false
}
},
inject: ['parentFetchData'],
beforeCreate () {
this.apiParams = this.$getApiParams('addNodesToKubernetesCluster')
},
created () {
this.formRef = ref()
this.form = reactive({})
this.rules = reactive({
nodeids: [{ type: 'array' }]
})
this.fetchData()
},
methods: {
fetchData () {
this.fetchVms()
},
async fetchVms () {
this.loading = true
this.vms = await this.callListVms(this.resource.accountid, this.resource.domainid)
const cksVms = this.resource.virtualmachines.map(vm => vm.id)
this.vms = this.vms.filter(vm => !cksVms.includes(vm.id))
this.loading = false
},
callListVms (accountId, domainId) {
return new Promise((resolve) => {
this.volumes = []
api('listVirtualMachines', {
accountId: accountId,
domainId: domainId,
details: 'min',
listall: 'true'
}).then(json => {
const vms = json.listvirtualmachinesresponse.virtualmachine || []
resolve(vms)
})
})
},
closeAction () {
this.$emit('close-action')
},
handleSubmit (e) {
e.preventDefault()
if (this.loading) return
this.formRef.value.validate().then(async () => {
const values = toRaw(this.form)
const params = {
id: this.resource.id
}
if (values.nodeids) {
params.nodeids = values.nodeids.join(',')
}
if (values.mountcksiso) {
params.mountcksisoonvr = values.mountcksiso
}
this.loading = true
try {
const jobId = await this.addNodesToKubernetesCluster(params)
await this.$pollJob({
jobId,
title: this.$t('label.action.add.nodes.to.kubernetes.cluster'),
description: this.resource.name,
loadingMessage: `${this.$t('message.adding.nodes.to.cluster')} ${this.resource.name}`,
catchMessage: this.$t('error.fetching.async.job.result'),
successMessage: `${this.$t('message.success.add.nodes.to.cluster')} ${this.resource.name}`,
successMethod: () => {
this.parentFetchData()
},
action: {
isFetchData: false
}
})
this.closeAction()
this.loading = false
} catch (error) {
await this.$notifyError(error)
this.closeAction()
this.loading = false
}
})
},
addNodesToKubernetesCluster (params) {
return new Promise((resolve, reject) => {
api('addNodesToKubernetesCluster', params).then(json => {
const jobId = json.addnodestokubernetesclusterresponse.jobid
return resolve(jobId)
}).catch(error => {
return reject(error)
})
})
}
}
}
</script>
<style lang="scss" scoped></style>

151
ui/src/views/compute/KubernetesRemoveNodes.vue Normal file
View File

@ -0,0 +1,151 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
<template>
<a-spin :spinning="loading">
<a-form
:model="form"
:ref="formRef"
:rules="rules"
@finish="handleSubmit"
layout="vertical">
<a-form-item v-if="vms.length > 0" name="nodeids" ref="nodeids">
<template #label>
<tooltip-label :title="$t('label.remove.nodes')" :tooltip="apiParams.nodeids.description"/>
</template>
<a-select
v-model:value="form.nodeids"
:placeholder="$t('label.remove.nodes')"
mode="multiple"
:loading="loading"
showSearch
optionFilterProp="label"
:filterOption="(input, option) => {
return option.label.toLowerCase().indexOf(input.toLowerCase()) >= 0
}" >
<a-select-option v-for="vm in vms" :key="vm.id" :label="vm.name">
{{ vm.name }}
</a-select-option>
</a-select>
</a-form-item>
<p v-else v-html="$t('label.vms.remove.empty')" />
<div :span="24" class="action-button">
<a-button @click="closeAction">{{ $t('label.cancel') }}</a-button>
<a-button :loading="loading" ref="submit" type="primary" @click="handleSubmit">{{ $t('label.ok') }}</a-button>
</div>
</a-form>
</a-spin>
</template>
<script>
import { ref, reactive, toRaw } from 'vue'
import { api } from '@/api'
import TooltipLabel from '@/components/widgets/TooltipLabel'
export default {
name: 'AddNodesToKubernetesCluster',
components: {
TooltipLabel
},
props: {
resource: {
type: Object,
required: true
}
},
data () {
return {
vms: [],
loading: false
}
},
inject: ['parentFetchData'],
beforeCreate () {
this.apiParams = this.$getApiParams('removeNodesFromKubernetesCluster')
},
created () {
this.formRef = ref()
this.form = reactive({})
this.rules = reactive({
nodeids: [{ type: 'array' }]
})
this.fetchData()
},
methods: {
fetchData () {
this.fetchClusterVms()
},
async fetchClusterVms () {
this.loading = true
this.vms = this.resource.virtualmachines.filter(vm => vm.isexternalnode === true) || []
this.loading = false
},
closeAction () {
this.$emit('close-action')
},
handleSubmit (e) {
e.preventDefault()
if (this.loading) return
this.formRef.value.validate().then(async () => {
const values = toRaw(this.form)
const params = {
id: this.resource.id
}
if (values.nodeids) {
params.nodeids = values.nodeids.join(',')
}
this.loading = true
try {
const jobId = await this.removeNodesFromKubernetesCluster(params)
await this.$pollJob({
jobId,
title: this.$t('label.action.remove.nodes.from.kubernetes.cluster'),
description: this.resource.name,
loadingMessage: `${this.$t('message.removing.nodes.from.cluster')} ${this.resource.name}`,
catchMessage: this.$t('error.fetching.async.job.result'),
successMessage: `${this.$t('message.success.remove.nodes.from.cluster')} ${this.resource.name}`,
successMethod: () => {
this.parentFetchData()
},
action: {
isFetchData: false
}
})
this.closeAction()
this.loading = false
} catch (error) {
await this.$notifyError(error)
this.closeAction()
this.loading = false
}
})
},
removeNodesFromKubernetesCluster (params) {
return new Promise((resolve, reject) => {
api('removeNodesFromKubernetesCluster', params).then(json => {
const jobId = json.removenodesfromkubernetesclusterresponse.jobid
return resolve(jobId)
}).catch(error => {
return reject(error)
})
})
}
}
}
</script>
<style lang="scss" scoped></style>

9
ui/src/views/compute/RegisterUserData.vue
View File

@ -43,6 +43,9 @@
v-model:value="form.userdata"
:placeholder="apiParams.userdata.description"/>
</a-form-item>
<a-form-item name="isbase64" ref="isbase64" :label="$t('label.is.base64.encoded')">
<a-checkbox v-model:checked="form.isbase64"></a-checkbox>
</a-form-item>
<a-form-item name="params" ref="params">
<template #label>
<tooltip-label :title="$t('label.userdataparams')" :tooltip="apiParams.params.description"/>
@ -147,7 +150,9 @@ export default {
methods: {
initForm () {
this.formRef = ref()
this.form = reactive({})
this.form = reactive({
isbase64: false
})
this.rules = reactive({
name: [{ required: true, message: this.$t('message.error.name') }],
userdata: [{ required: true, message: this.$t('message.error.userdata') }]
@ -204,8 +209,8 @@ export default {
if (this.isValidValueForKey(values, 'account') && values.account.length > 0) {
params.account = values.account
}
params.userdata = this.$toBase64AndURIEncoded(values.userdata)
params.userdata = values.isbase64 ? values.userdata : this.$toBase64AndURIEncoded(values.userdata)
if (values.params != null && values.params.length > 0) {
var userdataparams = values.params.join(',')
params.params = userdataparams

2
ui/src/views/image/RegisterOrUploadTemplate.vue
View File

@ -435,7 +435,7 @@
</a-col>
<a-col :span="12">
<a-checkbox value="forCks" v-if="currentForm === 'Create'">
{{ $t('label.for.cks') }}
{{ $t('label.forcks') }}
</a-checkbox>
</a-col>
</a-row>

8
ui/src/views/image/UpdateTemplate.vue
View File

@ -151,6 +151,12 @@
</template>
<a-switch v-model:checked="form.isdynamicallyscalable" />
</a-form-item>
<a-form-item name="forcks" ref="forcks">
<template #label>
<tooltip-label :title="$t('label.forcks')" :tooltip="apiParams.forcks.description"/>
</template>
<a-switch v-model:checked="form.forcks" />
</a-form-item>
<a-form-item name="templatetype" ref="templatetype" v-if="isAdmin">
<template #label>
<tooltip-label :title="$t('label.templatetype')" :tooltip="apiParams.templatetype.description"/>
@ -254,7 +260,7 @@ export default {
displaytext: [{ required: true, message: this.$t('message.error.required.input') }],
ostypeid: [{ required: true, message: this.$t('message.error.select') }]
})
const resourceFields = ['name', 'displaytext', 'passwordenabled', 'ostypeid', 'isdynamicallyscalable', 'userdataid', 'userdatapolicy']
const resourceFields = ['name', 'displaytext', 'passwordenabled', 'ostypeid', 'isdynamicallyscalable', 'userdataid', 'userdatapolicy', 'forcks']
if (this.isAdmin) {
resourceFields.push('templatetype')
resourceFields.push('templatetag')